use codeberg.org/6543/docker-images/golang_just

Signed-off-by: 6543 <6543@obermui.de>

.gitignore

@@ -4,3 +4,4 @@ key-database.pogreb/
 acme-account.json
 build/
 vendor/
+pages

.woodpecker.yml

@@ -0,0 +1,106 @@
+pipeline:
+  # use vendor to cache dependencies
+  vendor:
+    image: golang:1.18
+    commands:
+      - go mod vendor
+
+  lint:
+    image: golangci/golangci-lint:latest
+    group: compliant
+    pull: true
+    commands:
+      - go version
+      - go install mvdan.cc/gofumpt@latest
+      - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
+      - golangci-lint run --timeout 5m --build-tags integration
+
+  build:
+    group: compliant
+    image: codeberg.org/6543/docker-images/golang_just
+    commands:
+      - go version
+      - just build
+    when:
+      event: [ "pull_request", "push" ]
+
+  docker-dryrun:
+    group: compliant
+    image: plugins/kaniko
+    settings:
+      dockerfile: Dockerfile
+      no_push: true
+      tags: latest
+    when:
+      event: [ "pull_request", "push" ]
+      path: Dockerfile
+
+  build-tag:
+    group: compliant
+    image: codeberg.org/6543/docker-images/golang_just
+    commands:
+      - go version
+      - just build-tag ${CI_COMMIT_TAG##v}
+    when:
+      event: [ "tag" ]
+
+  test:
+    group: test
+    image: codeberg.org/6543/docker-images/golang_just
+    commands:
+      - just test
+
+  integration-tests:
+    group: test
+    image: codeberg.org/6543/docker-images/golang_just
+    commands:
+      - just integration
+    environment:
+      - ACME_API=https://acme.mock.directory
+      - PAGES_DOMAIN=localhost.mock.directory
+      - RAW_DOMAIN=raw.localhost.mock.directory
+      - PORT=4430
+
+  release:
+    image: plugins/gitea-release
+    settings:
+      base_url: https://codeberg.org
+      file_exists: overwrite
+      files: build/codeberg-pages-server
+      api_key:
+        from_secret: bot_token
+    environment:
+      - DRONE_REPO_OWNER=${CI_REPO_OWNER}
+      - DRONE_REPO_NAME=${CI_REPO_NAME}
+      - DRONE_BUILD_EVENT=${CI_BUILD_EVENT}
+      - DRONE_COMMIT_REF=${CI_COMMIT_REF}
+    when:
+      event: [ "tag" ]
+
+  docker-next:
+    image: plugins/kaniko
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      repo: codeberg.org/codeberg/pages-server
+      tags: next
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      event: [ "push" ]
+
+  docker-tag:
+    image: plugins/kaniko
+    settings:
+      registry: codeberg.org
+      dockerfile: Dockerfile
+      repo: codeberg.org/codeberg/pages-server
+      tag: [ latest, "${CI_COMMIT_TAG}" ]
+      username:
+        from_secret: bot_user
+      password:
+        from_secret: bot_token
+    when:
+      event: [ "tag" ]

Dockerfile

@@ -0,0 +1,15 @@
+FROM golang:alpine as build
+
+WORKDIR /workspace
+
+RUN apk add ca-certificates
+COPY . .
+RUN CGO_ENABLED=0 go build .
+
+FROM scratch
+COPY --from=build /workspace/pages /pages
+COPY --from=build \
+    /etc/ssl/certs/ca-certificates.crt \
+    /etc/ssl/certs/ca-certificates.crt
+    
+ENTRYPOINT ["/pages"]

Justfile

@@ -6,7 +6,44 @@ dev:
     export PAGES_DOMAIN=localhost.mock.directory
     export RAW_DOMAIN=raw.localhost.mock.directory
     export PORT=4430
-    go run . --verbose
+    export LOG_LEVEL=trace
+    go run .
 
 build:
     CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
+
+build-tag VERSION:
+    CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./
+
+lint: tool-golangci tool-gofumpt
+    [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
+    golangci-lint run --timeout 5m --build-tags integration
+
+fmt: tool-gofumpt
+    gofumpt -w --extra .
+
+clean:
+    go clean ./...
+    rm -rf build/
+
+tool-golangci:
+    @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \
+    fi
+
+tool-gofumpt:
+    @hash gofumpt> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install mvdan.cc/gofumpt@latest; \
+    fi
+
+test:
+    go test -race codeberg.org/codeberg/pages/server/...
+
+test-run TEST:
+    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/...
+
+integration:
+    go test -race -tags integration codeberg.org/codeberg/pages/integration/...
+
+integration-run TEST:
+    go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

README.md

@@ -1,4 +1,59 @@
-## Environment
+# Codeberg Pages
+
+Gitea lacks the ability to host static pages from Git.
+The Codeberg Pages Server addresses this lack by implementing a standalone service
+that connects to Gitea via API.
+It is suitable to be deployed by other Gitea instances, too, to offer static pages hosting to their users.
+
+**End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
+and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
+
+
+## Quickstart
+
+This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
+Mapping custom domains is not static anymore, but can be done with DNS:
+
+1) add a `.domains` text file to your repository, containing the allowed domains, separated by new lines. The
+first line will be the canonical domain/URL; all other occurrences will be redirected to it.
+
+2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
+"pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
+If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
+	`www.example.org. IN CNAME main.pages.example.codeberg.page.`
+
+3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
+for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
+record that points to your repo (just like the CNAME record):
+	`example.org IN ALIAS codeberg.page.`
+	`example.org IN TXT main.pages.example.codeberg.page.`
+
+Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
+
+
+## Deployment
+
+**Warning: Some Caveats Apply**  
+> Currently, the deployment requires you to have some knowledge of system administration as well as understanding and building code,
+> so you can eventually edit non-configurable and codeberg-specific settings.
+> In the future, we'll try to reduce these and make hosting Codeberg Pages as easy as setting up Gitea.
+> If you consider using Pages in practice, please consider contacting us first,
+> we'll then try to share some basic steps and document the current usage for admins
+> (might be changing in the current state).
+
+Deploying the software itself is very easy. You can grab a current release binary or build yourself,
+configure the environment as described below, and you are done.
+
+The hard part is about adding **custom domain support** if you intend to use it.
+SSL certificates (request + renewal) is automatically handled by the Pages Server,
+but if you want to run it on a shared IP address (and not a standalone),
+you'll need to configure your reverse proxy not to terminate the TLS connections,
+but forward the requests on the IP level to the Pages Server.
+
+You can check out a proof of concept in the `haproxy-sni` folder,
+and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38).
+
+### Environment
 
 - `HOST` & `PORT` (default: `[::]` & `443`): listen address.
 - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
@@ -15,25 +70,40 @@
 - `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
 - `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.  
   See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
-- `DEBUG` (default: false): Set this to true to enable debug logging.
+- `LOG_LEVEL` (default: warn): Set this to specify the level of logging.
 
-```
-// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
-//
-// Mapping custom domains is not static anymore, but can be done with DNS:
-//
-// 1) add a ".domains" text file to your repository, containing the allowed domains, separated by new lines. The
-// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
-//
-// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
-// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
-//      www.example.org. IN CNAME main.pages.example.codeberg.page.
-//
-// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
-// for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
-// record that points to your repo (just like the CNAME record):
-//      example.org IN ALIAS codeberg.page.
-//      example.org IN TXT main.pages.example.codeberg.page.
-//
-// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
-```
+
+## Contributing to the development
+
+The Codeberg team is very open to your contribution.
+Since we are working nicely in a team, it might be hard at times to get started
+(still check out the issues, we always aim to have some things to get you started).
+
+If you have any questions, want to work on a feature or could imagine collaborating with us for some time,
+feel free to ping us in an issue or in a general Matrix chatgroup.
+
+You can also contact the maintainers of this project:
+
+- [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space)
+- [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de)
+
+### First steps
+
+The code of this repository is split in several modules.
+While heavy refactoring work is currently undergo, you can easily understand the basic structure:
+The `cmd` folder holds the data necessary for interacting with the service via the cli.
+If you are considering to deploy the service yourself, make sure to check it out.
+The heart of the software lives in the `server` folder and is split in several modules.
+After scanning the code, you should quickly be able to understand their function and start hacking on them.
+
+Again: Feel free to get in touch with us for any questions that might arise.
+Thank you very much.
+
+
+### Test Server
+
+run `just dev`
+now this pages should work:
+ - https://magiclike.localhost.mock.directory:4430/
+ - https://momar.localhost.mock.directory:4430/ci-testing/
+ - https://momar.localhost.mock.directory:4430/pag/@master/

cmd/certs.go

@@ -2,43 +2,71 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
+	"github.com/akrylysov/pogreb"
 	"github.com/urfave/cli/v2"
 
 	"codeberg.org/codeberg/pages/server/database"
 )
 
 var Certs = &cli.Command{
-	Name:   "certs",
-	Usage:  "manage certs manually",
-	Action: certs,
+	Name:  "certs",
+	Usage: "manage certs manually",
+	Subcommands: []*cli.Command{
+		{
+			Name:   "list",
+			Usage:  "list all certificates in the database",
+			Action: listCerts,
+		},
+		{
+			Name:   "remove",
+			Usage:  "remove a certificate from the database",
+			Action: removeCert,
+		},
+	},
 }
 
-func certs(ctx *cli.Context) error {
-	if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" {
-		if ctx.Args().Len() == 1 {
-			println("--remove-certificate requires at least one domain as an argument")
-			os.Exit(1)
-		}
+func listCerts(ctx *cli.Context) error {
+	// TODO: make "key-database.pogreb" set via flag
+	keyDatabase, err := database.New("key-database.pogreb")
+	if err != nil {
+		return fmt.Errorf("could not create database: %v", err)
+	}
 
-		domains := ctx.Args().Slice()[2:]
-
-		// TODO: make "key-database.pogreb" set via flag
-		keyDatabase, err := database.New("key-database.pogreb")
+	items := keyDatabase.Items()
+	for domain, _, err := items.Next(); err != pogreb.ErrIterationDone; domain, _, err = items.Next() {
 		if err != nil {
-			return fmt.Errorf("could not create database: %v", err)
+			return err
 		}
-
-		for _, domain := range domains {
-			if err := keyDatabase.Delete([]byte(domain)); err != nil {
-				panic(err)
-			}
+		if domain[0] == '.' {
+			fmt.Printf("*")
 		}
-		if err := keyDatabase.Close(); err != nil {
-			panic(err)
-		}
-		os.Exit(0)
+		fmt.Printf("%s\n", domain)
+	}
+	return nil
+}
+
+func removeCert(ctx *cli.Context) error {
+	if ctx.Args().Len() < 1 {
+		return fmt.Errorf("'certs remove' requires at least one domain as an argument")
+	}
+
+	domains := ctx.Args().Slice()
+
+	// TODO: make "key-database.pogreb" set via flag
+	keyDatabase, err := database.New("key-database.pogreb")
+	if err != nil {
+		return fmt.Errorf("could not create database: %v", err)
+	}
+
+	for _, domain := range domains {
+		fmt.Printf("Removing domain %s from the database...\n", domain)
+		if err := keyDatabase.Delete(domain); err != nil {
+			return err
+		}
+	}
+	if err := keyDatabase.Close(); err != nil {
+		return err
 	}
 	return nil
 }

cmd/flags.go

@@ -5,12 +5,6 @@ import (
 )
 
 var ServeFlags = []cli.Flag{
-	&cli.BoolFlag{
-		Name: "verbose",
-		// TODO: Usage
-		EnvVars: []string{"DEBUG"},
-	},
-	
 	// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
 	// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
 	// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
@@ -69,6 +63,25 @@ var ServeFlags = []cli.Flag{
 		// TODO: desc
 		EnvVars: []string{"ENABLE_HTTP_SERVER"},
 	},
+	// Server Options
+	&cli.BoolFlag{
+		Name:    "enable-lfs-support",
+		Usage:   "enable lfs support, require gitea v1.17.0 as backend",
+		EnvVars: []string{"ENABLE_LFS_SUPPORT"},
+		Value:   true,
+	},
+	&cli.BoolFlag{
+		Name:    "enable-symlink-support",
+		Usage:   "follow symlinks if enabled, require gitea v1.18.0 as backend",
+		EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
+		Value:   true,
+	},
+	&cli.StringFlag{
+		Name:    "log-level",
+		Value:   "warn",
+		Usage:   "specify at which log level should be logged. Possible options: info, warn, error, fatal",
+		EnvVars: []string{"LOG_LEVEL"},
+	},
 
 	// ACME
 	&cli.StringFlag{

cmd/main.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"os"
 	"strings"
 	"time"
 
@@ -18,6 +19,7 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -35,10 +37,12 @@ var BlacklistedPaths = [][]byte{
 
 // Serve sets up and starts the web server.
 func Serve(ctx *cli.Context) error {
-	verbose := ctx.Bool("verbose")
-	if !verbose {
-		zerolog.SetGlobalLevel(zerolog.InfoLevel)
+	// Initalize the logger.
+	logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
+	if err != nil {
+		return err
 	}
+	log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
 
 	giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
 	giteaAPIToken := ctx.String("gitea-api-token")
@@ -72,18 +76,24 @@ func Serve(ctx *cli.Context) error {
 	keyCache := cache.NewKeyValueCache()
 	challengeCache := cache.NewKeyValueCache()
 	// canonicalDomainCache stores canonical domains
-	var canonicalDomainCache = cache.NewKeyValueCache()
+	canonicalDomainCache := cache.NewKeyValueCache()
 	// dnsLookupCache stores DNS lookups for custom domains
-	var dnsLookupCache = cache.NewKeyValueCache()
+	dnsLookupCache := cache.NewKeyValueCache()
 	// branchTimestampCache stores branch timestamps for faster cache checking
-	var branchTimestampCache = cache.NewKeyValueCache()
+	branchTimestampCache := cache.NewKeyValueCache()
 	// fileResponseCache stores responses from the Gitea server
 	// TODO: make this an MRU cache with a size limit
-	var fileResponseCache = cache.NewKeyValueCache()
+	fileResponseCache := cache.NewKeyValueCache()
+
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
+	if err != nil {
+		return fmt.Errorf("could not create new gitea client: %v", err)
+	}
 
 	// Create handler based on settings
 	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
-		giteaRoot, rawInfoPage, giteaAPIToken,
+		giteaClient,
+		giteaRoot, rawInfoPage,
 		BlacklistedPaths, allowedCorsDomains,
 		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
 
@@ -94,7 +104,7 @@ func Serve(ctx *cli.Context) error {
 	log.Info().Msgf("Listening on https://%s", listeningAddress)
 	listener, err := net.Listen("tcp", listeningAddress)
 	if err != nil {
-		return fmt.Errorf("couldn't create listener: %s", err)
+		return fmt.Errorf("couldn't create listener: %v", err)
 	}
 
 	// TODO: make "key-database.pogreb" set via flag
@@ -105,7 +115,8 @@ func Serve(ctx *cli.Context) error {
 	defer certDB.Close() //nolint:errcheck    // database has no close ... sync behave like it
 
 	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
-		giteaRoot, giteaAPIToken, dnsProvider,
+		giteaClient,
+		dnsProvider,
 		acmeUseRateLimits,
 		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
 		certDB))
@@ -126,6 +137,7 @@ func Serve(ctx *cli.Context) error {
 
 	if enableHTTPServer {
 		go func() {
+			log.Info().Msg("Start HTTP server listening on :80")
 			err := httpServer.ListenAndServe("[::]:80")
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
@@ -134,6 +146,7 @@ func Serve(ctx *cli.Context) error {
 	}
 
 	// Start the web fastServer
+	log.Info().Msgf("Start listening on %s", listener.Addr())
 	err = fastServer.Serve(listener)
 	if err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")

go.mod

@@ -1,15 +1,128 @@
 module codeberg.org/codeberg/pages
 
-go 1.16
+go 1.18
 
 require (
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
 	github.com/akrylysov/pogreb v0.10.1
 	github.com/go-acme/lego/v4 v4.5.3
+	github.com/joho/godotenv v1.4.0
 	github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
-	github.com/rs/zerolog v1.26.0
+	github.com/rs/zerolog v1.27.0
 	github.com/stretchr/testify v1.7.0
 	github.com/urfave/cli/v2 v2.3.0
 	github.com/valyala/fasthttp v1.31.0
 	github.com/valyala/fastjson v1.6.3
 )
+
+require (
+	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/azure-sdk-for-go v32.4.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.19 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
+	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
+	github.com/andybalholm/brotli v1.0.2 // indirect
+	github.com/aws/aws-sdk-go v1.39.0 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/cloudflare/cloudflare-go v0.20.0 // indirect
+	github.com/cpu/goacmedns v0.1.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deepmap/oapi-codegen v1.6.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dnsimple/dnsimple-go v0.70.1 // indirect
+	github.com/exoscale/egoscale v0.67.0 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.1.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gophercloud/gophercloud v0.16.0 // indirect
+	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
+	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
+	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
+	github.com/jarcoal/httpmock v1.0.6 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/json-iterator/go v1.1.7 // indirect
+	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
+	github.com/klauspost/compress v1.13.4 // indirect
+	github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
+	github.com/labbsr0x/goh v1.0.1 // indirect
+	github.com/linode/linodego v0.31.1 // indirect
+	github.com/liquidweb/go-lwApi v0.0.5 // indirect
+	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
+	github.com/liquidweb/liquidweb-go v1.6.3 // indirect
+	github.com/mattn/go-colorable v0.1.12 // indirect
+	github.com/mattn/go-isatty v0.0.14 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
+	github.com/nrdcg/auroradns v1.0.1 // indirect
+	github.com/nrdcg/desec v0.6.0 // indirect
+	github.com/nrdcg/dnspod-go v0.4.0 // indirect
+	github.com/nrdcg/freemyip v0.2.0 // indirect
+	github.com/nrdcg/goinwx v0.8.1 // indirect
+	github.com/nrdcg/namesilo v0.2.1 // indirect
+	github.com/nrdcg/porkbun v0.1.1 // indirect
+	github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
+	github.com/ovh/go-ovh v1.1.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pquerna/otp v1.3.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/sacloud/libsacloud v1.36.2 // indirect
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
+	github.com/softlayer/softlayer-go v1.0.3 // indirect
+	github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/transip/gotransip/v6 v6.6.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
+	github.com/vultr/govultr/v2 v2.7.1 // indirect
+	go.opencensus.io v0.22.3 // indirect
+	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
+	google.golang.org/api v0.20.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect
+	google.golang.org/grpc v1.27.1 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

go.sum

@@ -95,7 +95,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
 github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
@@ -266,6 +266,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -319,12 +321,15 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
 github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -426,8 +431,8 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
-github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
-github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
+github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
+github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
 github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -486,7 +491,6 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
 github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
 github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
@@ -509,7 +513,6 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -568,7 +571,6 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
 golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -667,8 +669,9 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
-golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -722,7 +725,6 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
 golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
 golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

html/404.html

@@ -21,12 +21,13 @@
     </style>
 </head>
 <body>
-    <i class="fa fa-bug text-primary" style="font-size: 96px;"></i>
+    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
     <h1 class="mb-0 text-primary">
-    	You found a bug!
+		Page not found!
     </h1>
     <h5 class="text-center" style="max-width: 25em;">
-    	Sorry, this page doesn't exist or is inaccessible for other reasons (%status)
+    	Sorry, but this page couldn't be found or is inaccessible (%status).<br/>
+		We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
     </h5>
     <small class="text-muted">
         <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">

integration/get_test.go

@@ -0,0 +1,143 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"log"
+	"net/http"
+	"net/http/cookiejar"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetRedirect(t *testing.T) {
+	log.Println("=== TestGetRedirect ===")
+	// test custom domain redirect
+	resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
+	assert.EqualValues(t, 0, getSize(resp.Body))
+}
+
+func TestGetContent(t *testing.T) {
+	log.Println("=== TestGetContent ===")
+	// test get image
+	resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "image/jpeg", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "124635", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 124635, getSize(resp.Body))
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// specify branch
+	resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 1000)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// access branch name contains '/'
+	resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 100)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
+}
+
+func TestCustomDomain(t *testing.T) {
+	log.Println("=== TestCustomDomain ===")
+	resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 106, getSize(resp.Body))
+}
+
+func TestGetNotFound(t *testing.T) {
+	log.Println("=== TestGetNotFound ===")
+	// test custom not found pages
+	resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 37, getSize(resp.Body))
+}
+
+func TestFollowSymlink(t *testing.T) {
+	log.Printf("=== TestFollowSymlink ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
+	body := getBytes(resp.Body)
+	assert.EqualValues(t, 4, len(body))
+	assert.EqualValues(t, "abc\n", string(body))
+}
+
+func TestLFSSupport(t *testing.T) {
+	log.Printf("=== TestLFSSupport ===\n")
+
+	resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	body := strings.TrimSpace(string(getBytes(resp.Body)))
+	assert.EqualValues(t, 12, len(body))
+	assert.EqualValues(t, "actual value", body)
+}
+
+func getTestHTTPSClient() *http.Client {
+	cookieJar, _ := cookiejar.New(nil)
+	return &http.Client{
+		Jar: cookieJar,
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+}
+
+func getBytes(stream io.Reader) []byte {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Bytes()
+}
+
+func getSize(stream io.Reader) int {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Len()
+}

integration/main_test.go

@@ -0,0 +1,62 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"context"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"codeberg.org/codeberg/pages/cmd"
+
+	"github.com/urfave/cli/v2"
+)
+
+func TestMain(m *testing.M) {
+	log.Println("=== TestMain: START Server ===")
+	serverCtx, serverCancel := context.WithCancel(context.Background())
+	if err := startServer(serverCtx); err != nil {
+		log.Fatalf("could not start server: %v", err)
+	}
+	defer func() {
+		serverCancel()
+		log.Println("=== TestMain: Server STOPED ===")
+	}()
+
+	time.Sleep(10 * time.Second)
+
+	os.Exit(m.Run())
+}
+
+func startServer(ctx context.Context) error {
+	args := []string{
+		"--verbose",
+		"--acme-accept-terms", "true",
+	}
+	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
+	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
+	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PORT", "4430")
+
+	app := cli.NewApp()
+	app.Name = "pages-server"
+	app.Action = cmd.Serve
+	app.Flags = cmd.ServeFlags
+
+	go func() {
+		if err := app.RunContext(ctx, args); err != nil {
+			log.Fatalf("run server error: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+func setEnvIfNotSet(key, value string) {
+	if _, set := os.LookupEnv(key); !set {
+		os.Setenv(key, value)
+	}
+}

main.go

@@ -4,15 +4,14 @@ import (
 	"fmt"
 	"os"
 
+	_ "github.com/joho/godotenv/autoload"
 	"github.com/urfave/cli/v2"
 
 	"codeberg.org/codeberg/pages/cmd"
 )
 
-var (
-	// can be changed with -X on compile
-	version = "dev"
-)
+// can be changed with -X on compile
+var version = "dev"
 
 func main() {
 	app := cli.NewApp()

server/certificates/acme_account.go

@@ -19,9 +19,11 @@ var _ registration.User = &AcmeAccount{}
 func (u *AcmeAccount) GetEmail() string {
 	return u.Email
 }
+
 func (u AcmeAccount) GetRegistration() *registration.Resource {
 	return u.Registration
 }
+
 func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
 	return u.Key
 }

server/certificates/certificates.go

@@ -12,7 +12,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"strconv"
 	"strings"
@@ -32,15 +31,18 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/database"
 	dnsutils "codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
 func TLSConfig(mainDomainSuffix []byte,
-	giteaRoot, giteaAPIToken, dnsProvider string,
+	giteaClient *gitea.Client,
+	dnsProvider string,
 	acmeUseRateLimits bool,
 	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
-	certDB database.CertDB) *tls.Config {
+	certDB database.CertDB,
+) *tls.Config {
 	return &tls.Config{
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@@ -80,7 +82,7 @@ func TLSConfig(mainDomainSuffix []byte,
 					sni = string(sniBytes)
 				} else {
 					_, _ = targetRepo, targetBranch
-					_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+					_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
 					if !valid {
 						sniBytes = mainDomainSuffix
 						sni = string(sniBytes)
@@ -146,8 +148,10 @@ func checkUserLimit(user string) error {
 	return nil
 }
 
-var acmeClient, mainDomainAcmeClient *lego.Client
-var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+var (
+	acmeClient, mainDomainAcmeClient  *lego.Client
+	acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+)
 
 // rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
 // TODO: when this is used a lot, we probably have to think of a somewhat better solution?
@@ -166,6 +170,7 @@ var _ challenge.Provider = AcmeTLSChallengeProvider{}
 func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
 	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
 }
+
 func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
 	a.challengeCache.Remove(domain)
 	return nil
@@ -181,6 +186,7 @@ var _ challenge.Provider = AcmeHTTPChallengeProvider{}
 func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
 	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
 }
+
 func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 	a.challengeCache.Remove(domain + "/" + token)
 	return nil
@@ -188,7 +194,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 
 func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
 	// parse certificate from database
-	res, err := certDB.Get(sni)
+	res, err := certDB.Get(string(sni))
 	if err != nil {
 		panic(err) // TODO: no panic
 	}
@@ -209,7 +215,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
 		}
 
 		// renew certificates 7 days before they expire
-		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
+		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(7 * 24 * time.Hour)) {
 			// TODO: add ValidUntil to custom res struct
 			if res.CSR != nil && len(res.CSR) > 0 {
 				// CSR stores the time when the renewal shall be tried again
@@ -222,7 +228,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
 				res.CSR = nil // acme client doesn't like CSR to be set
 				tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 				if err != nil {
-					log.Printf("Couldn't renew certificate for %s: %s", sni, err)
+					log.Error().Msgf("Couldn't renew certificate for %s: %v", string(sni), err)
 				}
 			})()
 		}
@@ -265,10 +271,10 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 		if acmeUseRateLimits {
 			acmeClientRequestLimit.Take()
 		}
-		log.Printf("Renewing certificate for %v", domains)
+		log.Debug().Msgf("Renewing certificate for: %v", domains)
 		res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
 		if err != nil {
-			log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
+			log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
 			res = nil
 		}
 	}
@@ -283,7 +289,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 			acmeClientOrderLimit.Take()
 			acmeClientRequestLimit.Take()
 		}
-		log.Printf("Requesting new certificate for %v", domains)
+		log.Debug().Msgf("Re-requesting new certificate for %v", domains)
 		res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
 			Domains:    domains,
 			Bundle:     true,
@@ -291,7 +297,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 		})
 	}
 	if err != nil {
-		log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
+		log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
 		if renew != nil && renew.CertURL != "" {
 			tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
 			if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
@@ -305,7 +311,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
 		}
 		return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
 	}
-	log.Printf("Obtained certificate for %v", domains)
+	log.Debug().Msgf("Obtained certificate for %v", domains)
 
 	if err := keyDatabase.Put(name, res); err != nil {
 		return tls.Certificate{}, err
@@ -322,7 +328,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 	var myAcmeAccount AcmeAccount
 	var myAcmeConfig *lego.Config
 
-	if account, err := ioutil.ReadFile(configFile); err == nil {
+	if account, err := os.ReadFile(configFile); err == nil {
 		if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
 			return nil, err
 		}
@@ -338,7 +344,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 		_, err := lego.NewClient(myAcmeConfig)
 		if err != nil {
 			// TODO: should we fail hard instead?
-			log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+			log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 		}
 		return myAcmeConfig, nil
 	} else if !os.IsNotExist(err) {
@@ -359,13 +365,13 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 	myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
 	tempClient, err := lego.NewClient(myAcmeConfig)
 	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 	} else {
 		// accept terms & log in to EAB
 		if acmeEabKID == "" || acmeEabHmac == "" {
 			reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
 			if err != nil {
-				log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
 			} else {
 				myAcmeAccount.Registration = reg
 			}
@@ -376,7 +382,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 				HmacEncoded:          acmeEabHmac,
 			})
 			if err != nil {
-				log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
+				log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
 			} else {
 				myAcmeAccount.Registration = reg
 			}
@@ -385,12 +391,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 		if myAcmeAccount.Registration != nil {
 			acmeAccountJSON, err := json.Marshal(myAcmeAccount)
 			if err != nil {
-				log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
+				log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
 				select {}
 			}
-			err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
+			err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
 			if err != nil {
-				log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
+				log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
 				select {}
 			}
 		}
@@ -401,45 +407,45 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 
 func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
-	mainCertBytes, err := certDB.Get(mainDomainSuffix)
+	mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
 	if err != nil {
 		return fmt.Errorf("cert database is not working")
 	}
 
 	acmeClient, err = lego.NewClient(acmeConfig)
 	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 	} else {
 		err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
 		if err != nil {
-			log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
+			log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
 		}
 		if enableHTTPServer {
 			err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
 			if err != nil {
-				log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
+				log.Error().Err(err).Msg("Can't create HTTP-01 provider")
 			}
 		}
 	}
 
 	mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
 	if err != nil {
-		log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
+		log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
 	} else {
 		if dnsProvider == "" {
 			// using mock server, don't use wildcard certs
 			err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
 			if err != nil {
-				log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
+				log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
 			}
 		} else {
 			provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
 			if err != nil {
-				log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
+				log.Error().Err(err).Msg("Can't create DNS Challenge provider")
 			}
 			err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
 			if err != nil {
-				log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
+				log.Error().Err(err).Msg("Can't create DNS-01 provider")
 			}
 		}
 	}
@@ -447,7 +453,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
 	if mainCertBytes == nil {
 		_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 		if err != nil {
-			log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
+			log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
 		}
 	}
 
@@ -473,9 +479,9 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 
 				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 				if err != nil || !tlsCertificates[0].NotAfter.After(now) {
-					err := certDB.Delete(key)
+					err := certDB.Delete(string(key))
 					if err != nil {
-						log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
+						log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
 					} else {
 						expiredCertCount++
 					}
@@ -483,31 +489,31 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 			}
 			key, resBytes, err = keyDatabaseIterator.Next()
 		}
-		log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
+		log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
 
 		// compact the database
-		result, err := certDB.Compact()
+		msg, err := certDB.Compact()
 		if err != nil {
-			log.Printf("[ERROR] Compacting key database failed: %s", err)
+			log.Error().Err(err).Msg("Compacting key database failed")
 		} else {
-			log.Printf("[INFO] Compacted key database (%+v)", result)
+			log.Debug().Msgf("Compacted key database: %s", msg)
 		}
 
 		// update main cert
-		res, err := certDB.Get(mainDomainSuffix)
+		res, err := certDB.Get(string(mainDomainSuffix))
 		if err != nil {
-			log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
+			log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
 		} else if res == nil {
-			log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
+			log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", string(mainDomainSuffix))
 		} else {
 			tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 
 			// renew main certificate 30 days before it expires
-			if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
+			if !tlsCertificates[0].NotAfter.After(time.Now().Add(30 * 24 * time.Hour)) {
 				go (func() {
 					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 					if err != nil {
-						log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
+						log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
 					}
 				})()
 			}

server/certificates/mock_test.go

@@ -0,0 +1,17 @@
+package certificates
+
+import (
+	"testing"
+
+	"codeberg.org/codeberg/pages/server/database"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMockCert(t *testing.T) {
+	db, err := database.NewTmpDB()
+	assert.NoError(t, err)
+	cert := mockCert("example.com", "some error msg", "codeberg.page", db)
+	if assert.NotEmpty(t, cert) {
+		assert.NotEmpty(t, cert.Certificate)
+	}
+}

server/database/interface.go

@@ -8,8 +8,8 @@ import (
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error
-	Get(name []byte) (*certificate.Resource, error)
-	Delete(key []byte) error
-	Compact() (pogreb.CompactionResult, error)
+	Get(name string) (*certificate.Resource, error)
+	Delete(key string) error
+	Compact() (string, error)
 	Items() *pogreb.ItemIterator
 }

server/database/mock.go

@@ -0,0 +1,55 @@
+package database
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/OrlovEvgeny/go-mcache"
+	"github.com/akrylysov/pogreb"
+	"github.com/go-acme/lego/v4/certificate"
+)
+
+var _ CertDB = tmpDB{}
+
+type tmpDB struct {
+	intern *mcache.CacheDriver
+	ttl    time.Duration
+}
+
+func (p tmpDB) Close() error {
+	_ = p.intern.Close()
+	return nil
+}
+
+func (p tmpDB) Put(name string, cert *certificate.Resource) error {
+	return p.intern.Set(name, cert, p.ttl)
+}
+
+func (p tmpDB) Get(name string) (*certificate.Resource, error) {
+	cert, has := p.intern.Get(name)
+	if !has {
+		return nil, fmt.Errorf("cert for '%s' not found", name)
+	}
+	return cert.(*certificate.Resource), nil
+}
+
+func (p tmpDB) Delete(key string) error {
+	p.intern.Remove(key)
+	return nil
+}
+
+func (p tmpDB) Compact() (string, error) {
+	p.intern.Truncate()
+	return "Truncate done", nil
+}
+
+func (p tmpDB) Items() *pogreb.ItemIterator {
+	panic("ItemIterator not implemented for tmpDB")
+}
+
+func NewTmpDB() (CertDB, error) {
+	return &tmpDB{
+		intern: mcache.New(),
+		ttl:    time.Minute,
+	}, nil
+}

server/database/setup.go

@@ -13,6 +13,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+var _ CertDB = aDB{}
+
 type aDB struct {
 	ctx          context.Context
 	cancel       context.CancelFunc
@@ -33,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error {
 	return p.intern.Put([]byte(name), resGob.Bytes())
 }
 
-func (p aDB) Get(name []byte) (*certificate.Resource, error) {
+func (p aDB) Get(name string) (*certificate.Resource, error) {
 	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get(name)
+	resBytes, err := p.intern.Get([]byte(name))
 	if err != nil {
 		return nil, err
 	}
@@ -48,12 +50,16 @@ func (p aDB) Get(name []byte) (*certificate.Resource, error) {
 	return cert, nil
 }
 
-func (p aDB) Delete(key []byte) error {
-	return p.intern.Delete(key)
+func (p aDB) Delete(key string) error {
+	return p.intern.Delete([]byte(key))
 }
 
-func (p aDB) Compact() (pogreb.CompactionResult, error) {
-	return p.intern.Compact()
+func (p aDB) Compact() (string, error) {
+	result, err := p.intern.Compact()
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%+v", result), nil
 }
 
 func (p aDB) Items() *pogreb.ItemIterator {
@@ -66,21 +72,7 @@ func (p aDB) sync() {
 	for {
 		err := p.intern.Sync()
 		if err != nil {
-			log.Err(err).Msg("Syncing cert database failed")
-		}
-		select {
-		case <-p.ctx.Done():
-			return
-		case <-time.After(p.syncInterval):
-		}
-	}
-}
-
-func (p aDB) compact() {
-	for {
-		err := p.intern.Sync()
-		if err != nil {
-			log.Err(err).Msg("Syncing cert database failed")
+			log.Error().Err(err).Msg("Syncing cert database failed")
 		}
 		select {
 		case <-p.ctx.Done():

server/gitea/cache.go

@@ -0,0 +1,12 @@
+package gitea
+
+type FileResponse struct {
+	Exists   bool
+	ETag     []byte
+	MimeType string
+	Body     []byte
+}
+
+func (f FileResponse) IsEmpty() bool {
+	return len(f.Body) != 0
+}

server/gitea/client.go

@@ -0,0 +1,142 @@
+package gitea
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/rs/zerolog/log"
+	"github.com/valyala/fasthttp"
+	"github.com/valyala/fastjson"
+)
+
+const (
+	giteaAPIRepos         = "/api/v1/repos/"
+	giteaObjectTypeHeader = "X-Gitea-Object-Type"
+)
+
+var ErrorNotFound = errors.New("not found")
+
+type Client struct {
+	giteaRoot      string
+	giteaAPIToken  string
+	fastClient     *fasthttp.Client
+	infoTimeout    time.Duration
+	contentTimeout time.Duration
+
+	followSymlinks bool
+	supportLFS     bool
+}
+
+// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
+func joinURL(baseURL string, paths ...string) string {
+	p := make([]string, 0, len(paths))
+	for i := range paths {
+		path := strings.TrimSpace(paths[i])
+		path = strings.Trim(path, "/")
+		if len(path) != 0 {
+			p = append(p, path)
+		}
+	}
+
+	return baseURL + "/" + strings.Join(p, "/")
+}
+
+func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) {
+	rootURL, err := url.Parse(giteaRoot)
+	giteaRoot = strings.Trim(rootURL.String(), "/")
+
+	return &Client{
+		giteaRoot:      giteaRoot,
+		giteaAPIToken:  giteaAPIToken,
+		infoTimeout:    5 * time.Second,
+		contentTimeout: 10 * time.Second,
+		fastClient:     getFastHTTPClient(),
+
+		followSymlinks: followSymlinks,
+		supportLFS:     supportLFS,
+	}, err
+}
+
+func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
+	if err != nil {
+		return nil, err
+	}
+	return resp.Body(), nil
+}
+
+func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
+	var apiURL string
+	if client.supportLFS {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
+	} else {
+		apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	}
+	resp, err := client.do(client.contentTimeout, apiURL)
+	if err != nil {
+		return nil, err
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	switch resp.StatusCode() {
+	case fasthttp.StatusOK:
+		objType := string(resp.Header.Peek(giteaObjectTypeHeader))
+		log.Trace().Msgf("server raw content object: %s", objType)
+		if client.followSymlinks && objType == "symlink" {
+			// TODO: limit to 1000 chars if we switched to std
+			linkDest := strings.TrimSpace(string(resp.Body()))
+			log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
+			return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+		}
+
+		return resp, nil
+
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
+	}
+}
+
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return time.Time{}, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+}
+
+func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return fastjson.GetString(res.Body(), "default_branch"), nil
+}
+
+func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
+	req := fasthttp.AcquireRequest()
+
+	req.SetRequestURI(url)
+	req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	err := client.fastClient.DoTimeout(req, res, timeout)
+
+	return res, err
+}

server/gitea/client_test.go

@@ -0,0 +1,23 @@
+package gitea
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJoinURL(t *testing.T) {
+	baseURL := ""
+	assert.EqualValues(t, "/", joinURL(baseURL))
+	assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
+
+	baseURL = "http://wwow.url.com"
+	assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
+
+	baseURL = "http://wow.url.com/subpath/2"
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
+}

server/gitea/fasthttp.go

@@ -0,0 +1,15 @@
+package gitea
+
+import (
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+func getFastHTTPClient() *fasthttp.Client {
+	return &fasthttp.Client{
+		MaxConnDuration:    60 * time.Second,
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
+}

server/handler.go

@@ -4,25 +4,30 @@ import (
 	"bytes"
 	"strings"
 
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 	"codeberg.org/codeberg/pages/server/utils"
+	"codeberg.org/codeberg/pages/server/version"
 )
 
 // Handler handles a single HTTP request to the web server.
 func Handler(mainDomainSuffix, rawDomain []byte,
-	giteaRoot, rawInfoPage, giteaAPIToken string,
+	giteaClient *gitea.Client,
+	giteaRoot, rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains [][]byte,
-	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
+	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) func(ctx *fasthttp.RequestCtx) {
 	return func(ctx *fasthttp.RequestCtx) {
-		log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
+		log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger()
 
-		ctx.Response.Header.Set("Server", "Codeberg Pages")
+		ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
 
 		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
 		ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -53,41 +58,45 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 		}
 
 		// Allow CORS for specified domains
+		allowCors := false
+		for _, allowedCorsDomain := range allowedCorsDomains {
+			if bytes.Equal(trimmedHost, allowedCorsDomain) {
+				allowCors = true
+				break
+			}
+		}
+		if allowCors {
+			ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
+			ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
+		}
+		ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
 		if ctx.IsOptions() {
-			allowCors := false
-			for _, allowedCorsDomain := range allowedCorsDomains {
-				if bytes.Equal(trimmedHost, allowedCorsDomain) {
-					allowCors = true
-					break
-				}
-			}
-			if allowCors {
-				ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
-				ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
-			}
-			ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
 			ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
 			return
 		}
 
 		// Prepare request information to Gitea
 		var targetOwner, targetRepo, targetBranch, targetPath string
-		var targetOptions = &upstream.Options{
-			ForbiddenMimeTypes: map[string]struct{}{},
-			TryIndexPages:      true,
+		targetOptions := &upstream.Options{
+			TryIndexPages: true,
 		}
 
 		// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
 		// also disallow search indexing and add a Link header to the canonical URL.
-		var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
+		tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
 			if repo == "" {
+				log.Warn().Msg("tryBranch: repo is empty")
 				return false
 			}
 
+			// Replace "~" to "/" so we can access branch that contains slash character
+			// Branch name cannot contain "~" so doing this is okay
+			branch = strings.ReplaceAll(branch, "~", "/")
+
 			// Check if the branch exists, otherwise treat it as a file path
-			branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
+			branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
 			if branchTimestampResult == nil {
-				// branch doesn't exist
+				log.Warn().Msg("tryBranch: branch doesn't exist")
 				return false
 			}
 
@@ -107,16 +116,20 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				)
 			}
 
+			log.Debug().Msg("tryBranch: true")
 			return true
 		}
 
-		log.Debug().Msg("preparations")
+		log.Debug().Msg("Preparing")
 		if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
 			// Serve raw content from RawDomain
-			log.Debug().Msg("raw domain")
+			log.Debug().Msg("Serving raw domain")
 
 			targetOptions.TryIndexPages = false
-			targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
+			if targetOptions.ForbiddenMimeTypes == nil {
+				targetOptions.ForbiddenMimeTypes = make(map[string]bool)
+			}
+			targetOptions.ForbiddenMimeTypes["text/html"] = true
 			targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
 
 			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
@@ -130,36 +143,36 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 
 			// raw.codeberg.org/example/myrepo/@main/index.html
 			if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
-				log.Debug().Msg("raw domain preparations, now trying with specified branch")
-				if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
+				log.Debug().Msg("Preparing raw domain, now trying with specified branch")
+				if tryBranch(log,
+					targetRepo, pathElements[2][1:], pathElements[3:],
 					giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 				) {
-					log.Debug().Msg("tryBranch, now trying upstream")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					log.Info().Msg("tryBranch, now trying upstream 1")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 					return
 				}
-				log.Debug().Msg("missing branch")
+				log.Warn().Msg("Path missed a branch")
 				html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
 				return
 			}
 
-			log.Debug().Msg("raw domain preparations, now trying with default branch")
-			tryBranch(targetRepo, "", pathElements[2:],
+			log.Debug().Msg("Preparing raw domain, now trying with default branch")
+			tryBranch(log,
+				targetRepo, "", pathElements[2:],
 				giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 			)
-			log.Debug().Msg("tryBranch, now trying upstream")
-			tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			log.Info().Msg("tryBranch, now trying upstream 2")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 				targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-				giteaRoot, giteaAPIToken,
 				canonicalDomainCache, branchTimestampCache, fileResponseCache)
 			return
 
 		} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
 			// Serve pages from subdomains of MainDomainSuffix
-			log.Debug().Msg("main domain suffix")
+			log.Info().Msg("Serve pages from main domain suffix")
 
 			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
 			targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
@@ -181,16 +194,17 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 					return
 				}
 
-				log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
-				if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
+				log.Debug().Msg("Preparing main domain, now trying with specified repo & branch")
+				if tryBranch(log,
+					pathElements[0], pathElements[1][1:], pathElements[2:],
 					"/"+pathElements[0]+"/%p",
 				) {
-					log.Debug().Msg("tryBranch, now trying upstream")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					log.Info().Msg("tryBranch, now trying upstream 3")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
+					log.Warn().Msg("tryBranch: upstream 3 failed")
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
 				}
 				return
@@ -199,14 +213,15 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// Check if the first directory is a branch for the "pages" repo
 			// example.codeberg.page/@main/index.html
 			if strings.HasPrefix(pathElements[0], "@") {
-				log.Debug().Msg("main domain preparations, now trying with specified branch")
-				if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
-					log.Debug().Msg("tryBranch, now trying upstream")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				log.Debug().Msg("Preparing main domain, now trying with specified branch")
+				if tryBranch(log,
+					"pages", pathElements[0][1:], pathElements[1:], "/%p") {
+					log.Info().Msg("tryBranch, now trying upstream 4")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
+					log.Warn().Msg("tryBranch: upstream 4 failed")
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
 				}
 				return
@@ -216,11 +231,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/myrepo/index.html
 			// example.codeberg.page/pages/... is not allowed here.
 			log.Debug().Msg("main domain preparations, now trying with specified repo")
-			if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
-				log.Debug().Msg("tryBranch, now trying upstream")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			if pathElements[0] != "pages" && tryBranch(log,
+				pathElements[0], "pages", pathElements[1:], "") {
+				log.Info().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -228,16 +243,17 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// Try to use the "pages" repo on its default branch
 			// example.codeberg.page/index.html
 			log.Debug().Msg("main domain preparations, now trying with default repo/branch")
-			if tryBranch("pages", "", pathElements, "") {
-				log.Debug().Msg("tryBranch, now trying upstream")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			if tryBranch(log,
+				"pages", "", pathElements, "") {
+				log.Info().Msg("tryBranch, now trying upstream 6")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
 
 			// Couldn't find a valid repo/branch
+
 			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
 			return
 		} else {
@@ -259,10 +275,12 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			}
 
 			// Try to use the given repo on the given branch or the default branch
-			log.Debug().Msg("custom domain preparations, now trying with details from DNS")
-			if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
-				canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+			log.Debug().Msg("Preparing custom domain, now trying with details from DNS")
+			if tryBranch(log,
+				targetRepo, targetBranch, pathElements, canonicalLink) {
+				canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
 				if !valid {
+					log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical")
 					html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
 					return
 				} else if canonicalDomain != trimmedHostStr {
@@ -273,18 +291,19 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 						return
 					}
 
+					log.Warn().Msg("Custom domains, targetOwner from DNS is empty")
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
 					return
 				}
 
-				log.Debug().Msg("tryBranch, now trying upstream")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				log.Info().Msg("tryBranch, now trying upstream 7")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
 
+			log.Warn().Msg("Couldn't handle request, none of the options succeed")
 			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
 			return
 		}

server/handler_test.go

@@ -2,20 +2,22 @@ package server
 
 import (
 	"fmt"
-	"github.com/valyala/fasthttp"
 	"testing"
 	"time"
 
+	"github.com/valyala/fasthttp"
+
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 func TestHandlerPerformance(t *testing.T) {
+	giteaRoot := "https://codeberg.org"
+	giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
 	testHandler := Handler(
-		[]byte("codeberg.page"),
-		[]byte("raw.codeberg.org"),
-		"https://codeberg.org",
-		"https://docs.codeberg.org/pages/raw-content/",
-		"",
+		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
+		giteaClient,
+		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
 		[][]byte{[]byte("/.well-known/acme-challenge/")},
 		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
 		cache.NewKeyValueCache(),
@@ -24,46 +26,26 @@ func TestHandlerPerformance(t *testing.T) {
 		cache.NewKeyValueCache(),
 	)
 
-	ctx := &fasthttp.RequestCtx{
-		Request:  *fasthttp.AcquireRequest(),
-		Response: *fasthttp.AcquireResponse(),
-	}
-	ctx.Request.SetRequestURI("http://mondstern.codeberg.page/")
-	fmt.Printf("Start: %v\n", time.Now())
-	start := time.Now()
-	testHandler(ctx)
-	end := time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
+	testCase := func(uri string, status int) {
+		ctx := &fasthttp.RequestCtx{
+			Request:  *fasthttp.AcquireRequest(),
+			Response: *fasthttp.AcquireResponse(),
+		}
+		ctx.Request.SetRequestURI(uri)
+		fmt.Printf("Start: %v\n", time.Now())
+		start := time.Now()
+		testHandler(ctx)
+		end := time.Now()
+		fmt.Printf("Done: %v\n", time.Now())
+		if ctx.Response.StatusCode() != status {
+			t.Errorf("request failed with status code %d", ctx.Response.StatusCode())
+		} else {
+			t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
+		}
 	}
 
-	ctx.Response.Reset()
-	ctx.Response.ResetBody()
-	fmt.Printf("Start: %v\n", time.Now())
-	start = time.Now()
-	testHandler(ctx)
-	end = time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
-
-	ctx.Response.Reset()
-	ctx.Response.ResetBody()
-	ctx.Request.SetRequestURI("http://example.momar.xyz/")
-	fmt.Printf("Start: %v\n", time.Now())
-	start = time.Now()
-	testHandler(ctx)
-	end = time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
+	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
+	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
+	testCase("https://example.momar.xyz/", 424)       // TODO: expect 200
+	testCase("https://codeberg.page/", 424)           // TODO: expect 200
 }

server/setup.go

@@ -2,15 +2,23 @@ package server
 
 import (
 	"bytes"
+	"fmt"
 	"net/http"
 	"time"
 
+	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/utils"
 )
 
+type fasthttpLogger struct{}
+
+func (fasthttpLogger) Printf(format string, args ...interface{}) {
+	log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
+}
+
 func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
 	// Enable compression by wrapping the handler with the compression function provided by FastHTTP
 	compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
@@ -18,12 +26,10 @@ func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
 	return &fasthttp.Server{
 		Handler:                      compressedHandler,
 		DisablePreParseMultipartForm: true,
-		MaxRequestBodySize:           0,
 		NoDefaultServerHeader:        true,
 		NoDefaultDate:                true,
 		ReadTimeout:                  30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
-		Concurrency:                  1024 * 32,        // TODO: adjust bottlenecks for best performance with Gitea!
-		MaxConnsPerIP:                100,
+		Logger:                       fasthttpLogger{},
 	}
 }
 

server/try.go

@@ -8,22 +8,22 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx,
+func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	mainDomainSuffix, trimmedHost []byte,
 
 	targetOptions *upstream.Options,
-	targetOwner, targetRepo, targetBranch, targetPath,
-
-	giteaRoot, giteaAPIToken string,
-	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
+	targetOwner, targetRepo, targetBranch, targetPath string,
 
+	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) {
 	// check if a canonical domain exists on a request on MainDomain
 	if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
-		canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+		canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
 			canonicalPath := string(ctx.RequestURI())
 			if targetRepo != "pages" {
@@ -41,9 +41,10 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
 	targetOptions.TargetRepo = targetRepo
 	targetOptions.TargetBranch = targetBranch
 	targetOptions.TargetPath = targetPath
+	targetOptions.Host = string(trimmedHost)
 
 	// Try to request the file from the Gitea API
-	if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+	if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
 		html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
 	}
 }

server/upstream/const.go

@@ -12,6 +12,7 @@ var branchExistenceCacheTimeout = 5 * time.Minute
 
 // fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
 // on your available memory.
+// TODO: move as option into cache interface
 var fileCacheTimeout = 5 * time.Minute
 
 // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
@@ -19,3 +20,5 @@ var fileCacheSizeLimit = 1024 * 1024
 
 // canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
 var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"

server/upstream/domains.go

@@ -3,15 +3,16 @@ package upstream
 import (
 	"strings"
 
-	"github.com/valyala/fasthttp"
-
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
-	domains := []string{}
-	valid := false
+func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+	var (
+		domains []string
+		valid   bool
+	)
 	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
 		domains = cachedValue.([]string)
 		for _, domain := range domains {
@@ -21,13 +22,9 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
 			}
 		}
 	} else {
-		req := fasthttp.AcquireRequest()
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
-		res := fasthttp.AcquireResponse()
-
-		err := client.Do(req, res)
-		if err == nil && res.StatusCode() == fasthttp.StatusOK {
-			for _, domain := range strings.Split(string(res.Body()), "\n") {
+		body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
+		if err == nil {
+			for _, domain := range strings.Split(string(body), "\n") {
 				domain = strings.ToLower(domain)
 				domain = strings.TrimSpace(domain)
 				domain = strings.TrimPrefix(domain, "http://")

server/upstream/helper.go

@@ -1,12 +1,15 @@
 package upstream
 
 import (
+	"mime"
+	"path"
+	"strconv"
+	"strings"
 	"time"
 
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
+	"github.com/rs/zerolog/log"
 )
 
 type branchTimestamp struct {
@@ -16,40 +19,66 @@ type branchTimestamp struct {
 
 // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
 // (or nil if the branch doesn't exist)
-func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+	log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger()
 	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
 		if result == nil {
+			log.Debug().Msg("branchTimestampCache found item, but result is empty")
 			return nil
 		}
+		log.Debug().Msg("branchTimestampCache found item, returning result")
 		return result.(*branchTimestamp)
 	}
-	result := &branchTimestamp{}
-	result.Branch = branch
-	if branch == "" {
+	result := &branchTimestamp{
+		Branch: branch,
+	}
+	if len(branch) == 0 {
 		// Get default branch
-		var body = make([]byte, 0)
-		// TODO: use header for API key?
-		status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
-		if err != nil || status != 200 {
-			_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
+		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
+		if err != nil {
+			log.Err(err).Msg("Could't fetch default branch from repository")
+			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
 			return nil
 		}
-		result.Branch = fastjson.GetString(body, "default_branch")
+		log.Debug().Msg("Succesfully fetched default branch from Gitea")
+		result.Branch = defaultBranch
 	}
 
-	var body = make([]byte, 0)
-	status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
-	if err != nil || status != 200 {
+	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
+	if err != nil {
+		log.Err(err).Msg("Could not get latest commit's timestamp from branch")
 		return nil
 	}
-
-	result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
+	log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache")
+	result.Timestamp = timestamp
 	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
 	return result
 }
 
-type fileResponse struct {
-	exists   bool
-	mimeType string
-	body     []byte
+func (o *Options) getMimeTypeByExtension() string {
+	if o.ForbiddenMimeTypes == nil {
+		o.ForbiddenMimeTypes = make(map[string]bool)
+	}
+	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+	if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
+		if o.DefaultMimeType != "" {
+			mimeType = o.DefaultMimeType
+		} else {
+			mimeType = "application/octet-stream"
+		}
+	}
+	return mimeType
+}
+
+func (o *Options) generateUri() string {
+	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+}
+
+func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
+	return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
+}
+
+func (o *Options) timestamp() string {
+	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -2,11 +2,8 @@ package upstream
 
 import (
 	"bytes"
-	"fmt"
+	"errors"
 	"io"
-	"mime"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -15,6 +12,7 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // upstreamIndexPages lists pages that may be considered as index pages for directories.
@@ -22,6 +20,11 @@ var upstreamIndexPages = []string{
 	"index.html",
 }
 
+// upstreamNotFoundPages lists pages that may be considered as custom 404 Not Found pages.
+var upstreamNotFoundPages = []string{
+	"404.html",
+}
+
 // Options provides various options for the upstream request.
 type Options struct {
 	TargetOwner,
@@ -29,8 +32,11 @@ type Options struct {
 	TargetBranch,
 	TargetPath,
 
-	DefaultMimeType string
-	ForbiddenMimeTypes map[string]struct{}
+	// Used for debugging purposes.
+	Host string
+
+	DefaultMimeType    string
+	ForbiddenMimeTypes map[string]bool
 	TryIndexPages      bool
 	BranchTimestamp    time.Time
 	// internal
@@ -38,24 +44,13 @@ type Options struct {
 	redirectIfExists    string
 }
 
-var client = fasthttp.Client{
-	ReadTimeout:        10 * time.Second,
-	MaxConnDuration:    60 * time.Second,
-	MaxConnWaitTimeout: 1000 * time.Millisecond,
-	MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-}
-
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
-	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
-
-	if o.ForbiddenMimeTypes == nil {
-		o.ForbiddenMimeTypes = map[string]struct{}{}
-	}
+func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
 
 	// Check if the branch exists and when it was modified
 	if o.BranchTimestamp.IsZero() {
-		branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
+		branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
 
 		if branch == nil {
 			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -77,27 +72,23 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			return true
 		}
 	}
-	log.Debug().Msg("preparations")
+
+	log.Debug().Msg("Preparing")
 
 	// Make a GET request to the upstream URL
-	uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
-	var req *fasthttp.Request
+	uri := o.generateUri()
 	var res *fasthttp.Response
-	var cachedResponse fileResponse
+	var cachedResponse gitea.FileResponse
 	var err error
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
-		cachedResponse = cachedValue.(fileResponse)
+	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
+		cachedResponse = cachedValue.(gitea.FileResponse)
 	} else {
-		req = fasthttp.AcquireRequest()
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
-		res = fasthttp.AcquireResponse()
-		res.SetBodyStream(&strings.Reader{}, -1)
-		err = client.Do(req, res)
+		res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
 	}
-	log.Debug().Msg("acquisition")
+	log.Debug().Msg("Aquisting")
 
 	// Handle errors
-	if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
+	if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -105,9 +96,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
-						exists: false,
+				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
 					}, fileCacheTimeout)
 					return true
 				}
@@ -116,24 +107,39 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = false
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
-				_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
-					exists: false,
+			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+					Exists: false,
 				}, fileCacheTimeout)
 				return true
 			}
 		}
 		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+		if o.TryIndexPages {
+			// copy the o struct & try if a not found page exists
+			optionsForNotFoundPages := *o
+			optionsForNotFoundPages.TryIndexPages = false
+			optionsForNotFoundPages.appendTrailingSlash = false
+			for _, notFoundPage := range upstreamNotFoundPages {
+				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
+					}, fileCacheTimeout)
+					return true
+				}
+			}
+		}
 		if res != nil {
 			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
-				exists: false,
+			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+				Exists: false,
 			}, fileCacheTimeout)
 		}
 		return false
 	}
 	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
+		log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
@@ -152,50 +158,55 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 		ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
 		return true
 	}
-	log.Debug().Msg("error handling")
+	log.Debug().Msg("Handling error")
 
 	// Set the MIME type
-	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
-		if o.DefaultMimeType != "" {
-			mimeType = o.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
-		}
-	}
+	mimeType := o.getMimeTypeByExtension()
 	ctx.Response.Header.SetContentType(mimeType)
 
-	// Everything's okay so far
-	ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	// Set ETag
+	if cachedResponse.Exists {
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	} else if res != nil {
+		cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	}
+
+	if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
+		// Everything's okay so far
+		ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	}
 	ctx.Response.Header.SetLastModified(o.BranchTimestamp)
 
-	log.Debug().Msg("response preparations")
+	log.Debug().Msg("Prepare response")
 
 	// Write the response body to the original request
 	var cacheBodyWriter bytes.Buffer
 	if res != nil {
 		if res.Header.ContentLength() > fileCacheSizeLimit {
+			// fasthttp else will set "Content-Length: 0"
+			ctx.Response.SetBodyStream(&strings.Reader{}, -1)
+
 			err = res.BodyWriteTo(ctx.Response.BodyWriter())
 		} else {
 			// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
 			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
 		}
 	} else {
-		_, err = ctx.Write(cachedResponse.body)
+		_, err = ctx.Write(cachedResponse.Body)
 	}
 	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
+		log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
-	log.Debug().Msg("response")
+	log.Debug().Msg("Sending response")
 
-	if res != nil && ctx.Err() == nil {
-		cachedResponse.exists = true
-		cachedResponse.mimeType = mimeType
-		cachedResponse.body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
+	if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
+		cachedResponse.Exists = true
+		cachedResponse.MimeType = mimeType
+		cachedResponse.Body = cacheBodyWriter.Bytes()
+		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
 	}
 
 	return true

server/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string = "dev"