just fix bcaceda711

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/108
Reviewed-by: Otto <otto@codeberg.org>

.gitignore

@@ -4,3 +4,4 @@ key-database.pogreb/
 acme-account.json
 build/
 vendor/
+pages

.woodpecker.yml

@@ -0,0 +1,69 @@
+branches: main
+
+pipeline:
+  # use vendor to cache dependencies
+  vendor:
+    image: golang:1.18
+    commands:
+      - go mod vendor
+
+  lint:
+    image: golangci/golangci-lint:latest
+    group: compliant
+    pull: true
+    commands:
+      - go version
+      - go install mvdan.cc/gofumpt@latest
+      - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
+      - golangci-lint run --timeout 5m --build-tags integration
+
+  build:
+    group: compliant
+    image: a6543/golang_just
+    commands:
+      - go version
+      - just build
+    when:
+      event: [ "pull_request", "push" ]
+
+  build-tag:
+    group: compliant
+    image: a6543/golang_just
+    commands:
+      - go version
+      - just build-tag ${CI_COMMIT_TAG##v}
+    when:
+      event: [ "tag" ]
+
+  test:
+    image: a6543/golang_just
+    group: test
+    commands:
+      - just test
+
+  integration-tests:
+    image: a6543/golang_just
+    group: test
+    commands:
+      - just integration
+    environment:
+      - ACME_API=https://acme.mock.directory
+      - PAGES_DOMAIN=localhost.mock.directory
+      - RAW_DOMAIN=raw.localhost.mock.directory
+      - PORT=4430
+
+  release:
+    image: plugins/gitea-release
+    settings:
+      base_url: https://codeberg.org
+      file_exists: overwrite
+      files: build/codeberg-pages-server
+      api_key:
+        from_secret: bot_token
+    environment:
+      - DRONE_REPO_OWNER=${CI_REPO_OWNER}
+      - DRONE_REPO_NAME=${CI_REPO_NAME}
+      - DRONE_BUILD_EVENT=${CI_BUILD_EVENT}
+      - DRONE_COMMIT_REF=${CI_COMMIT_REF}
+    when:
+      event: [ "tag" ]

Justfile

@@ -10,3 +10,39 @@ dev:
 
 build:
     CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
+
+build-tag VERSION:
+    CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./
+
+lint: tool-golangci tool-gofumpt
+    [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
+    golangci-lint run --timeout 5m --build-tags integration
+
+fmt: tool-gofumpt
+    gofumpt -w --extra .
+
+clean:
+    go clean ./...
+    rm -rf build/
+
+tool-golangci:
+    @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \
+    fi
+
+tool-gofumpt:
+    @hash gofumpt> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install mvdan.cc/gofumpt@latest; \
+    fi
+
+test:
+    go test -race codeberg.org/codeberg/pages/server/...
+
+test-run TEST:
+    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/...
+
+integration:
+    go test -race -tags integration codeberg.org/codeberg/pages/integration/...
+
+integration-run TEST:
+    go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

README.md

@@ -1,4 +1,59 @@
-## Environment
+# Codeberg Pages
+
+Gitea lacks the ability to host static pages from Git.
+The Codeberg Pages Server addresses this lack by implementing a standalone service
+that connects to Gitea via API.
+It is suitable to be deployed by other Gitea instances, too, to offer static pages hosting to their users.
+
+**End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
+and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
+
+
+## Quickstart
+
+This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
+Mapping custom domains is not static anymore, but can be done with DNS:
+
+1) add a `.domains` text file to your repository, containing the allowed domains, separated by new lines. The
+first line will be the canonical domain/URL; all other occurrences will be redirected to it.
+
+2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
+"pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
+If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
+	`www.example.org. IN CNAME main.pages.example.codeberg.page.`
+
+3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
+for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
+record that points to your repo (just like the CNAME record):
+	`example.org IN ALIAS codeberg.page.`
+	`example.org IN TXT main.pages.example.codeberg.page.`
+
+Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
+
+
+## Deployment
+
+**Warning: Some Caveats Apply**  
+> Currently, the deployment requires you to have some knowledge of system administration as well as understanding and building code,
+> so you can eventually edit non-configurable and codeberg-specific settings.
+> In the future, we'll try to reduce these and make hosting Codeberg Pages as easy as setting up Gitea.
+> If you consider using Pages in practice, please consider contacting us first,
+> we'll then try to share some basic steps and document the current usage for admins
+> (might be changing in the current state).
+
+Deploying the software itself is very easy. You can grab a current release binary or build yourself,
+configure the environment as described below, and you are done.
+
+The hard part is about adding **custom domain support** if you intend to use it.
+SSL certificates (request + renewal) is automatically handled by the Pages Server,
+but if you want to run it on a shared IP address (and not a standalone),
+you'll need to configure your reverse proxy not to terminate the TLS connections,
+but forward the requests on the IP level to the Pages Server.
+
+You can check out a proof of concept in the `haproxy-sni` folder,
+and especially have a look at [this section of the haproxy.cfg](https://codeberg.org/Codeberg/pages-server/src/branch/main/haproxy-sni/haproxy.cfg#L38).
+
+### Environment
 
 - `HOST` & `PORT` (default: `[::]` & `443`): listen address.
 - `PAGES_DOMAIN` (default: `codeberg.page`): main domain for pages.
@@ -17,23 +72,38 @@
   See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
 - `DEBUG` (default: false): Set this to true to enable debug logging.
 
-```
-// Package main is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
-//
-// Mapping custom domains is not static anymore, but can be done with DNS:
-//
-// 1) add a ".domains" text file to your repository, containing the allowed domains, separated by new lines. The
-// first line will be the canonical domain/URL; all other occurrences will be redirected to it.
-//
-// 2) add a CNAME entry to your domain, pointing to "[[{branch}.]{repo}.]{owner}.codeberg.page" (repo defaults to
-// "pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
-//      www.example.org. IN CNAME main.pages.example.codeberg.page.
-//
-// 3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
-// for "example.org" (if your provider allows ALIAS or similar records, otherwise use A/AAAA), together with a TXT
-// record that points to your repo (just like the CNAME record):
-//      example.org IN ALIAS codeberg.page.
-//      example.org IN TXT main.pages.example.codeberg.page.
-//
-// Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
-```
+
+## Contributing to the development
+
+The Codeberg team is very open to your contribution.
+Since we are working nicely in a team, it might be hard at times to get started
+(still check out the issues, we always aim to have some things to get you started).
+
+If you have any questions, want to work on a feature or could imagine collaborating with us for some time,
+feel free to ping us in an issue or in a general Matrix chatgroup.
+
+You can also contact the maintainers of this project:
+
+- [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space)
+- [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de)
+
+### First steps
+
+The code of this repository is split in several modules.
+While heavy refactoring work is currently undergo, you can easily understand the basic structure:
+The `cmd` folder holds the data necessary for interacting with the service via the cli.
+If you are considering to deploy the service yourself, make sure to check it out.
+The heart of the software lives in the `server` folder and is split in several modules.
+After scanning the code, you should quickly be able to understand their function and start hacking on them.
+
+Again: Feel free to get in touch with us for any questions that might arise.
+Thank you very much.
+
+
+### Test Server
+
+run `just dev`
+now this pages should work:
+ - https://magiclike.localhost.mock.directory:4430/
+ - https://momar.localhost.mock.directory:4430/ci-testing/
+ - https://momar.localhost.mock.directory:4430/pag/@master/

cmd/certs.go

@@ -2,8 +2,8 @@ package cmd
 
 import (
 	"fmt"
-	"os"
 
+	"github.com/akrylysov/pogreb"
 	"github.com/urfave/cli/v2"
 
 	"codeberg.org/codeberg/pages/server/database"
@@ -12,17 +12,46 @@ import (
 var Certs = &cli.Command{
 	Name:  "certs",
 	Usage: "manage certs manually",
-	Action: certs,
+	Subcommands: []*cli.Command{
+		{
+			Name:   "list",
+			Usage:  "list all certificates in the database",
+			Action: listCerts,
+		},
+		{
+			Name:   "remove",
+			Usage:  "remove a certificate from the database",
+			Action: removeCert,
+		},
+	},
 }
 
-func certs(ctx *cli.Context) error {
-	if ctx.Args().Len() >= 1 && ctx.Args().First() == "--remove-certificate" {
-		if ctx.Args().Len() == 1 {
-			println("--remove-certificate requires at least one domain as an argument")
-			os.Exit(1)
+func listCerts(ctx *cli.Context) error {
+	// TODO: make "key-database.pogreb" set via flag
+	keyDatabase, err := database.New("key-database.pogreb")
+	if err != nil {
+		return fmt.Errorf("could not create database: %v", err)
 	}
 
-		domains := ctx.Args().Slice()[2:]
+	items := keyDatabase.Items()
+	for domain, _, err := items.Next(); err != pogreb.ErrIterationDone; domain, _, err = items.Next() {
+		if err != nil {
+			return err
+		}
+		if domain[0] == '.' {
+			fmt.Printf("*")
+		}
+		fmt.Printf("%s\n", domain)
+	}
+	return nil
+}
+
+func removeCert(ctx *cli.Context) error {
+	if ctx.Args().Len() < 1 {
+		return fmt.Errorf("'certs remove' requires at least one domain as an argument")
+	}
+
+	domains := ctx.Args().Slice()
 
 	// TODO: make "key-database.pogreb" set via flag
 	keyDatabase, err := database.New("key-database.pogreb")
@@ -31,14 +60,13 @@ func certs(ctx *cli.Context) error {
 	}
 
 	for _, domain := range domains {
-			if err := keyDatabase.Delete([]byte(domain)); err != nil {
-				panic(err)
+		fmt.Printf("Removing domain %s from the database...\n", domain)
+		if err := keyDatabase.Delete(domain); err != nil {
+			return err
 		}
 	}
 	if err := keyDatabase.Close(); err != nil {
-			panic(err)
-		}
-		os.Exit(0)
+		return err
 	}
 	return nil
 }

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -72,18 +73,24 @@ func Serve(ctx *cli.Context) error {
 	keyCache := cache.NewKeyValueCache()
 	challengeCache := cache.NewKeyValueCache()
 	// canonicalDomainCache stores canonical domains
-	var canonicalDomainCache = cache.NewKeyValueCache()
+	canonicalDomainCache := cache.NewKeyValueCache()
 	// dnsLookupCache stores DNS lookups for custom domains
-	var dnsLookupCache = cache.NewKeyValueCache()
+	dnsLookupCache := cache.NewKeyValueCache()
 	// branchTimestampCache stores branch timestamps for faster cache checking
-	var branchTimestampCache = cache.NewKeyValueCache()
+	branchTimestampCache := cache.NewKeyValueCache()
 	// fileResponseCache stores responses from the Gitea server
 	// TODO: make this an MRU cache with a size limit
-	var fileResponseCache = cache.NewKeyValueCache()
+	fileResponseCache := cache.NewKeyValueCache()
+
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken)
+	if err != nil {
+		return fmt.Errorf("could not create new gitea client: %v", err)
+	}
 
 	// Create handler based on settings
 	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
-		giteaRoot, rawInfoPage, giteaAPIToken,
+		giteaClient,
+		giteaRoot, rawInfoPage,
 		BlacklistedPaths, allowedCorsDomains,
 		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
 
@@ -105,7 +112,8 @@ func Serve(ctx *cli.Context) error {
 	defer certDB.Close() //nolint:errcheck    // database has no close ... sync behave like it
 
 	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
-		giteaRoot, giteaAPIToken, dnsProvider,
+		giteaClient,
+		dnsProvider,
 		acmeUseRateLimits,
 		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
 		certDB))
@@ -126,6 +134,7 @@ func Serve(ctx *cli.Context) error {
 
 	if enableHTTPServer {
 		go func() {
+			log.Info().Timestamp().Msg("Start listening on :80")
 			err := httpServer.ListenAndServe("[::]:80")
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
@@ -134,6 +143,7 @@ func Serve(ctx *cli.Context) error {
 	}
 
 	// Start the web fastServer
+	log.Info().Timestamp().Msgf("Start listening on %s", listener.Addr())
 	err = fastServer.Serve(listener)
 	if err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")

go.mod

@@ -1,6 +1,6 @@
 module codeberg.org/codeberg/pages
 
-go 1.16
+go 1.18
 
 require (
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
@@ -13,3 +13,115 @@ require (
 	github.com/valyala/fasthttp v1.31.0
 	github.com/valyala/fastjson v1.6.3
 )
+
+require (
+	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/azure-sdk-for-go v32.4.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.19 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
+	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
+	github.com/andybalholm/brotli v1.0.2 // indirect
+	github.com/aws/aws-sdk-go v1.39.0 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/cloudflare/cloudflare-go v0.20.0 // indirect
+	github.com/cpu/goacmedns v0.1.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deepmap/oapi-codegen v1.6.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dnsimple/dnsimple-go v0.70.1 // indirect
+	github.com/exoscale/egoscale v0.67.0 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.1.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gophercloud/gophercloud v0.16.0 // indirect
+	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
+	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
+	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
+	github.com/jarcoal/httpmock v1.0.6 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/joho/godotenv v1.4.0 // indirect
+	github.com/json-iterator/go v1.1.7 // indirect
+	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
+	github.com/klauspost/compress v1.13.4 // indirect
+	github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
+	github.com/labbsr0x/goh v1.0.1 // indirect
+	github.com/linode/linodego v0.31.1 // indirect
+	github.com/liquidweb/go-lwApi v0.0.5 // indirect
+	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
+	github.com/liquidweb/liquidweb-go v1.6.3 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
+	github.com/nrdcg/auroradns v1.0.1 // indirect
+	github.com/nrdcg/desec v0.6.0 // indirect
+	github.com/nrdcg/dnspod-go v0.4.0 // indirect
+	github.com/nrdcg/freemyip v0.2.0 // indirect
+	github.com/nrdcg/goinwx v0.8.1 // indirect
+	github.com/nrdcg/namesilo v0.2.1 // indirect
+	github.com/nrdcg/porkbun v0.1.1 // indirect
+	github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
+	github.com/ovh/go-ovh v1.1.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pquerna/otp v1.3.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/sacloud/libsacloud v1.36.2 // indirect
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
+	github.com/softlayer/softlayer-go v1.0.3 // indirect
+	github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/transip/gotransip/v6 v6.6.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
+	github.com/vultr/govultr/v2 v2.7.1 // indirect
+	go.opencensus.io v0.22.3 // indirect
+	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
+	google.golang.org/api v0.20.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect
+	google.golang.org/grpc v1.27.1 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

go.sum

@@ -266,6 +266,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -486,7 +488,6 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
 github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
 github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=

html/404.html

@@ -21,12 +21,13 @@
     </style>
 </head>
 <body>
-    <i class="fa fa-bug text-primary" style="font-size: 96px;"></i>
+    <i class="fa fa-search text-primary" style="font-size: 96px;"></i>
     <h1 class="mb-0 text-primary">
-    	You found a bug!
+		Page not found!
     </h1>
     <h5 class="text-center" style="max-width: 25em;">
-    	Sorry, this page doesn't exist or is inaccessible for other reasons (%status)
+    	Sorry, but this page couldn't be found or is inaccessible (%status).<br/>
+		We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
     </h5>
     <small class="text-muted">
         <img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">

integration/get_test.go

@@ -0,0 +1,108 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"net/http"
+	"net/http/cookiejar"
+	"testing"
+
+	"github.com/rs/zerolog/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetRedirect(t *testing.T) {
+	log.Printf("=== TestGetRedirect ===\n")
+	// test custom domain redirect
+	resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
+	assert.EqualValues(t, 0, getSize(resp.Body))
+}
+
+func TestGetContent(t *testing.T) {
+	log.Printf("=== TestGetContent ===\n")
+	// test get image
+	resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "image/jpeg", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "124635", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 124635, getSize(resp.Body))
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// specify branch
+	resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 1000)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// access branch name contains '/'
+	resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 100)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
+}
+
+func TestCustomDomain(t *testing.T) {
+	log.Printf("=== TestCustomDomain ===\n")
+	resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 106, getSize(resp.Body))
+}
+
+func TestGetNotFound(t *testing.T) {
+	log.Printf("=== TestGetNotFound ===\n")
+	// test custom not found pages
+	resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 37, getSize(resp.Body))
+}
+
+func getTestHTTPSClient() *http.Client {
+	cookieJar, _ := cookiejar.New(nil)
+	return &http.Client{
+		Jar: cookieJar,
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+}
+
+func getSize(stream io.Reader) int {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Len()
+}

integration/main_test.go

@@ -0,0 +1,62 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"codeberg.org/codeberg/pages/cmd"
+
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+)
+
+func TestMain(m *testing.M) {
+	log.Printf("=== TestMain: START Server ===\n")
+	serverCtx, serverCancel := context.WithCancel(context.Background())
+	if err := startServer(serverCtx); err != nil {
+		log.Fatal().Msgf("could not start server: %v", err)
+	}
+	defer func() {
+		serverCancel()
+		log.Printf("=== TestMain: Server STOPED ===\n")
+	}()
+
+	time.Sleep(10 * time.Second)
+
+	os.Exit(m.Run())
+}
+
+func startServer(ctx context.Context) error {
+	args := []string{
+		"--verbose",
+		"--acme-accept-terms", "true",
+	}
+	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
+	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
+	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PORT", "4430")
+
+	app := cli.NewApp()
+	app.Name = "pages-server"
+	app.Action = cmd.Serve
+	app.Flags = cmd.ServeFlags
+
+	go func() {
+		if err := app.RunContext(ctx, args); err != nil {
+			log.Fatal().Msgf("run server error: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+func setEnvIfNotSet(key, value string) {
+	if _, set := os.LookupEnv(key); !set {
+		os.Setenv(key, value)
+	}
+}

main.go

@@ -4,15 +4,14 @@ import (
 	"fmt"
 	"os"
 
+	_ "github.com/joho/godotenv/autoload"
 	"github.com/urfave/cli/v2"
 
 	"codeberg.org/codeberg/pages/cmd"
 )
 
-var (
 // can be changed with -X on compile
-	version = "dev"
-)
+var version = "dev"
 
 func main() {
 	app := cli.NewApp()

server/certificates/acme_account.go

@@ -19,9 +19,11 @@ var _ registration.User = &AcmeAccount{}
 func (u *AcmeAccount) GetEmail() string {
 	return u.Email
 }
+
 func (u AcmeAccount) GetRegistration() *registration.Resource {
 	return u.Registration
 }
+
 func (u *AcmeAccount) GetPrivateKey() crypto.PrivateKey {
 	return u.Key
 }

server/certificates/certificates.go

@@ -32,15 +32,18 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/database"
 	dnsutils "codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
 func TLSConfig(mainDomainSuffix []byte,
-	giteaRoot, giteaAPIToken, dnsProvider string,
+	giteaClient *gitea.Client,
+	dnsProvider string,
 	acmeUseRateLimits bool,
 	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
-	certDB database.CertDB) *tls.Config {
+	certDB database.CertDB,
+) *tls.Config {
 	return &tls.Config{
 		// check DNS name & get certificate from Let's Encrypt
 		GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
@@ -80,7 +83,7 @@ func TLSConfig(mainDomainSuffix []byte,
 					sni = string(sniBytes)
 				} else {
 					_, _ = targetRepo, targetBranch
-					_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+					_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
 					if !valid {
 						sniBytes = mainDomainSuffix
 						sni = string(sniBytes)
@@ -146,8 +149,10 @@ func checkUserLimit(user string) error {
 	return nil
 }
 
-var acmeClient, mainDomainAcmeClient *lego.Client
-var acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+var (
+	acmeClient, mainDomainAcmeClient  *lego.Client
+	acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
+)
 
 // rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
 // TODO: when this is used a lot, we probably have to think of a somewhat better solution?
@@ -166,6 +171,7 @@ var _ challenge.Provider = AcmeTLSChallengeProvider{}
 func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
 	return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
 }
+
 func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
 	a.challengeCache.Remove(domain)
 	return nil
@@ -181,6 +187,7 @@ var _ challenge.Provider = AcmeHTTPChallengeProvider{}
 func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
 	return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
 }
+
 func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 	a.challengeCache.Remove(domain + "/" + token)
 	return nil
@@ -188,7 +195,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 
 func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
 	// parse certificate from database
-	res, err := certDB.Get(sni)
+	res, err := certDB.Get(string(sni))
 	if err != nil {
 		panic(err) // TODO: no panic
 	}
@@ -209,7 +216,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
 		}
 
 		// renew certificates 7 days before they expire
-		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(-7 * 24 * time.Hour)) {
+		if !tlsCertificate.Leaf.NotAfter.After(time.Now().Add(7 * 24 * time.Hour)) {
 			// TODO: add ValidUntil to custom res struct
 			if res.CSR != nil && len(res.CSR) > 0 {
 				// CSR stores the time when the renewal shall be tried again
@@ -388,7 +395,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 				log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
 				select {}
 			}
-			err = ioutil.WriteFile(configFile, acmeAccountJSON, 0600)
+			err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
 			if err != nil {
 				log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
 				select {}
@@ -401,7 +408,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 
 func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
-	mainCertBytes, err := certDB.Get(mainDomainSuffix)
+	mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
 	if err != nil {
 		return fmt.Errorf("cert database is not working")
 	}
@@ -473,7 +480,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 
 				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 				if err != nil || !tlsCertificates[0].NotAfter.After(now) {
-					err := certDB.Delete(key)
+					err := certDB.Delete(string(key))
 					if err != nil {
 						log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
 					} else {
@@ -486,15 +493,15 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 		log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
 
 		// compact the database
-		result, err := certDB.Compact()
+		msg, err := certDB.Compact()
 		if err != nil {
 			log.Printf("[ERROR] Compacting key database failed: %s", err)
 		} else {
-			log.Printf("[INFO] Compacted key database (%+v)", result)
+			log.Printf("[INFO] Compacted key database (%s)", msg)
 		}
 
 		// update main cert
-		res, err := certDB.Get(mainDomainSuffix)
+		res, err := certDB.Get(string(mainDomainSuffix))
 		if err != nil {
 			log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
 		} else if res == nil {
@@ -503,7 +510,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 			tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 
 			// renew main certificate 30 days before it expires
-			if !tlsCertificates[0].NotAfter.After(time.Now().Add(-30 * 24 * time.Hour)) {
+			if !tlsCertificates[0].NotAfter.After(time.Now().Add(30 * 24 * time.Hour)) {
 				go (func() {
 					_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
 					if err != nil {

server/certificates/mock_test.go

@@ -0,0 +1,17 @@
+package certificates
+
+import (
+	"testing"
+
+	"codeberg.org/codeberg/pages/server/database"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMockCert(t *testing.T) {
+	db, err := database.NewTmpDB()
+	assert.NoError(t, err)
+	cert := mockCert("example.com", "some error msg", "codeberg.page", db)
+	if assert.NotEmpty(t, cert) {
+		assert.NotEmpty(t, cert.Certificate)
+	}
+}

server/database/interface.go

@@ -8,8 +8,8 @@ import (
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error
-	Get(name []byte) (*certificate.Resource, error)
-	Delete(key []byte) error
-	Compact() (pogreb.CompactionResult, error)
+	Get(name string) (*certificate.Resource, error)
+	Delete(key string) error
+	Compact() (string, error)
 	Items() *pogreb.ItemIterator
 }

server/database/mock.go

@@ -0,0 +1,55 @@
+package database
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/OrlovEvgeny/go-mcache"
+	"github.com/akrylysov/pogreb"
+	"github.com/go-acme/lego/v4/certificate"
+)
+
+var _ CertDB = tmpDB{}
+
+type tmpDB struct {
+	intern *mcache.CacheDriver
+	ttl    time.Duration
+}
+
+func (p tmpDB) Close() error {
+	_ = p.intern.Close()
+	return nil
+}
+
+func (p tmpDB) Put(name string, cert *certificate.Resource) error {
+	return p.intern.Set(name, cert, p.ttl)
+}
+
+func (p tmpDB) Get(name string) (*certificate.Resource, error) {
+	cert, has := p.intern.Get(name)
+	if !has {
+		return nil, fmt.Errorf("cert for '%s' not found", name)
+	}
+	return cert.(*certificate.Resource), nil
+}
+
+func (p tmpDB) Delete(key string) error {
+	p.intern.Remove(key)
+	return nil
+}
+
+func (p tmpDB) Compact() (string, error) {
+	p.intern.Truncate()
+	return "Truncate done", nil
+}
+
+func (p tmpDB) Items() *pogreb.ItemIterator {
+	panic("ItemIterator not implemented for tmpDB")
+}
+
+func NewTmpDB() (CertDB, error) {
+	return &tmpDB{
+		intern: mcache.New(),
+		ttl:    time.Minute,
+	}, nil
+}

server/database/setup.go

@@ -13,6 +13,8 @@ import (
 	"github.com/rs/zerolog/log"
 )
 
+var _ CertDB = aDB{}
+
 type aDB struct {
 	ctx          context.Context
 	cancel       context.CancelFunc
@@ -33,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error {
 	return p.intern.Put([]byte(name), resGob.Bytes())
 }
 
-func (p aDB) Get(name []byte) (*certificate.Resource, error) {
+func (p aDB) Get(name string) (*certificate.Resource, error) {
 	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get(name)
+	resBytes, err := p.intern.Get([]byte(name))
 	if err != nil {
 		return nil, err
 	}
@@ -48,12 +50,16 @@ func (p aDB) Get(name []byte) (*certificate.Resource, error) {
 	return cert, nil
 }
 
-func (p aDB) Delete(key []byte) error {
-	return p.intern.Delete(key)
+func (p aDB) Delete(key string) error {
+	return p.intern.Delete([]byte(key))
 }
 
-func (p aDB) Compact() (pogreb.CompactionResult, error) {
-	return p.intern.Compact()
+func (p aDB) Compact() (string, error) {
+	result, err := p.intern.Compact()
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%+v", result), nil
 }
 
 func (p aDB) Items() *pogreb.ItemIterator {
@@ -76,20 +82,6 @@ func (p aDB) sync() {
 	}
 }
 
-func (p aDB) compact() {
-	for {
-		err := p.intern.Sync()
-		if err != nil {
-			log.Err(err).Msg("Syncing cert database failed")
-		}
-		select {
-		case <-p.ctx.Done():
-			return
-		case <-time.After(p.syncInterval):
-		}
-	}
-}
-
 func New(path string) (CertDB, error) {
 	if path == "" {
 		return nil, fmt.Errorf("path not set")

server/gitea/client.go

@@ -0,0 +1,134 @@
+package gitea
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/valyala/fasthttp"
+	"github.com/valyala/fastjson"
+)
+
+const giteaAPIRepos = "/api/v1/repos/"
+
+var ErrorNotFound = errors.New("not found")
+
+type Client struct {
+	giteaRoot      string
+	giteaAPIToken  string
+	fastClient     *fasthttp.Client
+	infoTimeout    time.Duration
+	contentTimeout time.Duration
+}
+
+type FileResponse struct {
+	Exists   bool
+	ETag     []byte
+	MimeType string
+	Body     []byte
+}
+
+// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
+func joinURL(baseURL string, paths ...string) string {
+	p := make([]string, 0, len(paths))
+	for i := range paths {
+		path := strings.TrimSpace(paths[i])
+		path = strings.Trim(path, "/")
+		if len(path) != 0 {
+			p = append(p, path)
+		}
+	}
+
+	return baseURL + "/" + strings.Join(p, "/")
+}
+
+func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
+
+func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
+	rootURL, err := url.Parse(giteaRoot)
+	giteaRoot = strings.Trim(rootURL.String(), "/")
+
+	return &Client{
+		giteaRoot:      giteaRoot,
+		giteaAPIToken:  giteaAPIToken,
+		infoTimeout:    5 * time.Second,
+		contentTimeout: 10 * time.Second,
+		fastClient:     getFastHTTPClient(),
+	}, err
+}
+
+func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	res, err := client.do(client.contentTimeout, url)
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode() {
+	case fasthttp.StatusOK:
+		return res.Body(), nil
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+}
+
+func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
+	res, err := client.do(client.contentTimeout, url)
+	if err != nil {
+		return nil, err
+	}
+
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode() {
+	case fasthttp.StatusOK:
+		return res, nil
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+}
+
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return time.Time{}, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+}
+
+func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return fastjson.GetString(res.Body(), "default_branch"), nil
+}
+
+func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
+	req := fasthttp.AcquireRequest()
+
+	req.SetRequestURI(url)
+	req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	err := client.fastClient.DoTimeout(req, res, timeout)
+
+	return res, err
+}

server/gitea/client_test.go

@@ -0,0 +1,23 @@
+package gitea
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJoinURL(t *testing.T) {
+	baseURL := ""
+	assert.EqualValues(t, "/", joinURL(baseURL))
+	assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
+
+	baseURL = "http://wwow.url.com"
+	assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
+
+	baseURL = "http://wow.url.com/subpath/2"
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
+}

server/gitea/fasthttp.go

@@ -0,0 +1,15 @@
+package gitea
+
+import (
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+func getFastHTTPClient() *fasthttp.Client {
+	return &fasthttp.Client{
+		MaxConnDuration:    60 * time.Second,
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
+}

server/handler.go

@@ -4,25 +4,30 @@ import (
 	"bytes"
 	"strings"
 
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 	"codeberg.org/codeberg/pages/server/utils"
+	"codeberg.org/codeberg/pages/server/version"
 )
 
 // Handler handles a single HTTP request to the web server.
 func Handler(mainDomainSuffix, rawDomain []byte,
-	giteaRoot, rawInfoPage, giteaAPIToken string,
+	giteaClient *gitea.Client,
+	giteaRoot, rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains [][]byte,
-	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) func(ctx *fasthttp.RequestCtx) {
+	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) func(ctx *fasthttp.RequestCtx) {
 	return func(ctx *fasthttp.RequestCtx) {
 		log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
 
-		ctx.Response.Header.Set("Server", "Codeberg Pages")
+		ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
 
 		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
 		ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -53,7 +58,6 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 		}
 
 		// Allow CORS for specified domains
-		if ctx.IsOptions() {
 		allowCors := false
 		for _, allowedCorsDomain := range allowedCorsDomains {
 			if bytes.Equal(trimmedHost, allowedCorsDomain) {
@@ -66,28 +70,33 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
 		}
 		ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
+		if ctx.IsOptions() {
 			ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
 			return
 		}
 
 		// Prepare request information to Gitea
 		var targetOwner, targetRepo, targetBranch, targetPath string
-		var targetOptions = &upstream.Options{
-			ForbiddenMimeTypes: map[string]struct{}{},
+		targetOptions := &upstream.Options{
 			TryIndexPages: true,
 		}
 
 		// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
 		// also disallow search indexing and add a Link header to the canonical URL.
-		var tryBranch = func(repo string, branch string, path []string, canonicalLink string) bool {
+		tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
 			if repo == "" {
+				log.Debug().Msg("tryBranch: repo == ''")
 				return false
 			}
 
+			// Replace "~" to "/" so we can access branch that contains slash character
+			// Branch name cannot contain "~" so doing this is okay
+			branch = strings.ReplaceAll(branch, "~", "/")
+
 			// Check if the branch exists, otherwise treat it as a file path
-			branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
+			branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
 			if branchTimestampResult == nil {
-				// branch doesn't exist
+				log.Debug().Msg("tryBranch: branch doesn't exist")
 				return false
 			}
 
@@ -107,6 +116,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				)
 			}
 
+			log.Debug().Msg("tryBranch: true")
 			return true
 		}
 
@@ -116,7 +126,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			log.Debug().Msg("raw domain")
 
 			targetOptions.TryIndexPages = false
-			targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
+			if targetOptions.ForbiddenMimeTypes == nil {
+				targetOptions.ForbiddenMimeTypes = make(map[string]bool)
+			}
+			targetOptions.ForbiddenMimeTypes["text/html"] = true
 			targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
 
 			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
@@ -131,13 +144,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// raw.codeberg.org/example/myrepo/@main/index.html
 			if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
 				log.Debug().Msg("raw domain preparations, now trying with specified branch")
-				if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
+				if tryBranch(log,
+					targetRepo, pathElements[2][1:], pathElements[3:],
 					giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 				) {
-					log.Debug().Msg("tryBranch, now trying upstream")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					log.Debug().Msg("tryBranch, now trying upstream 1")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 					return
 				}
@@ -147,13 +160,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			}
 
 			log.Debug().Msg("raw domain preparations, now trying with default branch")
-			tryBranch(targetRepo, "", pathElements[2:],
+			tryBranch(log,
+				targetRepo, "", pathElements[2:],
 				giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 			)
-			log.Debug().Msg("tryBranch, now trying upstream")
-			tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			log.Debug().Msg("tryBranch, now trying upstream 2")
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 				targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-				giteaRoot, giteaAPIToken,
 				canonicalDomainCache, branchTimestampCache, fileResponseCache)
 			return
 
@@ -182,13 +195,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				}
 
 				log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
-				if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
+				if tryBranch(log,
+					pathElements[0], pathElements[1][1:], pathElements[2:],
 					"/"+pathElements[0]+"/%p",
 				) {
-					log.Debug().Msg("tryBranch, now trying upstream")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					log.Debug().Msg("tryBranch, now trying upstream 3")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -200,11 +213,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/@main/index.html
 			if strings.HasPrefix(pathElements[0], "@") {
 				log.Debug().Msg("main domain preparations, now trying with specified branch")
-				if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
-					log.Debug().Msg("tryBranch, now trying upstream")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				if tryBranch(log,
+					"pages", pathElements[0][1:], pathElements[1:], "/%p") {
+					log.Debug().Msg("tryBranch, now trying upstream 4")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -216,11 +229,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/myrepo/index.html
 			// example.codeberg.page/pages/... is not allowed here.
 			log.Debug().Msg("main domain preparations, now trying with specified repo")
-			if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
-				log.Debug().Msg("tryBranch, now trying upstream")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			if pathElements[0] != "pages" && tryBranch(log,
+				pathElements[0], "pages", pathElements[1:], "") {
+				log.Debug().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -228,11 +241,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// Try to use the "pages" repo on its default branch
 			// example.codeberg.page/index.html
 			log.Debug().Msg("main domain preparations, now trying with default repo/branch")
-			if tryBranch("pages", "", pathElements, "") {
-				log.Debug().Msg("tryBranch, now trying upstream")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			if tryBranch(log,
+				"pages", "", pathElements, "") {
+				log.Debug().Msg("tryBranch, now trying upstream 6")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -260,8 +273,9 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 
 			// Try to use the given repo on the given branch or the default branch
 			log.Debug().Msg("custom domain preparations, now trying with details from DNS")
-			if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
-				canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+			if tryBranch(log,
+				targetRepo, targetBranch, pathElements, canonicalLink) {
+				canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
 				if !valid {
 					html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
 					return
@@ -277,10 +291,9 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 					return
 				}
 
-				log.Debug().Msg("tryBranch, now trying upstream")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				log.Debug().Msg("tryBranch, now trying upstream 7")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}

server/handler_test.go

@@ -2,20 +2,22 @@ package server
 
 import (
 	"fmt"
-	"github.com/valyala/fasthttp"
 	"testing"
 	"time"
 
+	"github.com/valyala/fasthttp"
+
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 func TestHandlerPerformance(t *testing.T) {
+	giteaRoot := "https://codeberg.org"
+	giteaClient, _ := gitea.NewClient(giteaRoot, "")
 	testHandler := Handler(
-		[]byte("codeberg.page"),
-		[]byte("raw.codeberg.org"),
-		"https://codeberg.org",
-		"https://docs.codeberg.org/pages/raw-content/",
-		"",
+		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
+		giteaClient,
+		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
 		[][]byte{[]byte("/.well-known/acme-challenge/")},
 		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
 		cache.NewKeyValueCache(),
@@ -24,46 +26,26 @@ func TestHandlerPerformance(t *testing.T) {
 		cache.NewKeyValueCache(),
 	)
 
+	testCase := func(uri string, status int) {
 		ctx := &fasthttp.RequestCtx{
 			Request:  *fasthttp.AcquireRequest(),
 			Response: *fasthttp.AcquireResponse(),
 		}
-	ctx.Request.SetRequestURI("http://mondstern.codeberg.page/")
+		ctx.Request.SetRequestURI(uri)
 		fmt.Printf("Start: %v\n", time.Now())
 		start := time.Now()
 		testHandler(ctx)
 		end := time.Now()
 		fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
+		if ctx.Response.StatusCode() != status {
+			t.Errorf("request failed with status code %d", ctx.Response.StatusCode())
 		} else {
 			t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
 		}
+	}
 
-	ctx.Response.Reset()
-	ctx.Response.ResetBody()
-	fmt.Printf("Start: %v\n", time.Now())
-	start = time.Now()
-	testHandler(ctx)
-	end = time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 2048 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
-
-	ctx.Response.Reset()
-	ctx.Response.ResetBody()
-	ctx.Request.SetRequestURI("http://example.momar.xyz/")
-	fmt.Printf("Start: %v\n", time.Now())
-	start = time.Now()
-	testHandler(ctx)
-	end = time.Now()
-	fmt.Printf("Done: %v\n", time.Now())
-	if ctx.Response.StatusCode() != 200 || len(ctx.Response.Body()) < 1 {
-		t.Errorf("request failed with status code %d and body length %d", ctx.Response.StatusCode(), len(ctx.Response.Body()))
-	} else {
-		t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
-	}
+	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
+	testCase("https://mondstern.codeberg.page/", 424) // TODO: expect 200
+	testCase("https://example.momar.xyz/", 424)       // TODO: expect 200
+	testCase("https://codeberg.page/", 424)           // TODO: expect 200
 }

server/setup.go

@@ -5,12 +5,20 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/rs/zerolog/log"
+
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/utils"
 )
 
+type fasthttpLogger struct{}
+
+func (fasthttpLogger) Printf(format string, args ...interface{}) {
+	log.Printf("[FASTHTTP] "+format, args...)
+}
+
 func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
 	// Enable compression by wrapping the handler with the compression function provided by FastHTTP
 	compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
@@ -18,12 +26,10 @@ func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
 	return &fasthttp.Server{
 		Handler:                      compressedHandler,
 		DisablePreParseMultipartForm: true,
-		MaxRequestBodySize:           0,
 		NoDefaultServerHeader:        true,
 		NoDefaultDate:                true,
 		ReadTimeout:                  30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
-		Concurrency:                  1024 * 32,        // TODO: adjust bottlenecks for best performance with Gitea!
-		MaxConnsPerIP:                100,
+		Logger:                       fasthttpLogger{},
 	}
 }
 

server/try.go

@@ -8,22 +8,22 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx,
+func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	mainDomainSuffix, trimmedHost []byte,
 
 	targetOptions *upstream.Options,
-	targetOwner, targetRepo, targetBranch, targetPath,
-
-	giteaRoot, giteaAPIToken string,
-	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey) {
+	targetOwner, targetRepo, targetBranch, targetPath string,
 
+	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+) {
 	// check if a canonical domain exists on a request on MainDomain
 	if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
-		canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+		canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
 			canonicalPath := string(ctx.RequestURI())
 			if targetRepo != "pages" {
@@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
 	targetOptions.TargetPath = targetPath
 
 	// Try to request the file from the Gitea API
-	if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+	if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
 		html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
 	}
 }

server/upstream/const.go

@@ -12,6 +12,7 @@ var branchExistenceCacheTimeout = 5 * time.Minute
 
 // fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
 // on your available memory.
+// TODO: move as option into cache interface
 var fileCacheTimeout = 5 * time.Minute
 
 // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
@@ -19,3 +20,5 @@ var fileCacheSizeLimit = 1024 * 1024
 
 // canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
 var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"

server/upstream/domains.go

@@ -3,15 +3,16 @@ package upstream
 import (
 	"strings"
 
-	"github.com/valyala/fasthttp"
-
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
-	domains := []string{}
-	valid := false
+func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+	var (
+		domains []string
+		valid   bool
+	)
 	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
 		domains = cachedValue.([]string)
 		for _, domain := range domains {
@@ -21,13 +22,9 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
 			}
 		}
 	} else {
-		req := fasthttp.AcquireRequest()
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
-		res := fasthttp.AcquireResponse()
-
-		err := client.Do(req, res)
-		if err == nil && res.StatusCode() == fasthttp.StatusOK {
-			for _, domain := range strings.Split(string(res.Body()), "\n") {
+		body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
+		if err == nil {
+			for _, domain := range strings.Split(string(body), "\n") {
 				domain = strings.ToLower(domain)
 				domain = strings.TrimSpace(domain)
 				domain = strings.TrimPrefix(domain, "http://")

server/upstream/helper.go

@@ -1,12 +1,14 @@
 package upstream
 
 import (
+	"mime"
+	"path"
+	"strconv"
+	"strings"
 	"time"
 
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 type branchTimestamp struct {
@@ -16,40 +18,55 @@ type branchTimestamp struct {
 
 // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
 // (or nil if the branch doesn't exist)
-func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
 	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
 		if result == nil {
 			return nil
 		}
 		return result.(*branchTimestamp)
 	}
-	result := &branchTimestamp{}
-	result.Branch = branch
-	if branch == "" {
+	result := &branchTimestamp{
+		Branch: branch,
+	}
+	if len(branch) == 0 {
 		// Get default branch
-		var body = make([]byte, 0)
-		// TODO: use header for API key?
-		status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
-		if err != nil || status != 200 {
-			_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
+		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
+		if err != nil {
+			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
 			return nil
 		}
-		result.Branch = fastjson.GetString(body, "default_branch")
+		result.Branch = defaultBranch
 	}
 
-	var body = make([]byte, 0)
-	status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
-	if err != nil || status != 200 {
+	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
+	if err != nil {
 		return nil
 	}
-
-	result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
+	result.Timestamp = timestamp
 	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
 	return result
 }
 
-type fileResponse struct {
-	exists   bool
-	mimeType string
-	body     []byte
+func (o *Options) getMimeTypeByExtension() string {
+	if o.ForbiddenMimeTypes == nil {
+		o.ForbiddenMimeTypes = make(map[string]bool)
+	}
+	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+	if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
+		if o.DefaultMimeType != "" {
+			mimeType = o.DefaultMimeType
+		} else {
+			mimeType = "application/octet-stream"
+		}
+	}
+	return mimeType
+}
+
+func (o *Options) generateUri() string {
+	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+}
+
+func (o *Options) timestamp() string {
+	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -2,11 +2,9 @@ package upstream
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
-	"mime"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -15,6 +13,7 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // upstreamIndexPages lists pages that may be considered as index pages for directories.
@@ -22,6 +21,11 @@ var upstreamIndexPages = []string{
 	"index.html",
 }
 
+// upstreamNotFoundPages lists pages that may be considered as custom 404 Not Found pages.
+var upstreamNotFoundPages = []string{
+	"404.html",
+}
+
 // Options provides various options for the upstream request.
 type Options struct {
 	TargetOwner,
@@ -30,7 +34,7 @@ type Options struct {
 	TargetPath,
 
 	DefaultMimeType string
-	ForbiddenMimeTypes map[string]struct{}
+	ForbiddenMimeTypes map[string]bool
 	TryIndexPages      bool
 	BranchTimestamp    time.Time
 	// internal
@@ -38,24 +42,13 @@ type Options struct {
 	redirectIfExists    string
 }
 
-var client = fasthttp.Client{
-	ReadTimeout:        10 * time.Second,
-	MaxConnDuration:    60 * time.Second,
-	MaxConnWaitTimeout: 1000 * time.Millisecond,
-	MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-}
-
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
 	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
 
-	if o.ForbiddenMimeTypes == nil {
-		o.ForbiddenMimeTypes = map[string]struct{}{}
-	}
-
 	// Check if the branch exists and when it was modified
 	if o.BranchTimestamp.IsZero() {
-		branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
+		branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
 
 		if branch == nil {
 			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -80,24 +73,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("preparations")
 
 	// Make a GET request to the upstream URL
-	uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
-	var req *fasthttp.Request
+	uri := o.generateUri()
 	var res *fasthttp.Response
-	var cachedResponse fileResponse
+	var cachedResponse gitea.FileResponse
 	var err error
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
-		cachedResponse = cachedValue.(fileResponse)
+	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
+		cachedResponse = cachedValue.(gitea.FileResponse)
 	} else {
-		req = fasthttp.AcquireRequest()
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
-		res = fasthttp.AcquireResponse()
-		res.SetBodyStream(&strings.Reader{}, -1)
-		err = client.Do(req, res)
+		res, err = giteaClient.ServeRawContent(uri)
 	}
 	log.Debug().Msg("acquisition")
 
 	// Handle errors
-	if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
+	if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -105,9 +93,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
-						exists: false,
+				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
 					}, fileCacheTimeout)
 					return true
 				}
@@ -116,24 +104,39 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = false
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
-				_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
-					exists: false,
+			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+					Exists: false,
 				}, fileCacheTimeout)
 				return true
 			}
 		}
 		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+		if o.TryIndexPages {
+			// copy the o struct & try if a not found page exists
+			optionsForNotFoundPages := *o
+			optionsForNotFoundPages.TryIndexPages = false
+			optionsForNotFoundPages.appendTrailingSlash = false
+			for _, notFoundPage := range upstreamNotFoundPages {
+				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
+					}, fileCacheTimeout)
+					return true
+				}
+			}
+		}
 		if res != nil {
 			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
-				exists: false,
+			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+				Exists: false,
 			}, fileCacheTimeout)
 		}
 		return false
 	}
 	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
+		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
@@ -155,19 +158,21 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("error handling")
 
 	// Set the MIME type
-	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
-		if o.DefaultMimeType != "" {
-			mimeType = o.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
-		}
-	}
+	mimeType := o.getMimeTypeByExtension()
 	ctx.Response.Header.SetContentType(mimeType)
 
+	// Set ETag
+	if cachedResponse.Exists {
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	} else if res != nil {
+		cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	}
+
+	if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
 		// Everything's okay so far
 		ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	}
 	ctx.Response.Header.SetLastModified(o.BranchTimestamp)
 
 	log.Debug().Msg("response preparations")
@@ -176,26 +181,29 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	var cacheBodyWriter bytes.Buffer
 	if res != nil {
 		if res.Header.ContentLength() > fileCacheSizeLimit {
+			// fasthttp else will set "Content-Length: 0"
+			ctx.Response.SetBodyStream(&strings.Reader{}, -1)
+
 			err = res.BodyWriteTo(ctx.Response.BodyWriter())
 		} else {
 			// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
 			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
 		}
 	} else {
-		_, err = ctx.Write(cachedResponse.body)
+		_, err = ctx.Write(cachedResponse.Body)
 	}
 	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
+		fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
 	log.Debug().Msg("response")
 
-	if res != nil && ctx.Err() == nil {
-		cachedResponse.exists = true
-		cachedResponse.mimeType = mimeType
-		cachedResponse.body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
+	if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
+		cachedResponse.Exists = true
+		cachedResponse.MimeType = mimeType
+		cachedResponse.Body = cacheBodyWriter.Bytes()
+		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
 	}
 
 	return true

server/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string = "dev"