Release via CI (#94)

* release via CI
* general CI improvements

close #76, close #92

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/94

.woodpecker.yml

@@ -1,24 +1,69 @@
+branches: main
+
 pipeline:
   # use vendor to cache dependencies
   vendor:
-    image: golang
+    image: golang:1.18
     commands:
       - go mod vendor
 
   lint:
-    image: golangci/golangci-lint:v1.45.2
+    image: golangci/golangci-lint:latest
+    group: compliant
+    pull: true
     commands:
       - go version
       - go install mvdan.cc/gofumpt@latest
       - "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
-      - golangci-lint run
+      - golangci-lint run --timeout 5m --build-tags integration
-
-  test:
-    image: golang
-    commands:
-      - go test ./...
 
   build:
-    image: golang
+    group: compliant
+    image: a6543/golang_just
     commands:
-      - go build
+      - go version
+      - just build
+    when:
+      event: [ "pull_request", "push" ]
+
+  build-tag:
+    group: compliant
+    image: a6543/golang_just
+    commands:
+      - go version
+      - just build-tag ${CI_COMMIT_TAG##v}
+    when:
+      event: [ "tag" ]
+
+  test:
+    image: a6543/golang_just
+    group: test
+    commands:
+      - just test
+
+  integration-tests:
+    image: a6543/golang_just
+    group: test
+    commands:
+      - just integration
+    environment:
+      - ACME_API=https://acme.mock.directory
+      - PAGES_DOMAIN=localhost.mock.directory
+      - RAW_DOMAIN=raw.localhost.mock.directory
+      - PORT=4430
+
+  release:
+    image: plugins/gitea-release
+    settings:
+      base_url: https://codeberg.org
+      file_exists: overwrite
+      files: build/codeberg-pages-server
+      api_key:
+        from_secret: bot_token
+    environment:
+      - DRONE_REPO_OWNER=${CI_REPO_OWNER}
+      - DRONE_REPO_NAME=${CI_REPO_NAME}
+      - DRONE_BUILD_EVENT=${CI_BUILD_EVENT}
+      - DRONE_COMMIT_REF=${CI_COMMIT_REF}
+    when:
+      event: [ "tag" ]

Justfile

@@ -10,3 +10,35 @@ dev:
 
 build:
     CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
+
+build-tag VERSION:
+    CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./
+
+lint: tool-golangci tool-gofumpt
+    [ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
+    golangci-lint run --timeout 5m --build-tags integration
+
+fmt: tool-gofumpt
+    gofumpt -w --extra .
+
+tool-golangci:
+    @hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \
+    fi
+
+tool-gofumpt:
+    @hash gofumpt> /dev/null 2>&1; if [ $? -ne 0 ]; then \
+    go install mvdan.cc/gofumpt@latest; \
+    fi
+
+test:
+    go test -race codeberg.org/codeberg/pages/server/...
+
+test-run TEST:
+    go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/...
+
+integration:
+    go test -race -tags integration codeberg.org/codeberg/pages/integration/...
+
+integration-run TEST:
+    go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

README.md

@@ -97,3 +97,12 @@ After scanning the code, you should quickly be able to understand their function
 
 Again: Feel free to get in touch with us for any questions that might arise.
 Thank you very much.
+
+
+### Test Server
+
+run `just dev`
+now this pages should work:
+ - https://magiclike.localhost.mock.directory:4430/
+ - https://momar.localhost.mock.directory:4430/ci-testing/
+ - https://momar.localhost.mock.directory:4430/pag/@master/

cmd/certs.go

@@ -61,7 +61,7 @@ func removeCert(ctx *cli.Context) error {
 
 	for _, domain := range domains {
 		fmt.Printf("Removing domain %s from the database...\n", domain)
-		if err := keyDatabase.Delete([]byte(domain)); err != nil {
+		if err := keyDatabase.Delete(domain); err != nil {
 			return err
 		}
 	}

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/certificates"
 	"codeberg.org/codeberg/pages/server/database"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -81,9 +82,15 @@ func Serve(ctx *cli.Context) error {
 	// TODO: make this an MRU cache with a size limit
 	fileResponseCache := cache.NewKeyValueCache()
 
+	giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken)
+	if err != nil {
+		return fmt.Errorf("could not create new gitea client: %v", err)
+	}
+
 	// Create handler based on settings
 	handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
-		giteaRoot, rawInfoPage, giteaAPIToken,
+		giteaClient,
+		giteaRoot, rawInfoPage,
 		BlacklistedPaths, allowedCorsDomains,
 		dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
 
@@ -105,7 +112,8 @@ func Serve(ctx *cli.Context) error {
 	defer certDB.Close() //nolint:errcheck    // database has no close ... sync behave like it
 
 	listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
-		giteaRoot, giteaAPIToken, dnsProvider,
+		giteaClient,
+		dnsProvider,
 		acmeUseRateLimits,
 		keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
 		certDB))
@@ -126,6 +134,7 @@ func Serve(ctx *cli.Context) error {
 
 	if enableHTTPServer {
 		go func() {
+			log.Info().Timestamp().Msg("Start listening on :80")
 			err := httpServer.ListenAndServe("[::]:80")
 			if err != nil {
 				log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
@@ -134,6 +143,7 @@ func Serve(ctx *cli.Context) error {
 	}
 
 	// Start the web fastServer
+	log.Info().Timestamp().Msgf("Start listening on %s", listener.Addr())
 	err = fastServer.Serve(listener)
 	if err != nil {
 		log.Panic().Err(err).Msg("Couldn't start fastServer")

go.mod

@@ -1,6 +1,6 @@
 module codeberg.org/codeberg/pages
 
-go 1.16
+go 1.18
 
 require (
 	github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
@@ -13,3 +13,115 @@ require (
 	github.com/valyala/fasthttp v1.31.0
 	github.com/valyala/fastjson v1.6.3
 )
+
+require (
+	cloud.google.com/go v0.54.0 // indirect
+	github.com/Azure/azure-sdk-for-go v32.4.0+incompatible // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.11.19 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
+	github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
+	github.com/Azure/go-autorest/logger v0.2.1 // indirect
+	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
+	github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
+	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
+	github.com/andybalholm/brotli v1.0.2 // indirect
+	github.com/aws/aws-sdk-go v1.39.0 // indirect
+	github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
+	github.com/cenkalti/backoff/v4 v4.1.1 // indirect
+	github.com/cloudflare/cloudflare-go v0.20.0 // indirect
+	github.com/cpu/goacmedns v0.1.1 // indirect
+	github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/deepmap/oapi-codegen v1.6.1 // indirect
+	github.com/dimchansky/utfbom v1.1.1 // indirect
+	github.com/dnsimple/dnsimple-go v0.70.1 // indirect
+	github.com/exoscale/egoscale v0.67.0 // indirect
+	github.com/fatih/structs v1.1.0 // indirect
+	github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
+	github.com/go-errors/errors v1.0.1 // indirect
+	github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
+	github.com/gofrs/uuid v3.2.0+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.1.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gophercloud/gophercloud v0.16.0 // indirect
+	github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
+	github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
+	github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
+	github.com/jarcoal/httpmock v1.0.6 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/joho/godotenv v1.4.0 // indirect
+	github.com/json-iterator/go v1.1.7 // indirect
+	github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
+	github.com/klauspost/compress v1.13.4 // indirect
+	github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
+	github.com/labbsr0x/goh v1.0.1 // indirect
+	github.com/linode/linodego v0.31.1 // indirect
+	github.com/liquidweb/go-lwApi v0.0.5 // indirect
+	github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
+	github.com/liquidweb/liquidweb-go v1.6.3 // indirect
+	github.com/mattn/go-isatty v0.0.12 // indirect
+	github.com/miekg/dns v1.1.43 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
+	github.com/nrdcg/auroradns v1.0.1 // indirect
+	github.com/nrdcg/desec v0.6.0 // indirect
+	github.com/nrdcg/dnspod-go v0.4.0 // indirect
+	github.com/nrdcg/freemyip v0.2.0 // indirect
+	github.com/nrdcg/goinwx v0.8.1 // indirect
+	github.com/nrdcg/namesilo v0.2.1 // indirect
+	github.com/nrdcg/porkbun v0.1.1 // indirect
+	github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
+	github.com/ovh/go-ovh v1.1.0 // indirect
+	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/pquerna/otp v1.3.0 // indirect
+	github.com/russross/blackfriday/v2 v2.0.1 // indirect
+	github.com/sacloud/libsacloud v1.36.2 // indirect
+	github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
+	github.com/softlayer/softlayer-go v1.0.3 // indirect
+	github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/stretchr/objx v0.3.0 // indirect
+	github.com/transip/gotransip/v6 v6.6.1 // indirect
+	github.com/valyala/bytebufferpool v1.0.0 // indirect
+	github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
+	github.com/vultr/govultr/v2 v2.7.1 // indirect
+	go.opencensus.io v0.22.3 // indirect
+	go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
+	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
+	golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
+	golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e // indirect
+	golang.org/x/text v0.3.6 // indirect
+	golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
+	google.golang.org/api v0.20.0 // indirect
+	google.golang.org/appengine v1.6.5 // indirect
+	google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect
+	google.golang.org/grpc v1.27.1 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
+	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
+)

go.sum

@@ -266,6 +266,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
+github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
+github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -486,7 +488,6 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
 github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
 github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
-github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
 github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=

integration/get_test.go

@@ -0,0 +1,96 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"net/http"
+	"net/http/cookiejar"
+	"testing"
+
+	"github.com/rs/zerolog/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetRedirect(t *testing.T) {
+	log.Printf("=== TestGetRedirect ===\n")
+	// test custom domain redirect
+	resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
+	assert.EqualValues(t, 0, getSize(resp.Body))
+}
+
+func TestGetContent(t *testing.T) {
+	log.Printf("=== TestGetContent ===\n")
+	// test get image
+	resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "image/jpeg", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "124635", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 124635, getSize(resp.Body))
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+
+	// specify branch
+	resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.True(t, getSize(resp.Body) > 1000)
+	assert.Len(t, resp.Header.Get("ETag"), 42)
+}
+
+func TestCustomDomain(t *testing.T) {
+	log.Printf("=== TestCustomDomain ===\n")
+	resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 106, getSize(resp.Body))
+}
+
+func TestGetNotFound(t *testing.T) {
+	log.Printf("=== TestGetNotFound ===\n")
+	// test custom not found pages
+	resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
+	assert.NoError(t, err)
+	if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
+		t.FailNow()
+	}
+	assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
+	assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
+	assert.EqualValues(t, 37, getSize(resp.Body))
+}
+
+func getTestHTTPSClient() *http.Client {
+	cookieJar, _ := cookiejar.New(nil)
+	return &http.Client{
+		Jar: cookieJar,
+		CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		},
+	}
+}
+
+func getSize(stream io.Reader) int {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.Len()
+}

integration/main_test.go

@@ -0,0 +1,62 @@
+//go:build integration
+// +build integration
+
+package integration
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"codeberg.org/codeberg/pages/cmd"
+
+	"github.com/rs/zerolog/log"
+	"github.com/urfave/cli/v2"
+)
+
+func TestMain(m *testing.M) {
+	log.Printf("=== TestMain: START Server ===\n")
+	serverCtx, serverCancel := context.WithCancel(context.Background())
+	if err := startServer(serverCtx); err != nil {
+		log.Fatal().Msgf("could not start server: %v", err)
+	}
+	defer func() {
+		serverCancel()
+		log.Printf("=== TestMain: Server STOPED ===\n")
+	}()
+
+	time.Sleep(10 * time.Second)
+
+	os.Exit(m.Run())
+}
+
+func startServer(ctx context.Context) error {
+	args := []string{
+		"--verbose",
+		"--acme-accept-terms", "true",
+	}
+	setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
+	setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
+	setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
+	setEnvIfNotSet("PORT", "4430")
+
+	app := cli.NewApp()
+	app.Name = "pages-server"
+	app.Action = cmd.Serve
+	app.Flags = cmd.ServeFlags
+
+	go func() {
+		if err := app.RunContext(ctx, args); err != nil {
+			log.Fatal().Msgf("run server error: %v", err)
+		}
+	}()
+
+	return nil
+}
+
+func setEnvIfNotSet(key, value string) {
+	if _, set := os.LookupEnv(key); !set {
+		os.Setenv(key, value)
+	}
+}

main.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	_ "github.com/joho/godotenv/autoload"
 	"github.com/urfave/cli/v2"
 
 	"codeberg.org/codeberg/pages/cmd"

server/certificates/certificates.go

@@ -32,12 +32,14 @@ import (
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/database"
 	dnsutils "codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
 func TLSConfig(mainDomainSuffix []byte,
-	giteaRoot, giteaAPIToken, dnsProvider string,
+	giteaClient *gitea.Client,
+	dnsProvider string,
 	acmeUseRateLimits bool,
 	keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
 	certDB database.CertDB,
@@ -81,7 +83,7 @@ func TLSConfig(mainDomainSuffix []byte,
 					sni = string(sniBytes)
 				} else {
 					_, _ = targetRepo, targetBranch
-					_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+					_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
 					if !valid {
 						sniBytes = mainDomainSuffix
 						sni = string(sniBytes)
@@ -193,7 +195,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
 
 func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
 	// parse certificate from database
-	res, err := certDB.Get(sni)
+	res, err := certDB.Get(string(sni))
 	if err != nil {
 		panic(err) // TODO: no panic
 	}
@@ -406,7 +408,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
 
 func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
 	// getting main cert before ACME account so that we can fail here without hitting rate limits
-	mainCertBytes, err := certDB.Get(mainDomainSuffix)
+	mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
 	if err != nil {
 		return fmt.Errorf("cert database is not working")
 	}
@@ -478,7 +480,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 
 				tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
 				if err != nil || !tlsCertificates[0].NotAfter.After(now) {
-					err := certDB.Delete(key)
+					err := certDB.Delete(string(key))
 					if err != nil {
 						log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
 					} else {
@@ -491,15 +493,15 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
 		log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
 
 		// compact the database
-		result, err := certDB.Compact()
+		msg, err := certDB.Compact()
 		if err != nil {
 			log.Printf("[ERROR] Compacting key database failed: %s", err)
 		} else {
-			log.Printf("[INFO] Compacted key database (%+v)", result)
+			log.Printf("[INFO] Compacted key database (%s)", msg)
 		}
 
 		// update main cert
-		res, err := certDB.Get(mainDomainSuffix)
+		res, err := certDB.Get(string(mainDomainSuffix))
 		if err != nil {
 			log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
 		} else if res == nil {

server/certificates/mock_test.go

@@ -0,0 +1,17 @@
+package certificates
+
+import (
+	"testing"
+
+	"codeberg.org/codeberg/pages/server/database"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMockCert(t *testing.T) {
+	db, err := database.NewTmpDB()
+	assert.NoError(t, err)
+	cert := mockCert("example.com", "some error msg", "codeberg.page", db)
+	if assert.NotEmpty(t, cert) {
+		assert.NotEmpty(t, cert.Certificate)
+	}
+}

server/database/interface.go

@@ -8,8 +8,8 @@ import (
 type CertDB interface {
 	Close() error
 	Put(name string, cert *certificate.Resource) error
-	Get(name []byte) (*certificate.Resource, error)
+	Get(name string) (*certificate.Resource, error)
-	Delete(key []byte) error
+	Delete(key string) error
-	Compact() (pogreb.CompactionResult, error)
+	Compact() (string, error)
 	Items() *pogreb.ItemIterator
 }

server/database/mock.go

@@ -0,0 +1,55 @@
+package database
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/OrlovEvgeny/go-mcache"
+	"github.com/akrylysov/pogreb"
+	"github.com/go-acme/lego/v4/certificate"
+)
+
+var _ CertDB = tmpDB{}
+
+type tmpDB struct {
+	intern *mcache.CacheDriver
+	ttl    time.Duration
+}
+
+func (p tmpDB) Close() error {
+	_ = p.intern.Close()
+	return nil
+}
+
+func (p tmpDB) Put(name string, cert *certificate.Resource) error {
+	return p.intern.Set(name, cert, p.ttl)
+}
+
+func (p tmpDB) Get(name string) (*certificate.Resource, error) {
+	cert, has := p.intern.Get(name)
+	if !has {
+		return nil, fmt.Errorf("cert for '%s' not found", name)
+	}
+	return cert.(*certificate.Resource), nil
+}
+
+func (p tmpDB) Delete(key string) error {
+	p.intern.Remove(key)
+	return nil
+}
+
+func (p tmpDB) Compact() (string, error) {
+	p.intern.Truncate()
+	return "Truncate done", nil
+}
+
+func (p tmpDB) Items() *pogreb.ItemIterator {
+	panic("ItemIterator not implemented for tmpDB")
+}
+
+func NewTmpDB() (CertDB, error) {
+	return &tmpDB{
+		intern: mcache.New(),
+		ttl:    time.Minute,
+	}, nil
+}

server/database/setup.go

@@ -35,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error {
 	return p.intern.Put([]byte(name), resGob.Bytes())
 }
 
-func (p aDB) Get(name []byte) (*certificate.Resource, error) {
+func (p aDB) Get(name string) (*certificate.Resource, error) {
 	cert := &certificate.Resource{}
-	resBytes, err := p.intern.Get(name)
+	resBytes, err := p.intern.Get([]byte(name))
 	if err != nil {
 		return nil, err
 	}
@@ -50,12 +50,16 @@ func (p aDB) Get(name []byte) (*certificate.Resource, error) {
 	return cert, nil
 }
 
-func (p aDB) Delete(key []byte) error {
+func (p aDB) Delete(key string) error {
-	return p.intern.Delete(key)
+	return p.intern.Delete([]byte(key))
 }
 
-func (p aDB) Compact() (pogreb.CompactionResult, error) {
+func (p aDB) Compact() (string, error) {
-	return p.intern.Compact()
+	result, err := p.intern.Compact()
+	if err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("%+v", result), nil
 }
 
 func (p aDB) Items() *pogreb.ItemIterator {

server/gitea/client.go

@@ -0,0 +1,135 @@
+package gitea
+
+import (
+	"errors"
+	"fmt"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/valyala/fasthttp"
+	"github.com/valyala/fastjson"
+)
+
+const giteaAPIRepos = "/api/v1/repos/"
+
+var ErrorNotFound = errors.New("not found")
+
+type Client struct {
+	giteaRoot      string
+	giteaAPIToken  string
+	fastClient     *fasthttp.Client
+	infoTimeout    time.Duration
+	contentTimeout time.Duration
+}
+
+type FileResponse struct {
+	Exists   bool
+	ETag     []byte
+	MimeType string
+	Body     []byte
+}
+
+// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
+func joinURL(baseURL string, paths ...string) string {
+	p := make([]string, 0, len(paths))
+	for i := range paths {
+		path := strings.TrimSpace(paths[i])
+		path = strings.Trim(path, "/")
+		if len(path) != 0 {
+			p = append(p, path)
+		}
+	}
+
+	return baseURL + "/" + strings.Join(p, "/")
+}
+
+func (f FileResponse) IsEmpty() bool { return len(f.Body) != 0 }
+
+func NewClient(giteaRoot, giteaAPIToken string) (*Client, error) {
+	rootURL, err := url.Parse(giteaRoot)
+	giteaRoot = strings.Trim(rootURL.String(), "/")
+
+	return &Client{
+		giteaRoot:      giteaRoot,
+		giteaAPIToken:  giteaAPIToken,
+		infoTimeout:    5 * time.Second,
+		contentTimeout: 10 * time.Second,
+		fastClient:     getFastHTTPClient(),
+	}, err
+}
+
+func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
+	res, err := client.do(client.contentTimeout, url)
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode() {
+	case fasthttp.StatusOK:
+		return res.Body(), nil
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+}
+
+func (client *Client) ServeRawContent(uri string) (*fasthttp.Response, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, uri)
+	res, err := client.do(client.contentTimeout, url)
+	if err != nil {
+		return nil, err
+	}
+	// resp.SetBodyStream(&strings.Reader{}, -1)
+
+	if err != nil {
+		return nil, err
+	}
+
+	switch res.StatusCode() {
+	case fasthttp.StatusOK:
+		return res, nil
+	case fasthttp.StatusNotFound:
+		return nil, ErrorNotFound
+	default:
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+}
+
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return time.Time{}, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+}
+
+func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
+	url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
+	res, err := client.do(client.infoTimeout, url)
+	if err != nil {
+		return "", err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return fastjson.GetString(res.Body(), "default_branch"), nil
+}
+
+func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
+	req := fasthttp.AcquireRequest()
+
+	req.SetRequestURI(url)
+	req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	err := client.fastClient.DoTimeout(req, res, timeout)
+
+	return res, err
+}

server/gitea/client_test.go

@@ -0,0 +1,23 @@
+package gitea
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestJoinURL(t *testing.T) {
+	baseURL := ""
+	assert.EqualValues(t, "/", joinURL(baseURL))
+	assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
+
+	baseURL = "http://wwow.url.com"
+	assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
+
+	baseURL = "http://wow.url.com/subpath/2"
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
+	assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
+}

server/gitea/fasthttp.go

@@ -0,0 +1,15 @@
+package gitea
+
+import (
+	"time"
+
+	"github.com/valyala/fasthttp"
+)
+
+func getFastHTTPClient() *fasthttp.Client {
+	return &fasthttp.Client{
+		MaxConnDuration:    60 * time.Second,
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
+}

server/handler.go

@@ -4,26 +4,30 @@ import (
 	"bytes"
 	"strings"
 
+	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
 	"codeberg.org/codeberg/pages/server/dns"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 	"codeberg.org/codeberg/pages/server/utils"
+	"codeberg.org/codeberg/pages/server/version"
 )
 
 // Handler handles a single HTTP request to the web server.
 func Handler(mainDomainSuffix, rawDomain []byte,
-	giteaRoot, rawInfoPage, giteaAPIToken string,
+	giteaClient *gitea.Client,
+	giteaRoot, rawInfoPage string,
 	blacklistedPaths, allowedCorsDomains [][]byte,
 	dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
 ) func(ctx *fasthttp.RequestCtx) {
 	return func(ctx *fasthttp.RequestCtx) {
 		log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
 
-		ctx.Response.Header.Set("Server", "Codeberg Pages")
+		ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
 
 		// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
 		ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -74,21 +78,21 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 		// Prepare request information to Gitea
 		var targetOwner, targetRepo, targetBranch, targetPath string
 		targetOptions := &upstream.Options{
-			ForbiddenMimeTypes: map[string]struct{}{},
+			TryIndexPages: true,
-			TryIndexPages:      true,
 		}
 
 		// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
 		// also disallow search indexing and add a Link header to the canonical URL.
-		tryBranch := func(repo, branch string, path []string, canonicalLink string) bool {
+		tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
 			if repo == "" {
+				log.Debug().Msg("tryBranch: repo == ''")
 				return false
 			}
 
 			// Check if the branch exists, otherwise treat it as a file path
-			branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
+			branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
 			if branchTimestampResult == nil {
-				// branch doesn't exist
+				log.Debug().Msg("tryBranch: branch doesn't exist")
 				return false
 			}
 
@@ -108,6 +112,7 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				)
 			}
 
+			log.Debug().Msg("tryBranch: true")
 			return true
 		}
 
@@ -117,7 +122,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			log.Debug().Msg("raw domain")
 
 			targetOptions.TryIndexPages = false
-			targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
+			if targetOptions.ForbiddenMimeTypes == nil {
+				targetOptions.ForbiddenMimeTypes = make(map[string]bool)
+			}
+			targetOptions.ForbiddenMimeTypes["text/html"] = true
 			targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
 
 			pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
@@ -132,13 +140,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// raw.codeberg.org/example/myrepo/@main/index.html
 			if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
 				log.Debug().Msg("raw domain preparations, now trying with specified branch")
-				if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
+				if tryBranch(log,
+					targetRepo, pathElements[2][1:], pathElements[3:],
 					giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 				) {
-					log.Debug().Msg("tryBranch, now trying upstream")
+					log.Debug().Msg("tryBranch, now trying upstream 1")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 					return
 				}
@@ -148,13 +156,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			}
 
 			log.Debug().Msg("raw domain preparations, now trying with default branch")
-			tryBranch(targetRepo, "", pathElements[2:],
+			tryBranch(log,
+				targetRepo, "", pathElements[2:],
 				giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
 			)
-			log.Debug().Msg("tryBranch, now trying upstream")
+			log.Debug().Msg("tryBranch, now trying upstream 2")
-			tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+			tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 				targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-				giteaRoot, giteaAPIToken,
 				canonicalDomainCache, branchTimestampCache, fileResponseCache)
 			return
 
@@ -183,13 +191,13 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 				}
 
 				log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
-				if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
+				if tryBranch(log,
+					pathElements[0], pathElements[1][1:], pathElements[2:],
 					"/"+pathElements[0]+"/%p",
 				) {
-					log.Debug().Msg("tryBranch, now trying upstream")
+					log.Debug().Msg("tryBranch, now trying upstream 3")
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -201,11 +209,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/@main/index.html
 			if strings.HasPrefix(pathElements[0], "@") {
 				log.Debug().Msg("main domain preparations, now trying with specified branch")
-				if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
+				if tryBranch(log,
-					log.Debug().Msg("tryBranch, now trying upstream")
+					"pages", pathElements[0][1:], pathElements[1:], "/%p") {
-					tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+					log.Debug().Msg("tryBranch, now trying upstream 4")
+					tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 						targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-						giteaRoot, giteaAPIToken,
 						canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				} else {
 					html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -217,11 +225,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// example.codeberg.page/myrepo/index.html
 			// example.codeberg.page/pages/... is not allowed here.
 			log.Debug().Msg("main domain preparations, now trying with specified repo")
-			if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
+			if pathElements[0] != "pages" && tryBranch(log,
-				log.Debug().Msg("tryBranch, now trying upstream")
+				pathElements[0], "pages", pathElements[1:], "") {
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				log.Debug().Msg("tryBranch, now trying upstream 5")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -229,11 +237,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 			// Try to use the "pages" repo on its default branch
 			// example.codeberg.page/index.html
 			log.Debug().Msg("main domain preparations, now trying with default repo/branch")
-			if tryBranch("pages", "", pathElements, "") {
+			if tryBranch(log,
-				log.Debug().Msg("tryBranch, now trying upstream")
+				"pages", "", pathElements, "") {
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				log.Debug().Msg("tryBranch, now trying upstream 6")
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}
@@ -261,8 +269,9 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 
 			// Try to use the given repo on the given branch or the default branch
 			log.Debug().Msg("custom domain preparations, now trying with details from DNS")
-			if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
+			if tryBranch(log,
-				canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+				targetRepo, targetBranch, pathElements, canonicalLink) {
+				canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
 				if !valid {
 					html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
 					return
@@ -278,10 +287,9 @@ func Handler(mainDomainSuffix, rawDomain []byte,
 					return
 				}
 
-				log.Debug().Msg("tryBranch, now trying upstream")
+				log.Debug().Msg("tryBranch, now trying upstream 7")
-				tryUpstream(ctx, mainDomainSuffix, trimmedHost,
+				tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
 					targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
-					giteaRoot, giteaAPIToken,
 					canonicalDomainCache, branchTimestampCache, fileResponseCache)
 				return
 			}

server/handler_test.go

@@ -8,15 +8,16 @@ import (
 	"github.com/valyala/fasthttp"
 
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 func TestHandlerPerformance(t *testing.T) {
+	giteaRoot := "https://codeberg.org"
+	giteaClient, _ := gitea.NewClient(giteaRoot, "")
 	testHandler := Handler(
-		[]byte("codeberg.page"),
+		[]byte("codeberg.page"), []byte("raw.codeberg.org"),
-		[]byte("raw.codeberg.org"),
+		giteaClient,
-		"https://codeberg.org",
+		giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
-		"https://docs.codeberg.org/pages/raw-content/",
-		"",
 		[][]byte{[]byte("/.well-known/acme-challenge/")},
 		[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
 		cache.NewKeyValueCache(),

server/setup.go

@@ -18,12 +18,10 @@ func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
 	return &fasthttp.Server{
 		Handler:                      compressedHandler,
 		DisablePreParseMultipartForm: true,
-		MaxRequestBodySize:           0,
 		NoDefaultServerHeader:        true,
 		NoDefaultDate:                true,
 		ReadTimeout:                  30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
 		Concurrency:                  1024 * 32,        // TODO: adjust bottlenecks for best performance with Gitea!
-		MaxConnsPerIP:                100,
 	}
 }
 

server/try.go

@@ -8,22 +8,22 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 	"codeberg.org/codeberg/pages/server/upstream"
 )
 
 // tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx,
+func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
 	mainDomainSuffix, trimmedHost []byte,
 
 	targetOptions *upstream.Options,
-	targetOwner, targetRepo, targetBranch, targetPath,
+	targetOwner, targetRepo, targetBranch, targetPath string,
 
-	giteaRoot, giteaAPIToken string,
 	canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
 ) {
 	// check if a canonical domain exists on a request on MainDomain
 	if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
-		canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
+		canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
 		if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
 			canonicalPath := string(ctx.RequestURI())
 			if targetRepo != "pages" {
@@ -43,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx,
 	targetOptions.TargetPath = targetPath
 
 	// Try to request the file from the Gitea API
-	if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+	if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
 		html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
 	}
 }

server/upstream/const.go

@@ -12,6 +12,7 @@ var branchExistenceCacheTimeout = 5 * time.Minute
 
 // fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
 // on your available memory.
+// TODO: move as option into cache interface
 var fileCacheTimeout = 5 * time.Minute
 
 // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
@@ -19,3 +20,5 @@ var fileCacheSizeLimit = 1024 * 1024
 
 // canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
 var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"

server/upstream/domains.go

@@ -3,15 +3,16 @@ package upstream
 import (
 	"strings"
 
-	"github.com/valyala/fasthttp"
-
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
-func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
-	domains := []string{}
+	var (
-	valid := false
+		domains []string
+		valid   bool
+	)
 	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
 		domains = cachedValue.([]string)
 		for _, domain := range domains {
@@ -21,13 +22,9 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
 			}
 		}
 	} else {
-		req := fasthttp.AcquireRequest()
+		body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
+		if err == nil {
-		res := fasthttp.AcquireResponse()
+			for _, domain := range strings.Split(string(body), "\n") {
-
-		err := client.Do(req, res)
-		if err == nil && res.StatusCode() == fasthttp.StatusOK {
-			for _, domain := range strings.Split(string(res.Body()), "\n") {
 				domain = strings.ToLower(domain)
 				domain = strings.TrimSpace(domain)
 				domain = strings.TrimPrefix(domain, "http://")

server/upstream/helper.go

@@ -1,12 +1,14 @@
 package upstream
 
 import (
+	"mime"
+	"path"
+	"strconv"
+	"strings"
 	"time"
 
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 type branchTimestamp struct {
@@ -16,40 +18,55 @@ type branchTimestamp struct {
 
 // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
 // (or nil if the branch doesn't exist)
-func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
 	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
 		if result == nil {
 			return nil
 		}
 		return result.(*branchTimestamp)
 	}
-	result := &branchTimestamp{}
+	result := &branchTimestamp{
-	result.Branch = branch
+		Branch: branch,
-	if branch == "" {
+	}
+	if len(branch) == 0 {
 		// Get default branch
-		body := make([]byte, 0)
+		defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
-		// TODO: use header for API key?
+		if err != nil {
-		status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
+			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
-		if err != nil || status != 200 {
-			_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
 			return nil
 		}
-		result.Branch = fastjson.GetString(body, "default_branch")
+		result.Branch = defaultBranch
 	}
 
-	body := make([]byte, 0)
+	timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
-	status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
+	if err != nil {
-	if err != nil || status != 200 {
 		return nil
 	}
-
+	result.Timestamp = timestamp
-	result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
 	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
 	return result
 }
 
-type fileResponse struct {
+func (o *Options) getMimeTypeByExtension() string {
-	exists   bool
+	if o.ForbiddenMimeTypes == nil {
-	mimeType string
+		o.ForbiddenMimeTypes = make(map[string]bool)
-	body     []byte
+	}
+	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+	if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
+		if o.DefaultMimeType != "" {
+			mimeType = o.DefaultMimeType
+		} else {
+			mimeType = "application/octet-stream"
+		}
+	}
+	return mimeType
+}
+
+func (o *Options) generateUri() string {
+	return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
+}
+
+func (o *Options) timestamp() string {
+	return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
 }

server/upstream/upstream.go

@@ -2,11 +2,9 @@ package upstream
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"io"
-	"mime"
-	"path"
-	"strconv"
 	"strings"
 	"time"
 
@@ -15,6 +13,7 @@ import (
 
 	"codeberg.org/codeberg/pages/html"
 	"codeberg.org/codeberg/pages/server/cache"
+	"codeberg.org/codeberg/pages/server/gitea"
 )
 
 // upstreamIndexPages lists pages that may be considered as index pages for directories.
@@ -22,6 +21,11 @@ var upstreamIndexPages = []string{
 	"index.html",
 }
 
+// upstreamNotFoundPages lists pages that may be considered as custom 404 Not Found pages.
+var upstreamNotFoundPages = []string{
+	"404.html",
+}
+
 // Options provides various options for the upstream request.
 type Options struct {
 	TargetOwner,
@@ -30,7 +34,7 @@ type Options struct {
 	TargetPath,
 
 	DefaultMimeType string
-	ForbiddenMimeTypes map[string]struct{}
+	ForbiddenMimeTypes map[string]bool
 	TryIndexPages      bool
 	BranchTimestamp    time.Time
 	// internal
@@ -38,24 +42,13 @@ type Options struct {
 	redirectIfExists    string
 }
 
-var client = fasthttp.Client{
-	ReadTimeout:        10 * time.Second,
-	MaxConnDuration:    60 * time.Second,
-	MaxConnWaitTimeout: 1000 * time.Millisecond,
-	MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
-}
-
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
+func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
 	log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
 
-	if o.ForbiddenMimeTypes == nil {
-		o.ForbiddenMimeTypes = map[string]struct{}{}
-	}
-
 	// Check if the branch exists and when it was modified
 	if o.BranchTimestamp.IsZero() {
-		branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
+		branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
 
 		if branch == nil {
 			html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -80,24 +73,19 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("preparations")
 
 	// Make a GET request to the upstream URL
-	uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
+	uri := o.generateUri()
-	var req *fasthttp.Request
 	var res *fasthttp.Response
-	var cachedResponse fileResponse
+	var cachedResponse gitea.FileResponse
 	var err error
-	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
+	if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
-		cachedResponse = cachedValue.(fileResponse)
+		cachedResponse = cachedValue.(gitea.FileResponse)
 	} else {
-		req = fasthttp.AcquireRequest()
+		res, err = giteaClient.ServeRawContent(uri)
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
-		res = fasthttp.AcquireResponse()
-		res.SetBodyStream(&strings.Reader{}, -1)
-		err = client.Do(req, res)
 	}
 	log.Debug().Msg("acquisition")
 
 	// Handle errors
-	if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
+	if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
 		if o.TryIndexPages {
 			// copy the o struct & try if an index page exists
 			optionsForIndexPages := *o
@@ -105,9 +93,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = true
 			for _, indexPage := range upstreamIndexPages {
 				optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
-				if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+				if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-					_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-						exists: false,
+						Exists: false,
 					}, fileCacheTimeout)
 					return true
 				}
@@ -116,24 +104,39 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			optionsForIndexPages.appendTrailingSlash = false
 			optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
 			optionsForIndexPages.TargetPath = o.TargetPath + ".html"
-			if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
+			if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
-				_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+				_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-					exists: false,
+					Exists: false,
 				}, fileCacheTimeout)
 				return true
 			}
 		}
 		ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+		if o.TryIndexPages {
+			// copy the o struct & try if a not found page exists
+			optionsForNotFoundPages := *o
+			optionsForNotFoundPages.TryIndexPages = false
+			optionsForNotFoundPages.appendTrailingSlash = false
+			for _, notFoundPage := range upstreamNotFoundPages {
+				optionsForNotFoundPages.TargetPath = "/" + notFoundPage
+				if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
+					_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
+						Exists: false,
+					}, fileCacheTimeout)
+					return true
+				}
+			}
+		}
 		if res != nil {
 			// Update cache if the request is fresh
-			_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
+			_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
-				exists: false,
+				Exists: false,
 			}, fileCacheTimeout)
 		}
 		return false
 	}
 	if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
-		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
+		fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", uri, err, res.StatusCode())
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
@@ -155,19 +158,21 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("error handling")
 
 	// Set the MIME type
-	mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
+	mimeType := o.getMimeTypeByExtension()
-	mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
-	if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
-		if o.DefaultMimeType != "" {
-			mimeType = o.DefaultMimeType
-		} else {
-			mimeType = "application/octet-stream"
-		}
-	}
 	ctx.Response.Header.SetContentType(mimeType)
 
-	// Everything's okay so far
+	// Set ETag
-	ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	if cachedResponse.Exists {
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	} else if res != nil {
+		cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
+		ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+	}
+
+	if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
+		// Everything's okay so far
+		ctx.Response.SetStatusCode(fasthttp.StatusOK)
+	}
 	ctx.Response.Header.SetLastModified(o.BranchTimestamp)
 
 	log.Debug().Msg("response preparations")
@@ -182,20 +187,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 			err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
 		}
 	} else {
-		_, err = ctx.Write(cachedResponse.body)
+		_, err = ctx.Write(cachedResponse.Body)
 	}
 	if err != nil {
-		fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
+		fmt.Printf("Couldn't write body for \"%s\": %s\n", uri, err)
 		html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
 		return true
 	}
 	log.Debug().Msg("response")
 
 	if res != nil && ctx.Err() == nil {
-		cachedResponse.exists = true
+		cachedResponse.Exists = true
-		cachedResponse.mimeType = mimeType
+		cachedResponse.MimeType = mimeType
-		cachedResponse.body = cacheBodyWriter.Bytes()
+		cachedResponse.Body = cacheBodyWriter.Bytes()
-		_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
+		_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
 	}
 
 	return true

server/version/version.go

@@ -0,0 +1,3 @@
+package version
+
+var Version string = "dev"