1 Commits

Author SHA1 Message Date
Moritz Marquardt
12cb305a65 Change MaxConnsPerIP to 0 to fix too many connections from HAProxy (#77)
Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/77
Reviewed-by: 6543 <6543@noreply.codeberg.org>
Co-authored-by: Moritz Marquardt <momar@noreply.codeberg.org>
Co-committed-by: Moritz Marquardt <momar@noreply.codeberg.org>
2022-06-03 14:10:56 +02:00
31 changed files with 241 additions and 1072 deletions

View File

@@ -1,106 +1,24 @@
pipeline:
# use vendor to cache dependencies
vendor:
image: golang:1.18
image: golang
commands:
- go mod vendor
lint:
image: golangci/golangci-lint:latest
group: compliant
pull: true
image: golangci/golangci-lint:v1.45.2
commands:
- go version
- go install mvdan.cc/gofumpt@latest
- "[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }"
- golangci-lint run --timeout 5m --build-tags integration
build:
group: compliant
image: codeberg.org/6543/docker-images/golang_just
commands:
- go version
- just build
when:
event: [ "pull_request", "push" ]
docker-dryrun:
group: compliant
image: plugins/kaniko
settings:
dockerfile: Dockerfile
no_push: true
tags: latest
when:
event: [ "pull_request", "push" ]
path: Dockerfile
build-tag:
group: compliant
image: codeberg.org/6543/docker-images/golang_just
commands:
- go version
- just build-tag ${CI_COMMIT_TAG##v}
when:
event: [ "tag" ]
- golangci-lint run
test:
group: test
image: codeberg.org/6543/docker-images/golang_just
image: golang
commands:
- just test
- go test ./...
integration-tests:
group: test
image: codeberg.org/6543/docker-images/golang_just
build:
image: golang
commands:
- just integration
environment:
- ACME_API=https://acme.mock.directory
- PAGES_DOMAIN=localhost.mock.directory
- RAW_DOMAIN=raw.localhost.mock.directory
- PORT=4430
release:
image: plugins/gitea-release
settings:
base_url: https://codeberg.org
file_exists: overwrite
files: build/codeberg-pages-server
api_key:
from_secret: bot_token
environment:
- DRONE_REPO_OWNER=${CI_REPO_OWNER}
- DRONE_REPO_NAME=${CI_REPO_NAME}
- DRONE_BUILD_EVENT=${CI_BUILD_EVENT}
- DRONE_COMMIT_REF=${CI_COMMIT_REF}
when:
event: [ "tag" ]
docker-next:
image: plugins/kaniko
settings:
registry: codeberg.org
dockerfile: Dockerfile
repo: codeberg.org/codeberg/pages-server
tags: next
username:
from_secret: bot_user
password:
from_secret: bot_token
when:
event: [ "push" ]
docker-tag:
image: plugins/kaniko
settings:
registry: codeberg.org
dockerfile: Dockerfile
repo: codeberg.org/codeberg/pages-server
tag: [ latest, "${CI_COMMIT_TAG}" ]
username:
from_secret: bot_user
password:
from_secret: bot_token
when:
event: [ "tag" ]
- go build

View File

@@ -1,15 +0,0 @@
FROM golang:alpine as build
WORKDIR /workspace
RUN apk add ca-certificates
COPY . .
RUN CGO_ENABLED=0 go build .
FROM scratch
COPY --from=build /workspace/pages /pages
COPY --from=build \
/etc/ssl/certs/ca-certificates.crt \
/etc/ssl/certs/ca-certificates.crt
ENTRYPOINT ["/pages"]

View File

@@ -6,44 +6,7 @@ dev:
export PAGES_DOMAIN=localhost.mock.directory
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
export LOG_LEVEL=trace
go run .
go run . --verbose
build:
CGO_ENABLED=0 go build -ldflags '-s -w' -v -o build/codeberg-pages-server ./
build-tag VERSION:
CGO_ENABLED=0 go build -ldflags '-s -w -X "codeberg.org/codeberg/pages/server/version.Version={{VERSION}}"' -v -o build/codeberg-pages-server ./
lint: tool-golangci tool-gofumpt
[ $(gofumpt -extra -l . | wc -l) != 0 ] && { echo 'code not formated'; exit 1; }; \
golangci-lint run --timeout 5m --build-tags integration
fmt: tool-gofumpt
gofumpt -w --extra .
clean:
go clean ./...
rm -rf build/
tool-golangci:
@hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest; \
fi
tool-gofumpt:
@hash gofumpt> /dev/null 2>&1; if [ $? -ne 0 ]; then \
go install mvdan.cc/gofumpt@latest; \
fi
test:
go test -race codeberg.org/codeberg/pages/server/...
test-run TEST:
go test -race -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/...
integration:
go test -race -tags integration codeberg.org/codeberg/pages/integration/...
integration-run TEST:
go test -race -tags integration -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...

View File

@@ -18,8 +18,7 @@ Mapping custom domains is not static anymore, but can be done with DNS:
first line will be the canonical domain/URL; all other occurrences will be redirected to it.
2) add a CNAME entry to your domain, pointing to `[[{branch}.]{repo}.]{owner}.codeberg.page` (repo defaults to
"pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else.
If the branch name contains slash characters, you need to replace "/" in the branch name to "~"):
"pages", "branch" defaults to the default branch if "repo" is "pages", or to "pages" if "repo" is something else):
`www.example.org. IN CNAME main.pages.example.codeberg.page.`
3) if a CNAME is set for "www.example.org", you can redirect there from the naked domain by adding an ALIAS record
@@ -70,7 +69,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg
- `ENABLE_HTTP_SERVER` (default: false): Set this to true to enable the HTTP-01 challenge and redirect all other HTTP requests to HTTPS. Currently only works with port 80.
- `DNS_PROVIDER` (default: use self-signed certificate): Code of the ACME DNS provider for the main domain wildcard.
See https://go-acme.github.io/lego/dns/ for available values & additional environment variables.
- `LOG_LEVEL` (default: warn): Set this to specify the level of logging.
- `DEBUG` (default: false): Set this to true to enable debug logging.
## Contributing to the development
@@ -98,12 +97,3 @@ After scanning the code, you should quickly be able to understand their function
Again: Feel free to get in touch with us for any questions that might arise.
Thank you very much.
### Test Server
run `just dev`
now this pages should work:
- https://magiclike.localhost.mock.directory:4430/
- https://momar.localhost.mock.directory:4430/ci-testing/
- https://momar.localhost.mock.directory:4430/pag/@master/

View File

@@ -61,7 +61,7 @@ func removeCert(ctx *cli.Context) error {
for _, domain := range domains {
fmt.Printf("Removing domain %s from the database...\n", domain)
if err := keyDatabase.Delete(domain); err != nil {
if err := keyDatabase.Delete([]byte(domain)); err != nil {
return err
}
}

View File

@@ -5,6 +5,12 @@ import (
)
var ServeFlags = []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
// TODO: Usage
EnvVars: []string{"DEBUG"},
},
// MainDomainSuffix specifies the main domain (starting with a dot) for which subdomains shall be served as static
// pages, or used for comparison in CNAME lookups. Static pages can be accessed through
// https://{owner}.{MainDomain}[/{repo}], with repo defaulting to "pages".
@@ -63,25 +69,6 @@ var ServeFlags = []cli.Flag{
// TODO: desc
EnvVars: []string{"ENABLE_HTTP_SERVER"},
},
// Server Options
&cli.BoolFlag{
Name: "enable-lfs-support",
Usage: "enable lfs support, require gitea v1.17.0 as backend",
EnvVars: []string{"ENABLE_LFS_SUPPORT"},
Value: true,
},
&cli.BoolFlag{
Name: "enable-symlink-support",
Usage: "follow symlinks if enabled, require gitea v1.18.0 as backend",
EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
Value: true,
},
&cli.StringFlag{
Name: "log-level",
Value: "warn",
Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal",
EnvVars: []string{"LOG_LEVEL"},
},
// ACME
&cli.StringFlag{

View File

@@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"net"
"os"
"strings"
"time"
@@ -19,7 +18,6 @@ import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/gitea"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
@@ -37,12 +35,10 @@ var BlacklistedPaths = [][]byte{
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
// Initalize the logger.
logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
if err != nil {
return err
verbose := ctx.Bool("verbose")
if !verbose {
zerolog.SetGlobalLevel(zerolog.InfoLevel)
}
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
giteaRoot := strings.TrimSuffix(ctx.String("gitea-root"), "/")
giteaAPIToken := ctx.String("gitea-api-token")
@@ -85,15 +81,9 @@ func Serve(ctx *cli.Context) error {
// TODO: make this an MRU cache with a size limit
fileResponseCache := cache.NewKeyValueCache()
giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
if err != nil {
return fmt.Errorf("could not create new gitea client: %v", err)
}
// Create handler based on settings
handler := server.Handler(mainDomainSuffix, []byte(rawDomain),
giteaClient,
giteaRoot, rawInfoPage,
giteaRoot, rawInfoPage, giteaAPIToken,
BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache)
@@ -104,7 +94,7 @@ func Serve(ctx *cli.Context) error {
log.Info().Msgf("Listening on https://%s", listeningAddress)
listener, err := net.Listen("tcp", listeningAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
return fmt.Errorf("couldn't create listener: %s", err)
}
// TODO: make "key-database.pogreb" set via flag
@@ -115,8 +105,7 @@ func Serve(ctx *cli.Context) error {
defer certDB.Close() //nolint:errcheck // database has no close ... sync behave like it
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaClient,
dnsProvider,
giteaRoot, giteaAPIToken, dnsProvider,
acmeUseRateLimits,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB))
@@ -137,7 +126,6 @@ func Serve(ctx *cli.Context) error {
if enableHTTPServer {
go func() {
log.Info().Msg("Start HTTP server listening on :80")
err := httpServer.ListenAndServe("[::]:80")
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
@@ -146,7 +134,6 @@ func Serve(ctx *cli.Context) error {
}
// Start the web fastServer
log.Info().Msgf("Start listening on %s", listener.Addr())
err = fastServer.Serve(listener)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer")

117
go.mod
View File

@@ -1,128 +1,15 @@
module codeberg.org/codeberg/pages
go 1.18
go 1.16
require (
github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
github.com/akrylysov/pogreb v0.10.1
github.com/go-acme/lego/v4 v4.5.3
github.com/joho/godotenv v1.4.0
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
github.com/rs/zerolog v1.27.0
github.com/rs/zerolog v1.26.0
github.com/stretchr/testify v1.7.0
github.com/urfave/cli/v2 v2.3.0
github.com/valyala/fasthttp v1.31.0
github.com/valyala/fastjson v1.6.3
)
require (
cloud.google.com/go v0.54.0 // indirect
github.com/Azure/azure-sdk-for-go v32.4.0+incompatible // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.19 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.13 // indirect
github.com/Azure/go-autorest/autorest/azure/auth v0.5.8 // indirect
github.com/Azure/go-autorest/autorest/azure/cli v0.4.2 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/OpenDNS/vegadns2client v0.0.0-20180418235048-a3fa4a771d87 // indirect
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 // indirect
github.com/aliyun/alibaba-cloud-sdk-go v1.61.1183 // indirect
github.com/andybalholm/brotli v1.0.2 // indirect
github.com/aws/aws-sdk-go v1.39.0 // indirect
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect
github.com/cenkalti/backoff/v4 v4.1.1 // indirect
github.com/cloudflare/cloudflare-go v0.20.0 // indirect
github.com/cpu/goacmedns v0.1.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deepmap/oapi-codegen v1.6.1 // indirect
github.com/dimchansky/utfbom v1.1.1 // indirect
github.com/dnsimple/dnsimple-go v0.70.1 // indirect
github.com/exoscale/egoscale v0.67.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect
github.com/go-errors/errors v1.0.1 // indirect
github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48 // indirect
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/uuid v1.1.1 // indirect
github.com/googleapis/gax-go/v2 v2.0.5 // indirect
github.com/gophercloud/gophercloud v0.16.0 // indirect
github.com/gophercloud/utils v0.0.0-20210216074907-f6de111f2eae // indirect
github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df // indirect
github.com/infobloxopen/infoblox-go-client v1.1.1 // indirect
github.com/jarcoal/httpmock v1.0.6 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.7 // indirect
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect
github.com/klauspost/compress v1.13.4 // indirect
github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b // indirect
github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
github.com/labbsr0x/bindman-dns-webhook v1.0.2 // indirect
github.com/labbsr0x/goh v1.0.1 // indirect
github.com/linode/linodego v0.31.1 // indirect
github.com/liquidweb/go-lwApi v0.0.5 // indirect
github.com/liquidweb/liquidweb-cli v0.6.9 // indirect
github.com/liquidweb/liquidweb-go v1.6.3 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/miekg/dns v1.1.43 // indirect
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04 // indirect
github.com/nrdcg/auroradns v1.0.1 // indirect
github.com/nrdcg/desec v0.6.0 // indirect
github.com/nrdcg/dnspod-go v0.4.0 // indirect
github.com/nrdcg/freemyip v0.2.0 // indirect
github.com/nrdcg/goinwx v0.8.1 // indirect
github.com/nrdcg/namesilo v0.2.1 // indirect
github.com/nrdcg/porkbun v0.1.1 // indirect
github.com/oracle/oci-go-sdk v24.3.0+incompatible // indirect
github.com/ovh/go-ovh v1.1.0 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pquerna/otp v1.3.0 // indirect
github.com/russross/blackfriday/v2 v2.0.1 // indirect
github.com/sacloud/libsacloud v1.36.2 // indirect
github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210127161313-bd30bebeac4f // indirect
github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
github.com/sirupsen/logrus v1.4.2 // indirect
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 // indirect
github.com/softlayer/softlayer-go v1.0.3 // indirect
github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/transip/gotransip/v6 v6.6.1 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
github.com/vultr/govultr/v2 v2.7.1 // indirect
go.opencensus.io v0.22.3 // indirect
go.uber.org/ratelimit v0.0.0-20180316092928-c15da0234277 // indirect
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
google.golang.org/api v0.20.0 // indirect
google.golang.org/appengine v1.6.5 // indirect
google.golang.org/genproto v0.0.0-20200305110556-506484158171 // indirect
google.golang.org/grpc v1.27.1 // indirect
google.golang.org/protobuf v1.26.0 // indirect
gopkg.in/ini.v1 v1.62.0 // indirect
gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
)

22
go.sum
View File

@@ -95,7 +95,7 @@ github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkE
github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
github.com/cpu/goacmedns v0.1.1 h1:DM3H2NiN2oam7QljgGY5ygy4yDXhK5Z4JUnqaugs2C4=
github.com/cpu/goacmedns v0.1.1/go.mod h1:MuaouqEhPAHxsbqjgnck5zeghuwBP1dLnPoobeGqugQ=
@@ -266,8 +266,6 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
@@ -321,15 +319,12 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -431,8 +426,8 @@ github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad/go.mod h1:h0+DiDRe
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs=
github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U=
github.com/rs/zerolog v1.26.0 h1:ORM4ibhEZeTeQlCojCK2kPz1ogAY4bGs4tD+SaAdGaE=
github.com/rs/zerolog v1.26.0/go.mod h1:yBiM87lvSqX8h0Ww4sdzNSkVYZ8dL2xjZJG1lAuGZEo=
github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
@@ -491,6 +486,7 @@ github.com/transip/gotransip/v6 v6.6.1 h1:nsCU1ErZS5G0FeOpgGXc4FsWvBff9GPswSMggs
github.com/transip/gotransip/v6 v6.6.1/go.mod h1:pQZ36hWWRahCUXkFWlx9Hs711gLd8J4qdgLdRzmtY+g=
github.com/uber-go/atomic v1.3.2 h1:Azu9lPBWRNKzYXSIwRfgRuDuS0YKsK4NFhiQv98gkxo=
github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g=
github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU=
github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
@@ -513,6 +509,7 @@ github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQ
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
@@ -571,6 +568,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -669,9 +667,8 @@ golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6 h1:foEbQz/B0Oz6YIqu/69kfXPYeFQAuuMYFkjaqXzl5Wo=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -725,6 +722,7 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.7/go.mod h1:LGqMHiF4EqQNHR1JncWGqT5BVaXmza+X+BDGol+dOxo=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

View File

@@ -23,11 +23,11 @@
<body>
<i class="fa fa-search text-primary" style="font-size: 96px;"></i>
<h1 class="mb-0 text-primary">
Page not found!
This page was not found!
</h1>
<h5 class="text-center" style="max-width: 25em;">
Sorry, but this page couldn't be found or is inaccessible (%status).<br/>
We hope this isn't a problem on our end ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
Sorry, this page doesn't exist or is inaccessible for other reasons (%status).<br/>
We hope this is not our fault ;) - Make sure to check the <a href="https://docs.codeberg.org/codeberg-pages/troubleshooting/" target="_blank">troubleshooting section in the Docs</a>!
</h5>
<small class="text-muted">
<img src="https://design.codeberg.org/logo-kit/icon.svg" class="align-top">

View File

@@ -1,143 +0,0 @@
//go:build integration
// +build integration
package integration
import (
"bytes"
"crypto/tls"
"io"
"log"
"net/http"
"net/http/cookiejar"
"strings"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetRedirect(t *testing.T) {
log.Println("=== TestGetRedirect ===")
// test custom domain redirect
resp, err := getTestHTTPSClient().Get("https://calciumdibromid.localhost.mock.directory:4430")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusTemporaryRedirect, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
assert.EqualValues(t, 0, getSize(resp.Body))
}
func TestGetContent(t *testing.T) {
log.Println("=== TestGetContent ===")
// test get image
resp, err := getTestHTTPSClient().Get("https://magiclike.localhost.mock.directory:4430/images/827679288a.jpg")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "image/jpeg", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "124635", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 124635, getSize(resp.Body))
assert.Len(t, resp.Header.Get("ETag"), 42)
// specify branch
resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.True(t, getSize(resp.Body) > 1000)
assert.Len(t, resp.Header.Get("ETag"), 42)
// access branch name contains '/'
resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.True(t, getSize(resp.Body) > 100)
assert.Len(t, resp.Header.Get("ETag"), 42)
// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
}
func TestCustomDomain(t *testing.T) {
log.Println("=== TestCustomDomain ===")
resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 106, getSize(resp.Body))
}
func TestGetNotFound(t *testing.T) {
log.Println("=== TestGetNotFound ===")
// test custom not found pages
resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 37, getSize(resp.Body))
}
func TestFollowSymlink(t *testing.T) {
log.Printf("=== TestFollowSymlink ===\n")
resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
t.FailNow()
}
assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
body := getBytes(resp.Body)
assert.EqualValues(t, 4, len(body))
assert.EqualValues(t, "abc\n", string(body))
}
func TestLFSSupport(t *testing.T) {
log.Printf("=== TestLFSSupport ===\n")
resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
assert.NoError(t, err)
if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
t.FailNow()
}
body := strings.TrimSpace(string(getBytes(resp.Body)))
assert.EqualValues(t, 12, len(body))
assert.EqualValues(t, "actual value", body)
}
func getTestHTTPSClient() *http.Client {
cookieJar, _ := cookiejar.New(nil)
return &http.Client{
Jar: cookieJar,
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
},
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
}
func getBytes(stream io.Reader) []byte {
buf := new(bytes.Buffer)
_, _ = buf.ReadFrom(stream)
return buf.Bytes()
}
func getSize(stream io.Reader) int {
buf := new(bytes.Buffer)
_, _ = buf.ReadFrom(stream)
return buf.Len()
}

View File

@@ -1,62 +0,0 @@
//go:build integration
// +build integration
package integration
import (
"context"
"log"
"os"
"testing"
"time"
"codeberg.org/codeberg/pages/cmd"
"github.com/urfave/cli/v2"
)
func TestMain(m *testing.M) {
log.Println("=== TestMain: START Server ===")
serverCtx, serverCancel := context.WithCancel(context.Background())
if err := startServer(serverCtx); err != nil {
log.Fatalf("could not start server: %v", err)
}
defer func() {
serverCancel()
log.Println("=== TestMain: Server STOPED ===")
}()
time.Sleep(10 * time.Second)
os.Exit(m.Run())
}
func startServer(ctx context.Context) error {
args := []string{
"--verbose",
"--acme-accept-terms", "true",
}
setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
setEnvIfNotSet("PORT", "4430")
app := cli.NewApp()
app.Name = "pages-server"
app.Action = cmd.Serve
app.Flags = cmd.ServeFlags
go func() {
if err := app.RunContext(ctx, args); err != nil {
log.Fatalf("run server error: %v", err)
}
}()
return nil
}
func setEnvIfNotSet(key, value string) {
if _, set := os.LookupEnv(key); !set {
os.Setenv(key, value)
}
}

View File

@@ -4,7 +4,6 @@ import (
"fmt"
"os"
_ "github.com/joho/godotenv/autoload"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/cmd"

View File

@@ -12,6 +12,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"strconv"
"strings"
@@ -31,14 +32,12 @@ import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/database"
dnsutils "codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix []byte,
giteaClient *gitea.Client,
dnsProvider string,
giteaRoot, giteaAPIToken, dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB,
@@ -82,7 +81,7 @@ func TLSConfig(mainDomainSuffix []byte,
sni = string(sniBytes)
} else {
_, _ = targetRepo, targetBranch
_, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
_, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
if !valid {
sniBytes = mainDomainSuffix
sni = string(sniBytes)
@@ -194,7 +193,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
// parse certificate from database
res, err := certDB.Get(string(sni))
res, err := certDB.Get(sni)
if err != nil {
panic(err) // TODO: no panic
}
@@ -228,7 +227,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
res.CSR = nil // acme client doesn't like CSR to be set
tlsCertificate, err = obtainCert(acmeClient, []string{string(sni)}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Msgf("Couldn't renew certificate for %s: %v", string(sni), err)
log.Printf("Couldn't renew certificate for %s: %s", sni, err)
}
})()
}
@@ -271,10 +270,10 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
if acmeUseRateLimits {
acmeClientRequestLimit.Take()
}
log.Debug().Msgf("Renewing certificate for: %v", domains)
log.Printf("Renewing certificate for %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
log.Printf("Couldn't renew certificate for %v, trying to request a new one: %s", domains, err)
res = nil
}
}
@@ -289,7 +288,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Debug().Msgf("Re-requesting new certificate for %v", domains)
log.Printf("Requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
Domains: domains,
Bundle: true,
@@ -297,7 +296,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
})
}
if err != nil {
log.Error().Err(err).Msgf("Couldn't obtain again a certificate or %v", domains)
log.Printf("Couldn't obtain certificate for %v: %s", domains, err)
if renew != nil && renew.CertURL != "" {
tlsCertificate, err := tls.X509KeyPair(renew.Certificate, renew.PrivateKey)
if err == nil && tlsCertificate.Leaf.NotAfter.After(time.Now()) {
@@ -311,7 +310,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
}
return mockCert(domains[0], err.Error(), string(mainDomainSuffix), keyDatabase), err
}
log.Debug().Msgf("Obtained certificate for %v", domains)
log.Printf("Obtained certificate for %v", domains)
if err := keyDatabase.Put(name, res); err != nil {
return tls.Certificate{}, err
@@ -328,7 +327,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := os.ReadFile(configFile); err == nil {
if account, err := ioutil.ReadFile(configFile); err == nil {
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
@@ -344,7 +343,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
// TODO: should we fail hard instead?
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
@@ -365,13 +364,13 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
} else {
myAcmeAccount.Registration = reg
}
@@ -382,7 +381,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
log.Printf("[ERROR] Can't register ACME account, continuing with mock certs only: %s", err)
} else {
myAcmeAccount.Registration = reg
}
@@ -391,12 +390,12 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
log.Printf("[FAIL] Error during json.Marshal(myAcmeAccount), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
err = ioutil.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
log.Printf("[FAIL] Error during ioutil.WriteFile(\"acme-account.json\"), waiting for manual restart to avoid rate limits: %s", err)
select {}
}
}
@@ -407,45 +406,45 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
// getting main cert before ACME account so that we can fail here without hitting rate limits
mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
mainCertBytes, err := certDB.Get(mainDomainSuffix)
if err != nil {
return fmt.Errorf("cert database is not working")
}
acmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create HTTP-01 provider")
log.Printf("[ERROR] Can't create HTTP-01 provider: %s", err)
}
}
}
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
log.Printf("[ERROR] Can't create ACME client, continuing with mock certs only: %s", err)
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
log.Printf("[ERROR] Can't create TLS-ALPN-01 provider: %s", err)
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
log.Error().Err(err).Msg("Can't create DNS Challenge provider")
log.Printf("[ERROR] Can't create DNS Challenge provider: %s", err)
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Error().Err(err).Msg("Can't create DNS-01 provider")
log.Printf("[ERROR] Can't create DNS-01 provider: %s", err)
}
}
}
@@ -453,7 +452,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
if mainCertBytes == nil {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
log.Printf("[ERROR] Couldn't renew main domain certificate, continuing with mock certs only: %s", err)
}
}
@@ -479,9 +478,9 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
if err != nil || !tlsCertificates[0].NotAfter.After(now) {
err := certDB.Delete(string(key))
err := certDB.Delete(key)
if err != nil {
log.Error().Err(err).Msgf("Deleting expired certificate for %q failed", string(key))
log.Printf("[ERROR] Deleting expired certificate for %s failed: %s", string(key), err)
} else {
expiredCertCount++
}
@@ -489,22 +488,22 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
}
key, resBytes, err = keyDatabaseIterator.Next()
}
log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
log.Printf("[INFO] Removed %d expired certificates from the database", expiredCertCount)
// compact the database
msg, err := certDB.Compact()
result, err := certDB.Compact()
if err != nil {
log.Error().Err(err).Msg("Compacting key database failed")
log.Printf("[ERROR] Compacting key database failed: %s", err)
} else {
log.Debug().Msgf("Compacted key database: %s", msg)
log.Printf("[INFO] Compacted key database (%+v)", result)
}
// update main cert
res, err := certDB.Get(string(mainDomainSuffix))
res, err := certDB.Get(mainDomainSuffix)
if err != nil {
log.Error().Msgf("Couldn't get cert for domain %q", mainDomainSuffix)
log.Err(err).Msgf("could not get cert for domain '%s'", mainDomainSuffix)
} else if res == nil {
log.Error().Msgf("Couldn't renew certificate for main domain %q expected main domain cert to exist, but it's missing - seems like the database is corrupted", string(mainDomainSuffix))
log.Error().Msgf("Couldn't renew certificate for main domain: %s", "expected main domain cert to exist, but it's missing - seems like the database is corrupted")
} else {
tlsCertificates, err := certcrypto.ParsePEMBundle(res.Certificate)
@@ -513,7 +512,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
go (func() {
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + string(mainDomainSuffix), string(mainDomainSuffix[1:])}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
log.Printf("[ERROR] Couldn't renew certificate for main domain: %s", err)
}
})()
}

View File

@@ -1,17 +0,0 @@
package certificates
import (
"testing"
"codeberg.org/codeberg/pages/server/database"
"github.com/stretchr/testify/assert"
)
func TestMockCert(t *testing.T) {
db, err := database.NewTmpDB()
assert.NoError(t, err)
cert := mockCert("example.com", "some error msg", "codeberg.page", db)
if assert.NotEmpty(t, cert) {
assert.NotEmpty(t, cert.Certificate)
}
}

View File

@@ -8,8 +8,8 @@ import (
type CertDB interface {
Close() error
Put(name string, cert *certificate.Resource) error
Get(name string) (*certificate.Resource, error)
Delete(key string) error
Compact() (string, error)
Get(name []byte) (*certificate.Resource, error)
Delete(key []byte) error
Compact() (pogreb.CompactionResult, error)
Items() *pogreb.ItemIterator
}

View File

@@ -1,55 +0,0 @@
package database
import (
"fmt"
"time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/akrylysov/pogreb"
"github.com/go-acme/lego/v4/certificate"
)
var _ CertDB = tmpDB{}
type tmpDB struct {
intern *mcache.CacheDriver
ttl time.Duration
}
func (p tmpDB) Close() error {
_ = p.intern.Close()
return nil
}
func (p tmpDB) Put(name string, cert *certificate.Resource) error {
return p.intern.Set(name, cert, p.ttl)
}
func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name)
if !has {
return nil, fmt.Errorf("cert for '%s' not found", name)
}
return cert.(*certificate.Resource), nil
}
func (p tmpDB) Delete(key string) error {
p.intern.Remove(key)
return nil
}
func (p tmpDB) Compact() (string, error) {
p.intern.Truncate()
return "Truncate done", nil
}
func (p tmpDB) Items() *pogreb.ItemIterator {
panic("ItemIterator not implemented for tmpDB")
}
func NewTmpDB() (CertDB, error) {
return &tmpDB{
intern: mcache.New(),
ttl: time.Minute,
}, nil
}

View File

@@ -35,9 +35,9 @@ func (p aDB) Put(name string, cert *certificate.Resource) error {
return p.intern.Put([]byte(name), resGob.Bytes())
}
func (p aDB) Get(name string) (*certificate.Resource, error) {
func (p aDB) Get(name []byte) (*certificate.Resource, error) {
cert := &certificate.Resource{}
resBytes, err := p.intern.Get([]byte(name))
resBytes, err := p.intern.Get(name)
if err != nil {
return nil, err
}
@@ -50,16 +50,12 @@ func (p aDB) Get(name string) (*certificate.Resource, error) {
return cert, nil
}
func (p aDB) Delete(key string) error {
return p.intern.Delete([]byte(key))
func (p aDB) Delete(key []byte) error {
return p.intern.Delete(key)
}
func (p aDB) Compact() (string, error) {
result, err := p.intern.Compact()
if err != nil {
return "", err
}
return fmt.Sprintf("%+v", result), nil
func (p aDB) Compact() (pogreb.CompactionResult, error) {
return p.intern.Compact()
}
func (p aDB) Items() *pogreb.ItemIterator {
@@ -72,7 +68,7 @@ func (p aDB) sync() {
for {
err := p.intern.Sync()
if err != nil {
log.Error().Err(err).Msg("Syncing cert database failed")
log.Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():

View File

@@ -1,12 +0,0 @@
package gitea
type FileResponse struct {
Exists bool
ETag []byte
MimeType string
Body []byte
}
func (f FileResponse) IsEmpty() bool {
return len(f.Body) != 0
}

View File

@@ -1,142 +0,0 @@
package gitea
import (
"errors"
"fmt"
"net/url"
"strings"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
)
const (
giteaAPIRepos = "/api/v1/repos/"
giteaObjectTypeHeader = "X-Gitea-Object-Type"
)
var ErrorNotFound = errors.New("not found")
type Client struct {
giteaRoot string
giteaAPIToken string
fastClient *fasthttp.Client
infoTimeout time.Duration
contentTimeout time.Duration
followSymlinks bool
supportLFS bool
}
// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
func joinURL(baseURL string, paths ...string) string {
p := make([]string, 0, len(paths))
for i := range paths {
path := strings.TrimSpace(paths[i])
path = strings.Trim(path, "/")
if len(path) != 0 {
p = append(p, path)
}
}
return baseURL + "/" + strings.Join(p, "/")
}
func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) {
rootURL, err := url.Parse(giteaRoot)
giteaRoot = strings.Trim(rootURL.String(), "/")
return &Client{
giteaRoot: giteaRoot,
giteaAPIToken: giteaAPIToken,
infoTimeout: 5 * time.Second,
contentTimeout: 10 * time.Second,
fastClient: getFastHTTPClient(),
followSymlinks: followSymlinks,
supportLFS: supportLFS,
}, err
}
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
if err != nil {
return nil, err
}
return resp.Body(), nil
}
func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
var apiURL string
if client.supportLFS {
apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
} else {
apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
}
resp, err := client.do(client.contentTimeout, apiURL)
if err != nil {
return nil, err
}
if err != nil {
return nil, err
}
switch resp.StatusCode() {
case fasthttp.StatusOK:
objType := string(resp.Header.Peek(giteaObjectTypeHeader))
log.Trace().Msgf("server raw content object: %s", objType)
if client.followSymlinks && objType == "symlink" {
// TODO: limit to 1000 chars if we switched to std
linkDest := strings.TrimSpace(string(resp.Body()))
log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
}
return resp, nil
case fasthttp.StatusNotFound:
return nil, ErrorNotFound
default:
return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
}
}
func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
res, err := client.do(client.infoTimeout, url)
if err != nil {
return time.Time{}, err
}
if res.StatusCode() != fasthttp.StatusOK {
return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
}
func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
res, err := client.do(client.infoTimeout, url)
if err != nil {
return "", err
}
if res.StatusCode() != fasthttp.StatusOK {
return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
}
return fastjson.GetString(res.Body(), "default_branch"), nil
}
func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
req := fasthttp.AcquireRequest()
req.SetRequestURI(url)
req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
res := fasthttp.AcquireResponse()
err := client.fastClient.DoTimeout(req, res, timeout)
return res, err
}

View File

@@ -1,23 +0,0 @@
package gitea
import (
"net/url"
"testing"
"github.com/stretchr/testify/assert"
)
func TestJoinURL(t *testing.T) {
baseURL := ""
assert.EqualValues(t, "/", joinURL(baseURL))
assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
baseURL = "http://wwow.url.com"
assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
baseURL = "http://wow.url.com/subpath/2"
assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
}

View File

@@ -1,15 +0,0 @@
package gitea
import (
"time"
"github.com/valyala/fasthttp"
)
func getFastHTTPClient() *fasthttp.Client {
return &fasthttp.Client{
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
}

View File

@@ -4,30 +4,26 @@ import (
"bytes"
"strings"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
"codeberg.org/codeberg/pages/server/utils"
"codeberg.org/codeberg/pages/server/version"
)
// Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain []byte,
giteaClient *gitea.Client,
giteaRoot, rawInfoPage string,
giteaRoot, rawInfoPage, giteaAPIToken string,
blacklistedPaths, allowedCorsDomains [][]byte,
dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) func(ctx *fasthttp.RequestCtx) {
return func(ctx *fasthttp.RequestCtx) {
log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger()
log := log.With().Str("Handler", string(ctx.Request.Header.RequestURI())).Logger()
ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
ctx.Response.Header.Set("Server", "Codeberg Pages")
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -78,25 +74,21 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Prepare request information to Gitea
var targetOwner, targetRepo, targetBranch, targetPath string
targetOptions := &upstream.Options{
TryIndexPages: true,
ForbiddenMimeTypes: map[string]struct{}{},
TryIndexPages: true,
}
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL.
tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
tryBranch := func(repo, branch string, path []string, canonicalLink string) bool {
if repo == "" {
log.Warn().Msg("tryBranch: repo is empty")
return false
}
// Replace "~" to "/" so we can access branch that contains slash character
// Branch name cannot contain "~" so doing this is okay
branch = strings.ReplaceAll(branch, "~", "/")
// Check if the branch exists, otherwise treat it as a file path
branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
branchTimestampResult := upstream.GetBranchTimestamp(targetOwner, repo, branch, giteaRoot, giteaAPIToken, branchTimestampCache)
if branchTimestampResult == nil {
log.Warn().Msg("tryBranch: branch doesn't exist")
// branch doesn't exist
return false
}
@@ -116,20 +108,16 @@ func Handler(mainDomainSuffix, rawDomain []byte,
)
}
log.Debug().Msg("tryBranch: true")
return true
}
log.Debug().Msg("Preparing")
log.Debug().Msg("preparations")
if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
// Serve raw content from RawDomain
log.Debug().Msg("Serving raw domain")
log.Debug().Msg("raw domain")
targetOptions.TryIndexPages = false
if targetOptions.ForbiddenMimeTypes == nil {
targetOptions.ForbiddenMimeTypes = make(map[string]bool)
}
targetOptions.ForbiddenMimeTypes["text/html"] = true
targetOptions.ForbiddenMimeTypes["text/html"] = struct{}{}
targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
@@ -143,36 +131,36 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
log.Debug().Msg("Preparing raw domain, now trying with specified branch")
if tryBranch(log,
targetRepo, pathElements[2][1:], pathElements[3:],
log.Debug().Msg("raw domain preparations, now trying with specified branch")
if tryBranch(targetRepo, pathElements[2][1:], pathElements[3:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) {
log.Info().Msg("tryBranch, now trying upstream 1")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
log.Warn().Msg("Path missed a branch")
log.Debug().Msg("missing branch")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
log.Debug().Msg("Preparing raw domain, now trying with default branch")
tryBranch(log,
targetRepo, "", pathElements[2:],
log.Debug().Msg("raw domain preparations, now trying with default branch")
tryBranch(targetRepo, "", pathElements[2:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
)
log.Info().Msg("tryBranch, now trying upstream 2")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
} else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
// Serve pages from subdomains of MainDomainSuffix
log.Info().Msg("Serve pages from main domain suffix")
log.Debug().Msg("main domain suffix")
pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
@@ -194,17 +182,16 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return
}
log.Debug().Msg("Preparing main domain, now trying with specified repo & branch")
if tryBranch(log,
pathElements[0], pathElements[1][1:], pathElements[2:],
log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
if tryBranch(pathElements[0], pathElements[1][1:], pathElements[2:],
"/"+pathElements[0]+"/%p",
) {
log.Info().Msg("tryBranch, now trying upstream 3")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
log.Warn().Msg("tryBranch: upstream 3 failed")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
@@ -213,15 +200,14 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Check if the first directory is a branch for the "pages" repo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
log.Debug().Msg("Preparing main domain, now trying with specified branch")
if tryBranch(log,
"pages", pathElements[0][1:], pathElements[1:], "/%p") {
log.Info().Msg("tryBranch, now trying upstream 4")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
log.Debug().Msg("main domain preparations, now trying with specified branch")
if tryBranch("pages", pathElements[0][1:], pathElements[1:], "/%p") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
} else {
log.Warn().Msg("tryBranch: upstream 4 failed")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
}
return
@@ -231,11 +217,11 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// example.codeberg.page/myrepo/index.html
// example.codeberg.page/pages/... is not allowed here.
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(log,
pathElements[0], "pages", pathElements[1:], "") {
log.Info().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
if pathElements[0] != "pages" && tryBranch(pathElements[0], "pages", pathElements[1:], "") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
@@ -243,17 +229,16 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Try to use the "pages" repo on its default branch
// example.codeberg.page/index.html
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if tryBranch(log,
"pages", "", pathElements, "") {
log.Info().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
if tryBranch("pages", "", pathElements, "") {
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
// Couldn't find a valid repo/branch
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
} else {
@@ -275,12 +260,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
// Try to use the given repo on the given branch or the default branch
log.Debug().Msg("Preparing custom domain, now trying with details from DNS")
if tryBranch(log,
targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if tryBranch(targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
if !valid {
log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical")
html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
@@ -291,19 +274,18 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return
}
log.Warn().Msg("Custom domains, targetOwner from DNS is empty")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}
log.Info().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
log.Debug().Msg("tryBranch, now trying upstream")
tryUpstream(ctx, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken,
canonicalDomainCache, branchTimestampCache, fileResponseCache)
return
}
log.Warn().Msg("Couldn't handle request, none of the options succeed")
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
return
}

View File

@@ -8,16 +8,15 @@ import (
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
func TestHandlerPerformance(t *testing.T) {
giteaRoot := "https://codeberg.org"
giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
testHandler := Handler(
[]byte("codeberg.page"), []byte("raw.codeberg.org"),
giteaClient,
giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
[]byte("codeberg.page"),
[]byte("raw.codeberg.org"),
"https://codeberg.org",
"https://docs.codeberg.org/pages/raw-content/",
"",
[][]byte{[]byte("/.well-known/acme-challenge/")},
[][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
cache.NewKeyValueCache(),

View File

@@ -2,23 +2,15 @@ package server
import (
"bytes"
"fmt"
"net/http"
"time"
"github.com/rs/zerolog/log"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/utils"
)
type fasthttpLogger struct{}
func (fasthttpLogger) Printf(format string, args ...interface{}) {
log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
}
func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
// Enable compression by wrapping the handler with the compression function provided by FastHTTP
compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
@@ -29,7 +21,7 @@ func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
NoDefaultServerHeader: true,
NoDefaultDate: true,
ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
Logger: fasthttpLogger{},
Concurrency: 1024 * 32, // TODO: adjust bottlenecks for best performance with Gitea!
}
}

View File

@@ -8,22 +8,22 @@ import (
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
func tryUpstream(ctx *fasthttp.RequestCtx,
mainDomainSuffix, trimmedHost []byte,
targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath string,
targetOwner, targetRepo, targetBranch, targetPath,
giteaRoot, giteaAPIToken string,
canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
canonicalDomain, _ := upstream.CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), giteaRoot, giteaAPIToken, canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
canonicalPath := string(ctx.RequestURI())
if targetRepo != "pages" {
@@ -41,10 +41,9 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
targetOptions.TargetRepo = targetRepo
targetOptions.TargetBranch = targetBranch
targetOptions.TargetPath = targetPath
targetOptions.Host = string(trimmedHost)
// Try to request the file from the Gitea API
if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
if !targetOptions.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
}
}

View File

@@ -12,7 +12,6 @@ var branchExistenceCacheTimeout = 5 * time.Minute
// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
// on your available memory.
// TODO: move as option into cache interface
var fileCacheTimeout = 5 * time.Minute
// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
@@ -20,5 +19,3 @@ var fileCacheSizeLimit = 1024 * 1024
// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
var canonicalDomainCacheTimeout = 15 * time.Minute
const canonicalDomainConfig = ".domains"

View File

@@ -3,16 +3,15 @@ package upstream
import (
"strings"
"github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var (
domains []string
valid bool
)
func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
domains := []string{}
valid := false
if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
domains = cachedValue.([]string)
for _, domain := range domains {
@@ -22,9 +21,13 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
}
}
} else {
body, err := giteaClient.GiteaRawContent(targetOwner, targetRepo, targetBranch, canonicalDomainConfig)
if err == nil {
for _, domain := range strings.Split(string(body), "\n") {
req := fasthttp.AcquireRequest()
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
res := fasthttp.AcquireResponse()
err := client.Do(req, res)
if err == nil && res.StatusCode() == fasthttp.StatusOK {
for _, domain := range strings.Split(string(res.Body()), "\n") {
domain = strings.ToLower(domain)
domain = strings.TrimSpace(domain)
domain = strings.TrimPrefix(domain, "http://")

View File

@@ -1,15 +1,12 @@
package upstream
import (
"mime"
"path"
"strconv"
"strings"
"time"
"github.com/valyala/fasthttp"
"github.com/valyala/fastjson"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
)
type branchTimestamp struct {
@@ -19,66 +16,40 @@ type branchTimestamp struct {
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger()
func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
if result == nil {
log.Debug().Msg("branchTimestampCache found item, but result is empty")
return nil
}
log.Debug().Msg("branchTimestampCache found item, returning result")
return result.(*branchTimestamp)
}
result := &branchTimestamp{
Branch: branch,
}
if len(branch) == 0 {
result := &branchTimestamp{}
result.Branch = branch
if branch == "" {
// Get default branch
defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
if err != nil {
log.Err(err).Msg("Could't fetch default branch from repository")
_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
body := make([]byte, 0)
// TODO: use header for API key?
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
return nil
}
log.Debug().Msg("Succesfully fetched default branch from Gitea")
result.Branch = defaultBranch
result.Branch = fastjson.GetString(body, "default_branch")
}
timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
if err != nil {
log.Err(err).Msg("Could not get latest commit's timestamp from branch")
body := make([]byte, 0)
status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
if err != nil || status != 200 {
return nil
}
log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache")
result.Timestamp = timestamp
result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
return result
}
func (o *Options) getMimeTypeByExtension() string {
if o.ForbiddenMimeTypes == nil {
o.ForbiddenMimeTypes = make(map[string]bool)
}
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
return mimeType
}
func (o *Options) generateUri() string {
return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
}
func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
}
func (o *Options) timestamp() string {
return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
type fileResponse struct {
exists bool
mimeType string
body []byte
}

View File

@@ -2,8 +2,11 @@ package upstream
import (
"bytes"
"errors"
"fmt"
"io"
"mime"
"path"
"strconv"
"strings"
"time"
@@ -12,7 +15,6 @@ import (
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
// upstreamIndexPages lists pages that may be considered as index pages for directories.
@@ -20,11 +22,6 @@ var upstreamIndexPages = []string{
"index.html",
}
// upstreamNotFoundPages lists pages that may be considered as custom 404 Not Found pages.
var upstreamNotFoundPages = []string{
"404.html",
}
// Options provides various options for the upstream request.
type Options struct {
TargetOwner,
@@ -32,11 +29,8 @@ type Options struct {
TargetBranch,
TargetPath,
// Used for debugging purposes.
Host string
DefaultMimeType string
ForbiddenMimeTypes map[string]bool
DefaultMimeType string
ForbiddenMimeTypes map[string]struct{}
TryIndexPages bool
BranchTimestamp time.Time
// internal
@@ -44,13 +38,24 @@ type Options struct {
redirectIfExists string
}
var client = fasthttp.Client{
ReadTimeout: 10 * time.Second,
MaxConnDuration: 60 * time.Second,
MaxConnWaitTimeout: 1000 * time.Millisecond,
MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken string, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.ForbiddenMimeTypes == nil {
o.ForbiddenMimeTypes = map[string]struct{}{}
}
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
branch := GetBranchTimestamp(o.TargetOwner, o.TargetRepo, o.TargetBranch, giteaRoot, giteaAPIToken, branchTimestampCache)
if branch == nil {
html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
@@ -72,23 +77,27 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
return true
}
}
log.Debug().Msg("Preparing")
log.Debug().Msg("preparations")
// Make a GET request to the upstream URL
uri := o.generateUri()
uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
var req *fasthttp.Request
var res *fasthttp.Response
var cachedResponse gitea.FileResponse
var cachedResponse fileResponse
var err error
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
cachedResponse = cachedValue.(gitea.FileResponse)
if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + strconv.FormatInt(o.BranchTimestamp.Unix(), 10)); ok && len(cachedValue.(fileResponse).body) > 0 {
cachedResponse = cachedValue.(fileResponse)
} else {
res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
req = fasthttp.AcquireRequest()
req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
res = fasthttp.AcquireResponse()
res.SetBodyStream(&strings.Reader{}, -1)
err = client.Do(req, res)
}
log.Debug().Msg("Aquisting")
log.Debug().Msg("acquisition")
// Handle errors
if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
if (res == nil && !cachedResponse.exists) || (res != nil && res.StatusCode() == fasthttp.StatusNotFound) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
@@ -96,9 +105,9 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
return true
}
@@ -107,39 +116,24 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
if optionsForIndexPages.Upstream(ctx, giteaRoot, giteaAPIToken, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
return true
}
}
ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
if o.TryIndexPages {
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
optionsForNotFoundPages.TryIndexPages = false
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
}, fileCacheTimeout)
return true
}
}
}
if res != nil {
// Update cache if the request is fresh
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
Exists: false,
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), fileResponse{
exists: false,
}, fileCacheTimeout)
}
return false
}
if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
fmt.Printf("Couldn't fetch contents from \"%s\": %s (status code %d)\n", req.RequestURI(), err, res.StatusCode())
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
@@ -158,55 +152,50 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
return true
}
log.Debug().Msg("Handling error")
log.Debug().Msg("error handling")
// Set the MIME type
mimeType := o.getMimeTypeByExtension()
mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
if _, ok := o.ForbiddenMimeTypes[mimeTypeSplit[0]]; ok || mimeType == "" {
if o.DefaultMimeType != "" {
mimeType = o.DefaultMimeType
} else {
mimeType = "application/octet-stream"
}
}
ctx.Response.Header.SetContentType(mimeType)
// Set ETag
if cachedResponse.Exists {
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
} else if res != nil {
cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
}
if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
}
// Everything's okay so far
ctx.Response.SetStatusCode(fasthttp.StatusOK)
ctx.Response.Header.SetLastModified(o.BranchTimestamp)
log.Debug().Msg("Prepare response")
log.Debug().Msg("response preparations")
// Write the response body to the original request
var cacheBodyWriter bytes.Buffer
if res != nil {
if res.Header.ContentLength() > fileCacheSizeLimit {
// fasthttp else will set "Content-Length: 0"
ctx.Response.SetBodyStream(&strings.Reader{}, -1)
err = res.BodyWriteTo(ctx.Response.BodyWriter())
} else {
// TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
}
} else {
_, err = ctx.Write(cachedResponse.Body)
_, err = ctx.Write(cachedResponse.body)
}
if err != nil {
log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
fmt.Printf("Couldn't write body for \"%s\": %s\n", req.RequestURI(), err)
html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
return true
}
log.Debug().Msg("Sending response")
log.Debug().Msg("response")
if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
cachedResponse.Exists = true
cachedResponse.MimeType = mimeType
cachedResponse.Body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
if res != nil && ctx.Err() == nil {
cachedResponse.exists = true
cachedResponse.mimeType = mimeType
cachedResponse.body = cacheBodyWriter.Bytes()
_ = fileResponseCache.Set(uri+"?timestamp="+strconv.FormatInt(o.BranchTimestamp.Unix(), 10), cachedResponse, fileCacheTimeout)
}
return true

View File

@@ -1,3 +0,0 @@
package version
var Version string = "dev"