4 Commits
v5.0 ... v4.6.3

Author SHA1 Message Date
6543
5fe4613813 Use http.NoBody as per linter (#231)
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/231
Reviewed-by: Gusted <gusted@noreply.codeberg.org>
2023-08-27 11:14:51 +02:00
Moritz Marquardt
5a6f415428 Fix CI pipeline (replace "pipeline" with "steps") 2023-08-27 11:10:55 +02:00
Moritz Marquardt
b7bf745863 Security Fix: clean paths correctly to avoid circumvention of BlacklistedPaths 2023-08-27 10:13:50 +02:00
Crystal
ce241fa40a Fix certificate renewal (#209)
A database bug in xorm.go prevents the pages-server from saving a
renewed certificate for a domain that already has one in the database.

Co-authored-by: crystal <crystal@noreply.codeberg.org>
Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/209
Reviewed-by: 6543 <6543@obermui.de>
Co-authored-by: Crystal <crystal@noreply.codeberg.org>
Co-committed-by: Crystal <crystal@noreply.codeberg.org>
2023-03-20 23:59:34 +01:00
33 changed files with 673 additions and 974 deletions

1
.gitignore vendored
View File

@@ -7,4 +7,3 @@ build/
vendor/
pages
certs.sqlite
.bash_history

View File

@@ -65,6 +65,19 @@ steps:
- RAW_DOMAIN=raw.localhost.mock.directory
- PORT=4430
# TODO: remove in next version
integration-tests-legacy:
group: test
image: codeberg.org/6543/docker-images/golang_just
commands:
- just integration
environment:
- ACME_API=https://acme.mock.directory
- PAGES_DOMAIN=localhost.mock.directory
- RAW_DOMAIN=raw.localhost.mock.directory
- PORT=4430
- DB_TYPE=
release:
image: plugins/gitea-release
settings:

View File

@@ -1,45 +0,0 @@
# Features
## Custom domains
...
## Redirects
Redirects can be created with a `_redirects` file with the following format:
```
# Comment
from to [status]
```
* Lines starting with `#` are ignored
* `from` - the path to redirect from (Note: repository and branch names are removed from request URLs)
* `to` - the path or URL to redirect to
* `status` - status code to use when redirecting (default 301)
### Status codes
* `200` - returns content from specified path (no external URLs) without changing the URL (rewrite)
* `301` - Moved Permanently (Permanent redirect)
* `302` - Found (Temporary redirect)
### Examples
#### SPA (single-page application) rewrite
Redirects all paths to `/index.html` for single-page apps.
```
/* /index.html 200
```
#### Splats
Redirects every path under `/articles` to `/posts` while keeping the path.
```
/articles/* /posts/:splat 302
```
Example: `/articles/2022/10/12/post-1/` -> `/posts/2022/10/12/post-1/`

View File

@@ -9,8 +9,6 @@ dev:
export PAGES_DOMAIN=localhost.mock.directory
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
export HTTP_PORT=8880
export ENABLE_HTTP_SERVER=true
export LOG_LEVEL=trace
go run -tags '{{TAGS}}' .
@@ -29,7 +27,7 @@ fmt: tool-gofumpt
clean:
go clean ./...
rm -rf build/ integration/certs.sqlite integration/acme-account.json
rm -rf build/ integration/certs.sqlite integration/key-database.pogreb/ integration/acme-account.json
tool-golangci:
@hash golangci-lint> /dev/null 2>&1; if [ $? -ne 0 ]; then \
@@ -52,6 +50,3 @@ integration:
integration-run TEST:
go test -race -tags 'integration {{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/integration/...
docker:
docker run --rm -it --user $(id -u) -v $(pwd):/work --workdir /work -e HOME=/work codeberg.org/6543/docker-images/golang_just

View File

@@ -1,11 +1,5 @@
# Codeberg Pages
[![License: EUPL-1.2](https://img.shields.io/badge/License-EUPL--1.2-blue)](https://opensource.org/license/eupl-1-2/)
[![status-badge](https://ci.codeberg.org/api/badges/Codeberg/pages-server/status.svg)](https://ci.codeberg.org/Codeberg/pages-server)
<a href="https://matrix.to/#/#gitea-pages-server:matrix.org" title="Join the Matrix room at https://matrix.to/#/#gitea-pages-server:matrix.org">
<img src="https://img.shields.io/matrix/gitea-pages-server:matrix.org?label=matrix">
</a>
Gitea lacks the ability to host static pages from Git.
The Codeberg Pages Server addresses this lack by implementing a standalone service
that connects to Gitea via API.
@@ -14,9 +8,6 @@ It is suitable to be deployed by other Gitea instances, too, to offer static pag
**End user documentation** can mainly be found at the [Wiki](https://codeberg.org/Codeberg/pages-server/wiki/Overview)
and the [Codeberg Documentation](https://docs.codeberg.org/codeberg-pages/).
<a href="https://codeberg.org/Codeberg/pages-server"> <img src="https://codeberg.org/Codeberg/GetItOnCodeberg/raw/branch/main/get-it-on-blue-on-white.svg" alt="Get It On Codeberg" width="250"/> <a/>
## Quickstart
This is the new Codeberg Pages server, a solution for serving static pages from Gitea repositories.
@@ -38,10 +29,6 @@ record that points to your repo (just like the CNAME record):
Certificates are generated, updated and cleaned up automatically via Let's Encrypt through a TLS challenge.
## Chat for admins & devs
[matrix: #gitea-pages-server:matrix.org](https://matrix.to/#/#gitea-pages-server:matrix.org)
## Deployment
**Warning: Some Caveats Apply**
@@ -74,7 +61,7 @@ and especially have a look at [this section of the haproxy.cfg](https://codeberg
- `RAW_INFO_PAGE` (default: https://docs.codeberg.org/pages/raw-content/): info page for raw resources, shown if no resource is provided.
- `ACME_API` (default: https://acme-v02.api.letsencrypt.org/directory): set this to https://acme.mock.director to use invalid certificates without any verification (great for debugging).
ZeroSSL might be better in the future as it doesn't have rate limits and doesn't clash with the official Codeberg certificates (which are using Let's Encrypt), but I couldn't get it to work yet.
- `ACME_EMAIL` (default: `noreply@example.email`): Set the email sent to the ACME API server to receive, for example, renewal reminders.
- `ACME_EMAIL` (default: `noreply@example.email`): Set this to "true" to accept the Terms of Service of your ACME provider.
- `ACME_EAB_KID` & `ACME_EAB_HMAC` (default: don't use EAB): EAB credentials, for example for ZeroSSL.
- `ACME_ACCEPT_TERMS` (default: use self-signed certificate): Set this to "true" to accept the Terms of Service of your ACME provider.
- `ACME_USE_RATE_LIMITS` (default: true): Set this to false to disable rate limits, e.g. with ZeroSSL.
@@ -91,33 +78,29 @@ Since we are working nicely in a team, it might be hard at times to get started
(still check out the issues, we always aim to have some things to get you started).
If you have any questions, want to work on a feature or could imagine collaborating with us for some time,
feel free to ping us in an issue or in a general [Matrix chat room](#chat-for-admins--devs).
feel free to ping us in an issue or in a general Matrix chatgroup.
You can also contact the maintainer(s) of this project:
- [crapStone](https://codeberg.org/crapStone) [(Matrix)](https://matrix.to/#/@crapstone:obermui.de)
Previous maintainers:
You can also contact the maintainers of this project:
- [momar](https://codeberg.org/momar) [(Matrix)](https://matrix.to/#/@moritz:wuks.space)
- [6543](https://codeberg.org/6543) [(Matrix)](https://matrix.to/#/@marddl:obermui.de)
### First steps
The code of this repository is split in several modules.
The [Architecture is explained](https://codeberg.org/Codeberg/pages-server/wiki/Architecture) in the wiki.
The code of this repository is split in several modules.
While heavy refactoring work is currently undergo, you can easily understand the basic structure:
The `cmd` folder holds the data necessary for interacting with the service via the cli.
If you are considering to deploy the service yourself, make sure to check it out.
The heart of the software lives in the `server` folder and is split in several modules.
After scanning the code, you should quickly be able to understand their function and start hacking on them.
Again: Feel free to get in touch with us for any questions that might arise.
Thank you very much.
### Test Server
Make sure you have [golang](https://go.dev) v1.20 or newer and [just](https://just.systems/man/en/) installed.
run `just dev`
run `just dev`
now this pages should work:
- https://cb_pages_tests.localhost.mock.directory:4430/images/827679288a.jpg
- https://momar.localhost.mock.directory:4430/ci-testing/

View File

@@ -4,7 +4,11 @@ import (
"fmt"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/database"
)
var Certs = &cli.Command{
@@ -21,8 +25,63 @@ var Certs = &cli.Command{
Usage: "remove a certificate from the database",
Action: removeCert,
},
{
Name: "migrate",
Usage: "migrate from \"pogreb\" driver to dbms driver",
Action: migrateCerts,
},
},
Flags: CertStorageFlags,
Flags: append(CertStorageFlags, []cli.Flag{
&cli.BoolFlag{
Name: "verbose",
Usage: "print trace info",
EnvVars: []string{"VERBOSE"},
Value: false,
},
}...),
}
func migrateCerts(ctx *cli.Context) error {
dbType := ctx.String("db-type")
if dbType == "" {
dbType = "sqlite3"
}
dbConn := ctx.String("db-conn")
dbPogrebConn := ctx.String("db-pogreb")
verbose := ctx.Bool("verbose")
log.Level(zerolog.InfoLevel)
if verbose {
log.Level(zerolog.TraceLevel)
}
xormDB, err := database.NewXormDB(dbType, dbConn)
if err != nil {
return fmt.Errorf("could not connect to database: %w", err)
}
defer xormDB.Close()
pogrebDB, err := database.NewPogreb(dbPogrebConn)
if err != nil {
return fmt.Errorf("could not open database: %w", err)
}
defer pogrebDB.Close()
fmt.Printf("Start migration from \"%s\" to \"%s:%s\" ...\n", dbPogrebConn, dbType, dbConn)
certs, err := pogrebDB.Items(0, 0)
if err != nil {
return err
}
for _, cert := range certs {
if err := xormDB.Put(cert.Domain, cert.Raw()); err != nil {
return err
}
}
fmt.Println("... done")
return nil
}
func listCerts(ctx *cli.Context) error {

View File

@@ -6,15 +6,20 @@ import (
var (
CertStorageFlags = []cli.Flag{
&cli.StringFlag{
// TODO: remove in next version
// DEPRICATED
Name: "db-pogreb",
Value: "key-database.pogreb",
EnvVars: []string{"DB_POGREB"},
},
&cli.StringFlag{
Name: "db-type",
Usage: "Specify the database driver. Valid options are \"sqlite3\", \"mysql\" and \"postgres\". Read more at https://xorm.io",
Value: "sqlite3",
Value: "", // TODO: "sqlite3" in next version
EnvVars: []string{"DB_TYPE"},
},
&cli.StringFlag{
Name: "db-conn",
Usage: "Specify the database connection. For \"sqlite3\" it's the filepath. Read more at https://go.dev/doc/tutorial/database-access",
Value: "certs.sqlite",
EnvVars: []string{"DB_CONN"},
},
@@ -89,21 +94,15 @@ var (
EnvVars: []string{"HOST"},
Value: "[::]",
},
&cli.UintFlag{
&cli.StringFlag{
Name: "port",
Usage: "specifies the https port to listen to ssl requests",
EnvVars: []string{"PORT", "HTTPS_PORT"},
Value: 443,
},
&cli.UintFlag{
Name: "http-port",
Usage: "specifies the http port, you also have to enable http server via ENABLE_HTTP_SERVER=true",
EnvVars: []string{"HTTP_PORT"},
Value: 80,
Usage: "specifies port of listening address",
EnvVars: []string{"PORT"},
Value: "443",
},
&cli.BoolFlag{
Name: "enable-http-server",
Usage: "start a http server to redirect to https and respond to http acme challenges",
Name: "enable-http-server",
// TODO: desc
EnvVars: []string{"ENABLE_HTTP_SERVER"},
},
&cli.StringFlag{
@@ -112,13 +111,6 @@ var (
Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal",
EnvVars: []string{"LOG_LEVEL"},
},
// Default branches to fetch assets from
&cli.StringSliceFlag{
Name: "pages-branch",
Usage: "define a branch to fetch assets from",
EnvVars: []string{"PAGES_BRANCHES"},
Value: cli.NewStringSlice("pages"),
},
// ############################
// ### ACME Client Settings ###
@@ -140,30 +132,24 @@ var (
Value: true,
},
&cli.BoolFlag{
Name: "acme-accept-terms",
Usage: "To accept the ACME ToS",
Name: "acme-accept-terms",
// TODO: Usage
EnvVars: []string{"ACME_ACCEPT_TERMS"},
},
&cli.StringFlag{
Name: "acme-eab-kid",
Usage: "Register the current account to the ACME server with external binding.",
Name: "acme-eab-kid",
// TODO: Usage
EnvVars: []string{"ACME_EAB_KID"},
},
&cli.StringFlag{
Name: "acme-eab-hmac",
Usage: "Register the current account to the ACME server with external binding.",
Name: "acme-eab-hmac",
// TODO: Usage
EnvVars: []string{"ACME_EAB_HMAC"},
},
&cli.StringFlag{
Name: "dns-provider",
Usage: "Use DNS-Challenge for main domain. Read more at: https://go-acme.github.io/lego/dns/",
Name: "dns-provider",
// TODO: Usage
EnvVars: []string{"DNS_PROVIDER"},
},
&cli.StringFlag{
Name: "acme-account-config",
Usage: "json file of acme account",
Value: "acme-account.json",
EnvVars: []string{"ACME_ACCOUNT_CONFIG"},
},
}...)
)

View File

@@ -3,6 +3,7 @@ package cmd
import (
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"net/http"
@@ -14,6 +15,7 @@ import (
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/gitea"
@@ -45,15 +47,22 @@ func Serve(ctx *cli.Context) error {
giteaRoot := ctx.String("gitea-root")
giteaAPIToken := ctx.String("gitea-api-token")
rawDomain := ctx.String("raw-domain")
defaultBranches := ctx.StringSlice("pages-branch")
mainDomainSuffix := ctx.String("pages-domain")
rawInfoPage := ctx.String("raw-info-page")
listeningHost := ctx.String("host")
listeningSSLPort := ctx.Uint("port")
listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort)
listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
listeningAddress := fmt.Sprintf("%s:%s", ctx.String("host"), ctx.String("port"))
enableHTTPServer := ctx.Bool("enable-http-server")
acmeAPI := ctx.String("acme-api-endpoint")
acmeMail := ctx.String("acme-email")
acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
acmeAcceptTerms := ctx.Bool("acme-accept-terms")
acmeEabKID := ctx.String("acme-eab-kid")
acmeEabHmac := ctx.String("acme-eab-hmac")
dnsProvider := ctx.String("dns-provider")
if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
return errors.New("you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory")
}
allowedCorsDomains := AllowedCorsDomains
if rawDomain != "" {
allowedCorsDomains = append(allowedCorsDomains, rawDomain)
@@ -64,10 +73,6 @@ func Serve(ctx *cli.Context) error {
mainDomainSuffix = "." + mainDomainSuffix
}
if len(defaultBranches) == 0 {
return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
}
// Init ssl cert database
certDB, closeFn, err := openCertDB(ctx)
if err != nil {
@@ -81,8 +86,6 @@ func Serve(ctx *cli.Context) error {
canonicalDomainCache := cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
dnsLookupCache := cache.NewKeyValueCache()
// redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewKeyValueCache()
// clientResponseCache stores responses from the Gitea server
clientResponseCache := cache.NewKeyValueCache()
@@ -91,60 +94,56 @@ func Serve(ctx *cli.Context) error {
return fmt.Errorf("could not create new gitea client: %v", err)
}
acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache)
if err != nil {
return err
}
// Create handler based on settings
httpsHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
rawInfoPage,
BlacklistedPaths, allowedCorsDomains,
dnsLookupCache, canonicalDomainCache)
if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil {
return err
}
httpHandler := server.SetupHTTPACMEChallengeServer(challengeCache)
// Create listener for SSL connections
log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
listener, err := net.Listen("tcp", listeningSSLAddress)
// Setup listener and TLS
log.Info().Msgf("Listening on https://%s", listeningAddress)
listener, err := net.Listen("tcp", listeningAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
}
// Setup listener for SSL connections
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaClient,
acmeClient,
defaultBranches[0],
dnsProvider,
acmeUseRateLimits,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB))
acmeConfig, err := certificates.SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
if err != nil {
return err
}
if err := certificates.SetupCertificates(mainDomainSuffix, dnsProvider, acmeConfig, acmeUseRateLimits, enableHTTPServer, challengeCache, certDB); err != nil {
return err
}
interval := 12 * time.Hour
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
defer cancelCertMaintain()
go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
go certificates.MaintainCertDB(certMaintainCtx, interval, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB)
if enableHTTPServer {
// Create handler for http->https redirect and http acme challenges
httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort)
// Create listener for http and start listening
go func() {
log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
log.Info().Msg("Start HTTP server listening on :80")
err := http.ListenAndServe("[::]:80", httpHandler)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
}
}()
}
// Create ssl handler based on settings
sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
rawInfoPage,
BlacklistedPaths, allowedCorsDomains,
defaultBranches,
dnsLookupCache, canonicalDomainCache, redirectsCache)
// Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
if err := http.Serve(listener, sslHandler); err != nil {
// Start the web fastServer
log.Info().Msgf("Start listening on %s", listener.Addr())
if err := http.Serve(listener, httpsHandler); err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer")
}

View File

@@ -1,23 +1,38 @@
package cmd
import (
"errors"
"fmt"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
)
var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return nil, nil, fmt.Errorf("could not connect to database: %w", err)
if ctx.String("db-type") != "" {
log.Trace().Msg("use xorm mode")
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return nil, nil, fmt.Errorf("could not connect to database: %w", err)
}
} else {
// TODO: remove in next version
fmt.Println(`
######################
## W A R N I N G !!! #
######################
You use "pogreb" witch is deprecated and will be removed in the next version.
Please switch to sqlite, mysql or postgres !!!
The simplest way is, to use './pages certs migrate' and set environment var DB_TYPE to 'sqlite' on next start.`)
log.Error().Msg("depricated \"pogreb\" used\n")
certDB, err = database.NewPogreb(ctx.String("db-pogreb"))
if err != nil {
return nil, nil, fmt.Errorf("could not create database: %w", err)
}
}
closeFn = func() {
@@ -28,37 +43,3 @@ func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err e
return certDB, closeFn, nil
}
func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) {
acmeAPI := ctx.String("acme-api-endpoint")
acmeMail := ctx.String("acme-email")
acmeEabHmac := ctx.String("acme-eab-hmac")
acmeEabKID := ctx.String("acme-eab-kid")
acmeAcceptTerms := ctx.Bool("acme-accept-terms")
dnsProvider := ctx.String("dns-provider")
acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
acmeAccountConf := ctx.String("acme-account-config")
// check config
if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
}
if acmeEabHmac != "" && acmeEabKID == "" {
return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
} else if acmeEabHmac == "" && acmeEabKID != "" {
return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
}
return certificates.NewAcmeClient(
acmeAccountConf,
acmeAPI,
acmeMail,
acmeEabHmac,
acmeEabKID,
dnsProvider,
acmeAcceptTerms,
enableHTTPServer,
acmeUseRateLimits,
challengeCache,
)
}

6
go.mod
View File

@@ -3,8 +3,9 @@ module codeberg.org/codeberg/pages
go 1.20
require (
code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f
code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa
github.com/OrlovEvgeny/go-mcache v0.0.0-20200121124330-1a8195b34f3a
github.com/akrylysov/pogreb v0.10.1
github.com/go-acme/lego/v4 v4.5.3
github.com/go-sql-driver/mysql v1.6.0
github.com/joho/godotenv v1.4.0
@@ -14,7 +15,6 @@ require (
github.com/rs/zerolog v1.27.0
github.com/stretchr/testify v1.7.0
github.com/urfave/cli/v2 v2.3.0
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb
xorm.io/xorm v1.3.2
)
@@ -118,7 +118,7 @@ require (
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d // indirect
golang.org/x/sys v0.1.0 // indirect
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect
google.golang.org/api v0.20.0 // indirect

19
go.sum
View File

@@ -22,8 +22,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA
cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk=
code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f h1:nMmwDgUIAWj9XQjzHz5unC3ZMfhhwHRk6rnuwLzdu1o=
code.gitea.io/sdk/gitea v0.16.1-0.20231115014337-e23e8aa3004f/go.mod h1:ndkDk99BnfiUCCYEUhpNzi0lpmApXlwRFqClBlOlEBg=
code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa h1:OVwgYrY6vr6gWZvgnmevFhtL0GVA4HKaFOhD+joPoNk=
code.gitea.io/sdk/gitea v0.15.1-0.20220729105105-cc14c63cccfa/go.mod h1:aRmrQC3CAHdJAU1LQt0C9zqzqI8tUB/5oQtNE746aYE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a h1:lSA0F4e9A2NcQSqGqTOXqu2aRi/XEQxDCBwM8yJtE6s=
gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0pAQhH8yz+DNjUbjppKQzKFAn28TMYPB6IU=
@@ -71,6 +71,8 @@ github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/
github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1 h1:bLzehmpyCwQiqCE1Qe9Ny6fbFqs7hPlmo9vKv2orUxs=
github.com/akamai/AkamaiOPEN-edgegrid-golang v1.1.1/go.mod h1:kX6YddBkXqqywAe8c9LyvgTCyFuZCTMF4cRPQhc3Fy8=
github.com/akrylysov/pogreb v0.10.1 h1:FqlR8VR7uCbJdfUob916tPM+idpKgeESDXOA1K0DK4w=
github.com/akrylysov/pogreb v0.10.1/go.mod h1:pNs6QmpQ1UlTJKDezuRWmaqkgUE2TuU0YTWyqJZ7+lI=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -249,8 +251,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-github/v32 v32.1.0/go.mod h1:rIEpZD9CTDQwDK9GDrtMTycQNA4JU3qBsCizh3q2WCI=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
@@ -768,8 +770,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w=
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -789,8 +789,8 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -899,8 +899,8 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20210902050250-f475640dd07b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10 h1:WIoqL4EROvwiPdUtaip4VcDdpZ4kha7wBWZrbVKCIZg=
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -962,13 +962,14 @@ golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapK
golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw=
golang.org/x/tools v0.0.0-20200410194907-79a7a3126eef/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20201124115921-2c860bdd6e78/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e h1:4nW4NLDYnU28ojHaHO8OVxFHk/aQ33U01a9cjED+pzE=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=

View File

@@ -64,7 +64,7 @@ func TestGetContent(t *testing.T) {
assert.True(t, getSize(resp.Body) > 100)
assert.Len(t, resp.Header.Get("ETag"), 44)
// TODO: test get of non cacheable content (content size > fileCacheSizeLimit)
// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
}
func TestCustomDomain(t *testing.T) {
@@ -109,34 +109,6 @@ func TestCustomDomainRedirects(t *testing.T) {
assert.EqualValues(t, "https://mock-pages.codeberg-test.org/README.md", resp.Header.Get("Location"))
}
func TestRawCustomDomain(t *testing.T) {
log.Println("=== TestRawCustomDomain ===")
// test raw domain response for custom domain branch
resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/example") // need cb_pages_tests fork
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "76", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 76, getSize(resp.Body))
}
func TestRawIndex(t *testing.T) {
log.Println("=== TestRawIndex ===")
// test raw domain response for index.html
resp, err := getTestHTTPSClient().Get("https://raw.localhost.mock.directory:4430/cb_pages_tests/raw-test/@branch-test/index.html") // need cb_pages_tests fork
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/plain; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "597", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 597, getSize(resp.Body))
}
func TestGetNotFound(t *testing.T) {
log.Println("=== TestGetNotFound ===")
// test custom not found pages
@@ -151,50 +123,9 @@ func TestGetNotFound(t *testing.T) {
assert.EqualValues(t, 37, getSize(resp.Body))
}
func TestRedirect(t *testing.T) {
log.Println("=== TestRedirect ===")
// test redirects
resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/redirect")
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
assert.EqualValues(t, "https://example.com/", resp.Header.Get("Location"))
}
func TestSPARedirect(t *testing.T) {
log.Println("=== TestSPARedirect ===")
// test SPA redirects
url := "https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/app/aqdjw"
resp, err := getTestHTTPSClient().Get(url)
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, url, resp.Request.URL.String())
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "258", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 258, getSize(resp.Body))
}
func TestSplatRedirect(t *testing.T) {
log.Println("=== TestSplatRedirect ===")
// test splat redirects
resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/some_redirects/articles/qfopefe")
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
assert.EqualValues(t, "/posts/qfopefe", resp.Header.Get("Location"))
}
func TestFollowSymlink(t *testing.T) {
log.Printf("=== TestFollowSymlink ===\n")
// file symlink
resp, err := getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
@@ -206,16 +137,6 @@ func TestFollowSymlink(t *testing.T) {
body := getBytes(resp.Body)
assert.EqualValues(t, 4, len(body))
assert.EqualValues(t, "abc\n", string(body))
// relative file links (../index.html file in this case)
resp, err = getTestHTTPSClient().Get("https://cb_pages_tests.localhost.mock.directory:4430/tests_for_pages-server/@main/dir_aim/some/")
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "an index\n", string(getBytes(resp.Body)))
}
func TestLFSSupport(t *testing.T) {
@@ -244,18 +165,6 @@ func TestGetOptions(t *testing.T) {
assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow"))
}
func TestHttpRedirect(t *testing.T) {
log.Println("=== TestHttpRedirect ===")
resp, err := getTestHTTPSClient().Get("http://mock-pages.codeberg-test.org:8880/README.md")
assert.NoError(t, err)
if !assert.NotNil(t, resp) {
t.FailNow()
}
assert.EqualValues(t, http.StatusMovedPermanently, resp.StatusCode)
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "https://mock-pages.codeberg-test.org:4430/README.md", resp.Header.Get("Location"))
}
func getTestHTTPSClient() *http.Client {
cookieJar, _ := cookiejar.New(nil)
return &http.Client{

View File

@@ -23,7 +23,7 @@ func TestMain(m *testing.M) {
}
defer func() {
serverCancel()
log.Println("=== TestMain: Server STOPPED ===")
log.Println("=== TestMain: Server STOPED ===")
}()
time.Sleep(10 * time.Second)
@@ -39,10 +39,7 @@ func startServer(ctx context.Context) error {
setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
setEnvIfNotSet("PAGES_BRANCHES", "pages,main,master")
setEnvIfNotSet("PORT", "4430")
setEnvIfNotSet("HTTP_PORT", "8880")
setEnvIfNotSet("ENABLE_HTTP_SERVER", "true")
setEnvIfNotSet("DB_TYPE", "sqlite3")
app := cli.NewApp()

View File

@@ -1,95 +0,0 @@
package certificates
import (
"fmt"
"sync"
"time"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/providers/dns"
"github.com/reugn/equalizer"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
)
type AcmeClient struct {
legoClient *lego.Client
dnsChallengerLegoClient *lego.Client
obtainLocks sync.Map
acmeUseRateLimits bool
// limiter
acmeClientOrderLimit *equalizer.TokenBucket
acmeClientRequestLimit *equalizer.TokenBucket
acmeClientFailLimit *equalizer.TokenBucket
acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket
}
func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) {
acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
if err != nil {
return nil, err
}
acmeClient, err := lego.NewClient(acmeConfig)
if err != nil {
log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create HTTP-01 provider")
}
}
}
mainDomainAcmeClient, err := lego.NewClient(acmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
} else {
// use DNS-Challenge https://go-acme.github.io/lego/dns/
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err)
}
if err := mainDomainAcmeClient.Challenge.SetDNS01Provider(provider); err != nil {
return nil, fmt.Errorf("can not create DNS-01 provider: %w", err)
}
}
}
return &AcmeClient{
legoClient: acmeClient,
dnsChallengerLegoClient: mainDomainAcmeClient,
acmeUseRateLimits: acmeUseRateLimits,
obtainLocks: sync.Map{},
// limiter
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
acmeClientOrderLimit: equalizer.NewTokenBucket(25, 15*time.Minute),
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
acmeClientRequestLimit: equalizer.NewTokenBucket(5, 1*time.Second),
// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
acmeClientFailLimit: equalizer.NewTokenBucket(5, 1*time.Hour),
// checkUserLimit() use this to rate also per user
acmeClientCertificateLimitPerUser: map[string]*equalizer.TokenBucket{},
}, nil
}

View File

@@ -1,102 +0,0 @@
package certificates
import (
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"encoding/json"
"fmt"
"os"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/registration"
"github.com/rs/zerolog/log"
)
const challengePath = "/.well-known/acme-challenge/"
func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := os.ReadFile(configFile); err == nil {
log.Info().Msgf("found existing acme account config file '%s'", configFile)
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
return nil, err
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
// Validate Config
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate")
return nil, fmt.Errorf("acme config validation failed: %w", err)
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
return nil, err
}
log.Info().Msgf("no existing acme account config found, try to create a new one")
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, err
}
myAcmeAccount = AcmeAccount{
Email: acmeMail,
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
TermsOfServiceAgreed: acmeAcceptTerms,
Kid: acmeEabKID,
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
}
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
log.Info().Msgf("new acme account created. write to config file '%s'", configFile)
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}
}
}
}
return myAcmeConfig, nil
}

View File

@@ -1,83 +0,0 @@
package certificates
import (
"fmt"
"net/http"
"net/url"
"strings"
"time"
"github.com/go-acme/lego/v4/challenge"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
)
type AcmeTLSChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeTLSChallengeProvider match Provider interface
var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain)
return nil
}
type AcmeHTTPChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeHTTPChallengeProvider match Provider interface
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
}
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc {
// handle custom-ssl-ports to be added on https redirects
portPart := ""
if sslPort != 443 {
portPart = fmt.Sprintf(":%d", sslPort)
}
return func(w http.ResponseWriter, req *http.Request) {
ctx := context.New(w, req)
domain := ctx.TrimHostPort()
// it's an acme request
if strings.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(domain + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
if !ok || challenge == nil {
log.Info().Msgf("HTTP-ACME challenge for '%s' failed: token not found", domain)
ctx.String("no challenge for this token", http.StatusNotFound)
}
log.Info().Msgf("HTTP-ACME challenge for '%s' succeeded", domain)
ctx.String(challenge.(string))
return
}
// it's a normal http request that needs to be redirected
u, err := url.Parse(fmt.Sprintf("https://%s%s%s", domain, portPart, ctx.Path()))
if err != nil {
log.Error().Err(err).Msg("could not craft http to https redirect")
ctx.String("", http.StatusInternalServerError)
}
newURL := u.String()
log.Debug().Msgf("redirect http to https: %s", newURL)
ctx.Redirect(newURL, http.StatusMovedPermanently)
}
}

View File

@@ -2,18 +2,27 @@ package certificates
import (
"context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/tls"
"crypto/x509"
"encoding/json"
"errors"
"fmt"
"os"
"strconv"
"strings"
"sync"
"time"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/certificate"
"github.com/go-acme/lego/v4/challenge"
"github.com/go-acme/lego/v4/challenge/tlsalpn01"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/providers/dns"
"github.com/go-acme/lego/v4/registration"
"github.com/reugn/equalizer"
"github.com/rs/zerolog/log"
@@ -24,37 +33,33 @@ import (
"codeberg.org/codeberg/pages/server/upstream"
)
var ErrUserRateLimitExceeded = errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client,
acmeClient *AcmeClient,
firstDefaultBranch string,
dnsProvider string,
acmeUseRateLimits bool,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
certDB database.CertDB,
) *tls.Config {
return &tls.Config{
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
domain := strings.ToLower(strings.TrimSpace(info.ServerName))
if len(domain) < 1 {
return nil, errors.New("missing domain info via SNI (RFC 4366, Section 3.1)")
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
if len(sni) < 1 {
return nil, errors.New("missing sni")
}
// https request init is actually a acme challenge
if info.SupportedProtos != nil {
for _, proto := range info.SupportedProtos {
if proto != tlsalpn01.ACMETLS1Protocol {
continue
}
log.Info().Msgf("Detect ACME-TLS1 challenge for '%s'", domain)
challenge, ok := challengeCache.Get(domain)
challenge, ok := challengeCache.Get(sni)
if !ok {
return nil, errors.New("no challenge for this domain")
}
cert, err := tlsalpn01.ChallengeCert(domain, challenge.(string))
cert, err := tlsalpn01.ChallengeCert(sni, challenge.(string))
if err != nil {
return nil, err
}
@@ -64,22 +69,22 @@ func TLSConfig(mainDomainSuffix string,
targetOwner := ""
mayObtainCert := true
if strings.HasSuffix(domain, mainDomainSuffix) || strings.EqualFold(domain, mainDomainSuffix[1:]) {
if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
// deliver default certificate for the main domain (*.codeberg.page)
domain = mainDomainSuffix
sni = mainDomainSuffix
} else {
var targetRepo, targetBranch string
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs
domain = mainDomainSuffix
sni = mainDomainSuffix
} else {
targetOpt := &upstream.Options{
TargetOwner: targetOwner,
TargetRepo: targetRepo,
TargetBranch: targetBranch,
}
_, valid := targetOpt.CheckCanonicalDomain(giteaClient, domain, mainDomainSuffix, canonicalDomainCache)
_, valid := targetOpt.CheckCanonicalDomain(giteaClient, sni, mainDomainSuffix, canonicalDomainCache)
if !valid {
// We shouldn't obtain a certificate when we cannot check if the
// repository has specified this domain in the `.domains` file.
@@ -88,34 +93,30 @@ func TLSConfig(mainDomainSuffix string,
}
}
if tlsCertificate, ok := keyCache.Get(domain); ok {
if tlsCertificate, ok := keyCache.Get(sni); ok {
// we can use an existing certificate object
return tlsCertificate.(*tls.Certificate), nil
}
var tlsCertificate *tls.Certificate
var err error
if tlsCertificate, err = acmeClient.retrieveCertFromDB(domain, mainDomainSuffix, false, certDB); err != nil {
if !errors.Is(err, database.ErrNotFound) {
return nil, err
}
// we could not find a cert in db, request a new certificate
// first check if we are allowed to obtain a cert for this domain
if strings.EqualFold(domain, mainDomainSuffix) {
if tlsCertificate, err = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); err != nil {
// request a new certificate
if strings.EqualFold(sni, mainDomainSuffix) {
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
}
if !mayObtainCert {
return nil, fmt.Errorf("won't request certificate for %q", domain)
return nil, fmt.Errorf("won't request certificate for %q", sni)
}
tlsCertificate, err = acmeClient.obtainCert(acmeClient.legoClient, []string{domain}, nil, targetOwner, false, mainDomainSuffix, certDB)
tlsCertificate, err = obtainCert(acmeClient, []string{sni}, nil, targetOwner, dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
return nil, err
}
}
if err := keyCache.Set(domain, tlsCertificate, 15*time.Minute); err != nil {
if err := keyCache.Set(sni, tlsCertificate, 15*time.Minute); err != nil {
return nil, err
}
return tlsCertificate, nil
@@ -140,20 +141,67 @@ func TLSConfig(mainDomainSuffix string,
}
}
func (c *AcmeClient) checkUserLimit(user string) error {
userLimit, ok := c.acmeClientCertificateLimitPerUser[user]
func checkUserLimit(user string) error {
userLimit, ok := acmeClientCertificateLimitPerUser[user]
if !ok {
// Each user can only add 10 new domains per day.
// Each Codeberg user can only add 10 new domains per day.
userLimit = equalizer.NewTokenBucket(10, time.Hour*24)
c.acmeClientCertificateLimitPerUser[user] = userLimit
acmeClientCertificateLimitPerUser[user] = userLimit
}
if !userLimit.Ask() {
return fmt.Errorf("user '%s' error: %w", user, ErrUserRateLimitExceeded)
return errors.New("rate limit exceeded: 10 certificates per user per 24 hours")
}
return nil
}
func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProvider bool, certDB database.CertDB) (*tls.Certificate, error) {
var (
acmeClient, mainDomainAcmeClient *lego.Client
acmeClientCertificateLimitPerUser = map[string]*equalizer.TokenBucket{}
)
// rate limit is 300 / 3 hours, we want 200 / 2 hours but to refill more often, so that's 25 new domains every 15 minutes
// TODO: when this is used a lot, we probably have to think of a somewhat better solution?
var acmeClientOrderLimit = equalizer.NewTokenBucket(25, 15*time.Minute)
// rate limit is 20 / second, we want 5 / second (especially as one cert takes at least two requests)
var acmeClientRequestLimit = equalizer.NewTokenBucket(5, 1*time.Second)
// rate limit is 5 / hour https://letsencrypt.org/docs/failed-validation-limit/
var acmeClientFailLimit = equalizer.NewTokenBucket(5, 1*time.Hour)
type AcmeTLSChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeTLSChallengeProvider match Provider interface
var _ challenge.Provider = AcmeTLSChallengeProvider{}
func (a AcmeTLSChallengeProvider) Present(domain, _, keyAuth string) error {
return a.challengeCache.Set(domain, keyAuth, 1*time.Hour)
}
func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
a.challengeCache.Remove(domain)
return nil
}
type AcmeHTTPChallengeProvider struct {
challengeCache cache.SetGetKey
}
// make sure AcmeHTTPChallengeProvider match Provider interface
var _ challenge.Provider = AcmeHTTPChallengeProvider{}
func (a AcmeHTTPChallengeProvider) Present(domain, token, keyAuth string) error {
return a.challengeCache.Set(domain+"/"+token, keyAuth, 1*time.Hour)
}
func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
a.challengeCache.Remove(domain + "/" + token)
return nil
}
func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (*tls.Certificate, error) {
// parse certificate from database
res, err := certDB.Get(sni)
if err != nil {
@@ -171,7 +219,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
if !strings.EqualFold(sni, mainDomainSuffix) {
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
if err != nil {
return nil, fmt.Errorf("error parsing leaf tlsCert: %w", err)
return nil, fmt.Errorf("error parsin leaf tlsCert: %w", err)
}
// renew certificates 7 days before they expire
@@ -187,7 +235,7 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
// TODO: make a queue ?
go (func() {
res.CSR = nil // acme client doesn't like CSR to be set
if _, err := c.obtainCert(c.legoClient, []string{sni}, res, "", useDnsProvider, mainDomainSuffix, certDB); err != nil {
if _, err := obtainCert(acmeClient, []string{sni}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB); err != nil {
log.Error().Msgf("Couldn't renew certificate for %s: %v", sni, err)
}
})()
@@ -197,26 +245,28 @@ func (c *AcmeClient) retrieveCertFromDB(sni, mainDomainSuffix string, useDnsProv
return &tlsCertificate, nil
}
func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user string, useDnsProvider bool, mainDomainSuffix string, keyDatabase database.CertDB) (*tls.Certificate, error) {
var obtainLocks = sync.Map{}
func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (*tls.Certificate, error) {
name := strings.TrimPrefix(domains[0], "*")
if useDnsProvider && len(domains[0]) > 0 && domains[0][0] == '*' {
if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
domains = domains[1:]
}
// lock to avoid simultaneous requests
_, working := c.obtainLocks.LoadOrStore(name, struct{}{})
_, working := obtainLocks.LoadOrStore(name, struct{}{})
if working {
for working {
time.Sleep(100 * time.Millisecond)
_, working = c.obtainLocks.Load(name)
_, working = obtainLocks.Load(name)
}
cert, err := c.retrieveCertFromDB(name, mainDomainSuffix, useDnsProvider, keyDatabase)
cert, err := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
if err != nil {
return nil, fmt.Errorf("certificate failed in synchronous request: %w", err)
}
return cert, nil
}
defer c.obtainLocks.Delete(name)
defer obtainLocks.Delete(name)
if acmeClient == nil {
return mockCert(domains[0], "ACME client uninitialized. This is a server error, please report!", mainDomainSuffix, keyDatabase)
@@ -226,29 +276,29 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew
var res *certificate.Resource
var err error
if renew != nil && renew.CertURL != "" {
if c.acmeUseRateLimits {
c.acmeClientRequestLimit.Take()
if acmeUseRateLimits {
acmeClientRequestLimit.Take()
}
log.Debug().Msgf("Renewing certificate for: %v", domains)
res, err = acmeClient.Certificate.Renew(*renew, true, false, "")
if err != nil {
log.Error().Err(err).Msgf("Couldn't renew certificate for %v, trying to request a new one", domains)
if c.acmeUseRateLimits {
c.acmeClientFailLimit.Take()
if acmeUseRateLimits {
acmeClientFailLimit.Take()
}
res = nil
}
}
if res == nil {
if user != "" {
if err := c.checkUserLimit(user); err != nil {
if err := checkUserLimit(user); err != nil {
return nil, err
}
}
if c.acmeUseRateLimits {
c.acmeClientOrderLimit.Take()
c.acmeClientRequestLimit.Take()
if acmeUseRateLimits {
acmeClientOrderLimit.Take()
acmeClientRequestLimit.Take()
}
log.Debug().Msgf("Re-requesting new certificate for %v", domains)
res, err = acmeClient.Certificate.Obtain(certificate.ObtainRequest{
@@ -256,8 +306,8 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew
Bundle: true,
MustStaple: false,
})
if c.acmeUseRateLimits && err != nil {
c.acmeClientFailLimit.Take()
if acmeUseRateLimits && err != nil {
acmeClientFailLimit.Take()
}
}
if err != nil {
@@ -299,15 +349,136 @@ func (c *AcmeClient) obtainCert(acmeClient *lego.Client, domains []string, renew
return &tlsCertificate, nil
}
func SetupMainDomainCertificates(mainDomainSuffix string, acmeClient *AcmeClient, certDB database.CertDB) error {
func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
// TODO: make it a config flag
const configFile = "acme-account.json"
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := os.ReadFile(configFile); err == nil {
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
return nil, err
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
// Validate Config
_, err := lego.NewClient(myAcmeConfig)
if err != nil {
// TODO: should we fail hard instead?
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
return nil, err
}
privateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
if err != nil {
return nil, err
}
myAcmeAccount = AcmeAccount{
Email: acmeMail,
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
TermsOfServiceAgreed: acmeAcceptTerms,
Kid: acmeEabKID,
HmacEncoded: acmeEabHmac,
})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
myAcmeAccount.Registration = reg
}
}
if myAcmeAccount.Registration != nil {
acmeAccountJSON, err := json.Marshal(myAcmeAccount)
if err != nil {
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
if err != nil {
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}
}
}
}
return myAcmeConfig, nil
}
func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
// getting main cert before ACME account so that we can fail here without hitting rate limits
mainCertBytes, err := certDB.Get(mainDomainSuffix)
if err != nil && !errors.Is(err, database.ErrNotFound) {
return fmt.Errorf("cert database is not working: %w", err)
}
acmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Fatal().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
err = acmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
if enableHTTPServer {
err = acmeClient.Challenge.SetHTTP01Provider(AcmeHTTPChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create HTTP-01 provider")
}
}
}
mainDomainAcmeClient, err = lego.NewClient(acmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
if dnsProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
log.Error().Err(err).Msg("Can't create TLS-ALPN-01 provider")
}
} else {
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
if err != nil {
log.Error().Err(err).Msg("Can't create DNS Challenge provider")
}
err = mainDomainAcmeClient.Challenge.SetDNS01Provider(provider)
if err != nil {
log.Error().Err(err).Msg("Can't create DNS-01 provider")
}
}
}
if mainCertBytes == nil {
_, err = acmeClient.obtainCert(acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", true, mainDomainSuffix, certDB)
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, nil, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Err(err).Msg("Couldn't renew main domain certificate, continuing with mock certs only")
}
@@ -316,7 +487,7 @@ func SetupMainDomainCertificates(mainDomainSuffix string, acmeClient *AcmeClient
return nil
}
func MaintainCertDB(ctx context.Context, interval time.Duration, acmeClient *AcmeClient, mainDomainSuffix string, certDB database.CertDB) {
func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
for {
// delete expired certs that will be invalid until next clean up
threshold := time.Now().Add(interval)
@@ -339,6 +510,14 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, acmeClient *Acm
}
}
log.Debug().Msgf("Removed %d expired certificates from the database", expiredCertCount)
// compact the database
msg, err := certDB.Compact()
if err != nil {
log.Error().Err(err).Msg("Compacting key database failed")
} else {
log.Debug().Msgf("Compacted key database: %s", msg)
}
}
// update main cert
@@ -354,7 +533,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, acmeClient *Acm
} else if tlsCertificates[0].NotAfter.Before(time.Now().Add(30 * 24 * time.Hour)) {
// renew main certificate 30 days before it expires
go (func() {
_, err = acmeClient.obtainCert(acmeClient.dnsChallengerLegoClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", true, mainDomainSuffix, certDB)
_, err = obtainCert(mainDomainAcmeClient, []string{"*" + mainDomainSuffix, mainDomainSuffix[1:]}, res, "", dnsProvider, mainDomainSuffix, acmeUseRateLimits, certDB)
if err != nil {
log.Error().Err(err).Msg("Couldn't renew certificate for main domain")
}

View File

@@ -3,16 +3,13 @@ package certificates
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"codeberg.org/codeberg/pages/server/database"
"github.com/stretchr/testify/assert"
)
func TestMockCert(t *testing.T) {
db := database.NewMockCertDB(t)
db.Mock.On("Put", mock.Anything, mock.Anything).Return(nil)
db, err := database.NewTmpDB()
assert.NoError(t, err)
cert, err := mockCert("example.com", "some error msg", "codeberg.page", db)
assert.NoError(t, err)
if assert.NotEmpty(t, cert) {

View File

@@ -8,15 +8,14 @@ import (
"github.com/rs/zerolog/log"
)
//go:generate go install github.com/vektra/mockery/v2@latest
//go:generate mockery --name CertDB --output . --filename mock.go --inpackage --case underscore
type CertDB interface {
Close() error
Put(name string, cert *certificate.Resource) error
Get(name string) (*certificate.Resource, error)
Delete(key string) error
Items(page, pageSize int) ([]*Cert, error)
// Compact deprecated // TODO: remove in next version
Compact() (string, error)
}
type Cert struct {

View File

@@ -1,122 +1,54 @@
// Code generated by mockery v2.20.0. DO NOT EDIT.
package database
import (
certificate "github.com/go-acme/lego/v4/certificate"
mock "github.com/stretchr/testify/mock"
"fmt"
"time"
"github.com/OrlovEvgeny/go-mcache"
"github.com/go-acme/lego/v4/certificate"
)
// MockCertDB is an autogenerated mock type for the CertDB type
type MockCertDB struct {
mock.Mock
var _ CertDB = tmpDB{}
type tmpDB struct {
intern *mcache.CacheDriver
ttl time.Duration
}
// Close provides a mock function with given fields:
func (_m *MockCertDB) Close() error {
ret := _m.Called()
var r0 error
if rf, ok := ret.Get(0).(func() error); ok {
r0 = rf()
} else {
r0 = ret.Error(0)
}
return r0
func (p tmpDB) Close() error {
_ = p.intern.Close()
return nil
}
// Delete provides a mock function with given fields: key
func (_m *MockCertDB) Delete(key string) error {
ret := _m.Called(key)
var r0 error
if rf, ok := ret.Get(0).(func(string) error); ok {
r0 = rf(key)
} else {
r0 = ret.Error(0)
}
return r0
func (p tmpDB) Put(name string, cert *certificate.Resource) error {
return p.intern.Set(name, cert, p.ttl)
}
// Get provides a mock function with given fields: name
func (_m *MockCertDB) Get(name string) (*certificate.Resource, error) {
ret := _m.Called(name)
var r0 *certificate.Resource
var r1 error
if rf, ok := ret.Get(0).(func(string) (*certificate.Resource, error)); ok {
return rf(name)
func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name)
if !has {
return nil, fmt.Errorf("cert for %q not found", name)
}
if rf, ok := ret.Get(0).(func(string) *certificate.Resource); ok {
r0 = rf(name)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*certificate.Resource)
}
}
if rf, ok := ret.Get(1).(func(string) error); ok {
r1 = rf(name)
} else {
r1 = ret.Error(1)
}
return r0, r1
return cert.(*certificate.Resource), nil
}
// Items provides a mock function with given fields: page, pageSize
func (_m *MockCertDB) Items(page int, pageSize int) ([]*Cert, error) {
ret := _m.Called(page, pageSize)
var r0 []*Cert
var r1 error
if rf, ok := ret.Get(0).(func(int, int) ([]*Cert, error)); ok {
return rf(page, pageSize)
}
if rf, ok := ret.Get(0).(func(int, int) []*Cert); ok {
r0 = rf(page, pageSize)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]*Cert)
}
}
if rf, ok := ret.Get(1).(func(int, int) error); ok {
r1 = rf(page, pageSize)
} else {
r1 = ret.Error(1)
}
return r0, r1
func (p tmpDB) Delete(key string) error {
p.intern.Remove(key)
return nil
}
// Put provides a mock function with given fields: name, cert
func (_m *MockCertDB) Put(name string, cert *certificate.Resource) error {
ret := _m.Called(name, cert)
var r0 error
if rf, ok := ret.Get(0).(func(string, *certificate.Resource) error); ok {
r0 = rf(name, cert)
} else {
r0 = ret.Error(0)
}
return r0
func (p tmpDB) Compact() (string, error) {
p.intern.Truncate()
return "Truncate done", nil
}
type mockConstructorTestingTNewMockCertDB interface {
mock.TestingT
Cleanup(func())
func (p tmpDB) Items(page, pageSize int) ([]*Cert, error) {
return nil, fmt.Errorf("items not implemented for tmpDB")
}
// NewMockCertDB creates a new instance of MockCertDB. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
func NewMockCertDB(t mockConstructorTestingTNewMockCertDB) *MockCertDB {
mock := &MockCertDB{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
func NewTmpDB() (CertDB, error) {
return &tmpDB{
intern: mcache.New(),
ttl: time.Minute,
}, nil
}

134
server/database/pogreb.go Normal file
View File

@@ -0,0 +1,134 @@
package database
import (
"bytes"
"context"
"encoding/gob"
"errors"
"fmt"
"time"
"github.com/akrylysov/pogreb"
"github.com/akrylysov/pogreb/fs"
"github.com/go-acme/lego/v4/certificate"
"github.com/rs/zerolog/log"
)
var _ CertDB = aDB{}
type aDB struct {
ctx context.Context
cancel context.CancelFunc
intern *pogreb.DB
syncInterval time.Duration
}
func (p aDB) Close() error {
p.cancel()
return p.intern.Sync()
}
func (p aDB) Put(name string, cert *certificate.Resource) error {
var resGob bytes.Buffer
if err := gob.NewEncoder(&resGob).Encode(cert); err != nil {
return err
}
return p.intern.Put([]byte(name), resGob.Bytes())
}
func (p aDB) Get(name string) (*certificate.Resource, error) {
cert := &certificate.Resource{}
resBytes, err := p.intern.Get([]byte(name))
if err != nil {
return nil, err
}
if resBytes == nil {
return nil, nil
}
if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(cert); err != nil {
return nil, err
}
return cert, nil
}
func (p aDB) Delete(key string) error {
return p.intern.Delete([]byte(key))
}
func (p aDB) Compact() (string, error) {
result, err := p.intern.Compact()
if err != nil {
return "", err
}
return fmt.Sprintf("%+v", result), nil
}
func (p aDB) Items(_, _ int) ([]*Cert, error) {
items := make([]*Cert, 0, p.intern.Count())
iterator := p.intern.Items()
for {
key, resBytes, err := iterator.Next()
if err != nil {
if errors.Is(err, pogreb.ErrIterationDone) {
break
}
return nil, err
}
res := &certificate.Resource{}
if err := gob.NewDecoder(bytes.NewBuffer(resBytes)).Decode(res); err != nil {
return nil, err
}
cert, err := toCert(string(key), res)
if err != nil {
return nil, err
}
items = append(items, cert)
}
return items, nil
}
var _ CertDB = &aDB{}
func (p aDB) sync() {
for {
err := p.intern.Sync()
if err != nil {
log.Error().Err(err).Msg("Syncing cert database failed")
}
select {
case <-p.ctx.Done():
return
case <-time.After(p.syncInterval):
}
}
}
func NewPogreb(path string) (CertDB, error) {
if path == "" {
return nil, fmt.Errorf("path not set")
}
db, err := pogreb.Open(path, &pogreb.Options{
BackgroundSyncInterval: 30 * time.Second,
BackgroundCompactionInterval: 6 * time.Hour,
FileSystem: fs.OSMMap,
})
if err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background())
result := &aDB{
ctx: ctx,
cancel: cancel,
intern: db,
syncInterval: 5 * time.Minute,
}
go result.sync()
return result, nil
}

View File

@@ -37,7 +37,7 @@ func NewXormDB(dbType, dbConn string) (CertDB, error) {
}
if err := e.Sync2(new(Cert)); err != nil {
return nil, fmt.Errorf("could not sync db model :%w", err)
return nil, fmt.Errorf("cound not sync db model :%w", err)
}
return &xDB{
@@ -106,6 +106,11 @@ func (x xDB) Delete(domain string) error {
return err
}
func (x xDB) Compact() (string, error) {
// not needed
return "", nil
}
// Items return al certs from db, if pageSize is 0 it does not use limit
func (x xDB) Items(page, pageSize int) ([]*Cert, error) {
// paginated return

View File

@@ -11,11 +11,9 @@ import (
// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
var lookupCacheTimeout = 15 * time.Minute
var defaultPagesRepo = "pages"
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT
var cname string
var err error
@@ -52,10 +50,10 @@ func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLo
targetBranch = cnameParts[len(cnameParts)-3]
}
if targetRepo == "" {
targetRepo = defaultPagesRepo
targetRepo = "pages"
}
if targetBranch == "" && targetRepo != defaultPagesRepo {
targetBranch = firstDefaultBranch
if targetBranch == "" && targetRepo != "pages" {
targetBranch = "pages"
}
// if targetBranch is still empty, the caller must find the default branch
return

View File

@@ -17,13 +17,12 @@ import (
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/version"
)
var ErrorNotFound = errors.New("not found")
const (
// cache key prefixes
// cache key prefixe
branchTimestampCacheKeyPrefix = "branchTime"
defaultBranchCacheKeyPrefix = "defaultBranch"
rawContentCacheKeyPrefix = "rawContent"
@@ -77,13 +76,7 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
defaultMimeType = "application/octet-stream"
}
sdk, err := gitea.NewClient(
giteaRoot,
gitea.SetHTTPClient(&stdClient),
gitea.SetToken(giteaAPIToken),
gitea.SetUserAgent("pages-server/"+version.Version),
)
sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
return &Client{
sdkClient: sdk,
responseCache: respCache,
@@ -119,7 +112,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
if cache, ok := client.responseCache.Get(cacheKey); ok {
cache := cache.(FileResponse)
cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
// TODO: check against some timestamp mismatch?!?
// TODO: check against some timestamp missmatch?!?
if cache.Exists {
if cache.IsSymlink {
linkDest := string(cache.Body)
@@ -152,10 +145,6 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
}
linkDest := strings.TrimSpace(string(linkDestBytes))
// handle relative links
// we first remove the link from the path, and make a relative join (resolve parent paths like "/../" too)
linkDest = path.Join(path.Dir(resource), linkDest)
// we store symlink not content to reduce duplicates in cache
if err := client.responseCache.Set(cacheKey, FileResponse{
Exists: true,
@@ -179,7 +168,7 @@ func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource str
return reader, resp.Response.Header, resp.StatusCode, err
}
// now we write to cache and respond at the same time
// now we write to cache and respond at the sime time
fileResp := FileResponse{
Exists: true,
ETag: resp.Header.Get(ETagHeader),
@@ -285,11 +274,11 @@ func shouldRespBeSavedToCache(resp *http.Response) bool {
return false
}
contentLength, err := strconv.ParseInt(contentLengthRaw, 10, 64)
contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
if err != nil {
log.Error().Err(err).Msg("could not parse content length")
}
// if content to big or could not be determined we not cache it
return contentLength > 0 && contentLength < fileCacheSizeLimit
return contentLeng > 0 && contentLeng < fileCacheSizeLimit
}

View File

@@ -10,12 +10,14 @@ import (
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/version"
)
const (
headerAccessControlAllowOrigin = "Access-Control-Allow-Origin"
headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
defaultPagesRepo = "pages"
defaultPagesBranch = "pages"
)
// Handler handles a single HTTP request to the web server.
@@ -23,14 +25,13 @@ func Handler(mainDomainSuffix, rawDomain string,
giteaClient *gitea.Client,
rawInfoPage string,
blacklistedPaths, allowedCorsDomains []string,
defaultPagesBranches []string,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
ctx := context.New(w, req)
ctx.RespWriter.Header().Set("Server", "pages-server")
ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
@@ -87,28 +88,26 @@ func Handler(mainDomainSuffix, rawDomain string,
pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
log.Debug().Msg("raw domain request detected")
log.Debug().Msg("raw domain request detecded")
handleRaw(log, ctx, giteaClient,
mainDomainSuffix, rawInfoPage,
trimmedHost,
pathElements,
canonicalDomainCache, redirectsCache)
canonicalDomainCache)
} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
log.Debug().Msg("subdomain request detected")
log.Debug().Msg("subdomain request detecded")
handleSubDomain(log, ctx, giteaClient,
mainDomainSuffix,
defaultPagesBranches,
trimmedHost,
pathElements,
canonicalDomainCache, redirectsCache)
canonicalDomainCache)
} else {
log.Debug().Msg("custom domain request detected")
log.Debug().Msg("custom domain request detecded")
handleCustomDomain(log, ctx, giteaClient,
mainDomainSuffix,
trimmedHost,
pathElements,
defaultPagesBranches[0],
dnsLookupCache, canonicalDomainCache, redirectsCache)
dnsLookupCache, canonicalDomainCache)
}
}
}

View File

@@ -18,11 +18,10 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
mainDomainSuffix string,
trimmedHost string,
pathElements []string,
firstDefaultBranch string,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache cache.SetGetKey,
) {
// Serve pages from custom domains
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, dnsLookupCache)
if targetOwner == "" {
html.ReturnErrorPage(ctx,
"could not obtain repo owner from custom domain",
@@ -49,11 +48,11 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
}, canonicalLink); works {
canonicalDomain, valid := targetOpt.CheckCanonicalDomain(giteaClient, trimmedHost, mainDomainSuffix, canonicalDomainCache)
if !valid {
html.ReturnErrorPage(ctx, "", http.StatusMisdirectedRequest)
html.ReturnErrorPage(ctx, "domain not specified in <code>.domains</code> file", http.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHost {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, firstDefaultBranch, dnsLookupCache)
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix, dnsLookupCache)
if targetOwner != "" {
ctx.Redirect("https://"+canonicalDomain+"/"+targetOpt.TargetPath, http.StatusTemporaryRedirect)
return
@@ -64,7 +63,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
}
log.Debug().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}

View File

@@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
mainDomainSuffix, rawInfoPage string,
trimmedHost string,
pathElements []string,
canonicalDomainCache, redirectsCache cache.SetGetKey,
canonicalDomainCache cache.SetGetKey,
) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")
@@ -41,7 +41,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
TargetPath: path.Join(pathElements[3:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve raw domain with specified branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}
log.Debug().Msg("missing branch info")
@@ -58,7 +58,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
TargetPath: path.Join(pathElements[2:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve raw domain with default branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("raw domain could not find repo '%s/%s' or repo is empty", targetOpt.TargetOwner, targetOpt.TargetRepo),

View File

@@ -7,7 +7,6 @@ import (
"strings"
"github.com/rs/zerolog"
"golang.org/x/exp/slices"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
@@ -18,10 +17,9 @@ import (
func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix string,
defaultPagesBranches []string,
trimmedHost string,
pathElements []string,
canonicalDomainCache, redirectsCache cache.SetGetKey,
canonicalDomainCache cache.SetGetKey,
) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")
@@ -53,7 +51,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
TargetPath: path.Join(pathElements[2:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve with specified repo and branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
@@ -65,25 +63,16 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
// Check if the first directory is a branch for the defaultPagesRepo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
targetBranch := pathElements[0][1:]
// if the default pages branch can be determined exactly, it does not need to be set
if len(defaultPagesBranches) == 1 && slices.Contains(defaultPagesBranches, targetBranch) {
// example.codeberg.org/@pages/... redirects to example.codeberg.org/...
ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
return
}
log.Debug().Msg("main domain preparations, now trying with specified branch")
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: defaultPagesRepo,
TargetBranch: targetBranch,
TargetBranch: pathElements[0][1:],
TargetPath: path.Join(pathElements[1:]...),
}, true); works {
log.Trace().Msg("tryUpstream: serve default pages repo with specified branch")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
} else {
html.ReturnErrorPage(ctx,
fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", targetOpt.TargetBranch, targetOpt.TargetOwner, targetOpt.TargetRepo),
@@ -92,37 +81,20 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
return
}
for _, defaultPagesBranch := range defaultPagesBranches {
// Check if the first directory is a repo with a default pages branch
// example.codeberg.page/myrepo/index.html
// example.codeberg.page/{PAGES_BRANCHE}/... is not allowed here.
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != defaultPagesBranch {
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: pathElements[0],
TargetBranch: defaultPagesBranch,
TargetPath: path.Join(pathElements[1:]...),
}, false); works {
log.Debug().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
return
}
}
// Try to use the defaultPagesRepo on an default pages branch
// example.codeberg.page/index.html
log.Debug().Msg("main domain preparations, now trying with default repo")
// Check if the first directory is a repo with a defaultPagesRepo branch
// example.codeberg.page/myrepo/index.html
// example.codeberg.page/pages/... is not allowed here.
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != defaultPagesRepo {
if targetOpt, works := tryBranch(log, ctx, giteaClient, &upstream.Options{
TryIndexPages: true,
TargetOwner: targetOwner,
TargetRepo: defaultPagesRepo,
TargetRepo: pathElements[0],
TargetBranch: defaultPagesBranch,
TargetPath: path.Join(pathElements...),
TargetPath: path.Join(pathElements[1:]...),
}, false); works {
log.Debug().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
log.Debug().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}
}
@@ -137,7 +109,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
TargetPath: path.Join(pathElements...),
}, false); works {
log.Debug().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache, redirectsCache)
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost, targetOpt, canonicalDomainCache)
return
}

View File

@@ -19,8 +19,6 @@ func TestHandlerPerformance(t *testing.T) {
"https://docs.codeberg.org/pages/raw-content/",
[]string{"/.well-known/acme-challenge/"},
[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
[]string{"pages"},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
)

View File

@@ -18,10 +18,9 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost string,
options *upstream.Options,
canonicalDomainCache cache.SetGetKey,
redirectsCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {
if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := options.CheckCanonicalDomain(giteaClient, "", mainDomainSuffix, canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], mainDomainSuffix) {
canonicalPath := ctx.Req.RequestURI
@@ -40,7 +39,7 @@ func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
options.Host = trimmedHost
// Try to request the file from the Gitea API
if !options.Upstream(ctx, giteaClient, redirectsCache) {
if !options.Upstream(ctx, giteaClient) {
html.ReturnErrorPage(ctx, "", ctx.StatusCode)
}
}

27
server/setup.go Normal file
View File

@@ -0,0 +1,27 @@
package server
import (
"net/http"
"strings"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/utils"
)
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
challengePath := "/.well-known/acme-challenge/"
return func(w http.ResponseWriter, req *http.Request) {
ctx := context.New(w, req)
if strings.HasPrefix(ctx.Path(), challengePath) {
challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + strings.TrimPrefix(ctx.Path(), challengePath))
if !ok || challenge == nil {
ctx.String("no challenge for this token", http.StatusNotFound)
}
ctx.String(challenge.(string))
} else {
ctx.Redirect("https://"+ctx.Host()+ctx.Path(), http.StatusMovedPermanently)
}
}
}

View File

@@ -1,117 +0,0 @@
package upstream
import (
"strconv"
"strings"
"time"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
)
type Redirect struct {
From string
To string
StatusCode int
}
// redirectsCacheTimeout specifies the timeout for the redirects cache.
var redirectsCacheTimeout = 10 * time.Minute
const redirectsConfig = "_redirects"
// getRedirects returns redirects specified in the _redirects file.
func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect {
var redirects []Redirect
cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch
// Check for cached redirects
if cachedValue, ok := redirectsCache.Get(cacheKey); ok {
redirects = cachedValue.([]Redirect)
} else {
// Get _redirects file and parse
body, err := giteaClient.GiteaRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, redirectsConfig)
if err == nil {
for _, line := range strings.Split(string(body), "\n") {
redirectArr := strings.Fields(line)
// Ignore comments and invalid lines
if strings.HasPrefix(line, "#") || len(redirectArr) < 2 {
continue
}
// Get redirect status code
statusCode := 301
if len(redirectArr) == 3 {
statusCode, err = strconv.Atoi(redirectArr[2])
if err != nil {
log.Info().Err(err).Msgf("could not read %s of %s/%s", redirectsConfig, o.TargetOwner, o.TargetRepo)
}
}
redirects = append(redirects, Redirect{
From: redirectArr[0],
To: redirectArr[1],
StatusCode: statusCode,
})
}
}
_ = redirectsCache.Set(cacheKey, redirects, redirectsCacheTimeout)
}
return redirects
}
func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) {
if len(redirects) > 0 {
for _, redirect := range redirects {
reqUrl := ctx.Req.RequestURI
// remove repo and branch from request url
reqUrl = strings.TrimPrefix(reqUrl, "/"+o.TargetRepo)
reqUrl = strings.TrimPrefix(reqUrl, "/@"+o.TargetBranch)
// check if from url matches request url
if strings.TrimSuffix(redirect.From, "/") == strings.TrimSuffix(reqUrl, "/") {
// do rewrite if status code is 200
if redirect.StatusCode == 200 {
o.TargetPath = redirect.To
o.Upstream(ctx, giteaClient, redirectsCache)
return true
} else {
ctx.Redirect(redirect.To, redirect.StatusCode)
return true
}
}
// handle wildcard redirects
trimmedFromUrl := strings.TrimSuffix(redirect.From, "/*")
if strings.HasSuffix(redirect.From, "/*") && strings.HasPrefix(reqUrl, trimmedFromUrl) {
if strings.Contains(redirect.To, ":splat") {
splatUrl := strings.ReplaceAll(redirect.To, ":splat", strings.TrimPrefix(reqUrl, trimmedFromUrl))
// do rewrite if status code is 200
if redirect.StatusCode == 200 {
o.TargetPath = splatUrl
o.Upstream(ctx, giteaClient, redirectsCache)
return true
} else {
ctx.Redirect(splatUrl, redirect.StatusCode)
return true
}
} else {
// do rewrite if status code is 200
if redirect.StatusCode == 200 {
o.TargetPath = redirect.To
o.Upstream(ctx, giteaClient, redirectsCache)
return true
} else {
ctx.Redirect(redirect.To, redirect.StatusCode)
return true
}
}
}
}
}
return false
}

View File

@@ -11,7 +11,6 @@ import (
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
)
@@ -53,7 +52,7 @@ type Options struct {
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) (final bool) {
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.TargetOwner == "" || o.TargetRepo == "" {
@@ -104,12 +103,6 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
// Handle not found error
if err != nil && errors.Is(err, gitea.ErrorNotFound) {
// Get and match redirects
redirects := o.getRedirects(giteaClient, redirectsCache)
if o.matchRedirects(ctx, giteaClient, redirects, redirectsCache) {
return true
}
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
@@ -117,7 +110,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) {
if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
@@ -125,7 +118,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
optionsForIndexPages.appendTrailingSlash = false
optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
if optionsForIndexPages.Upstream(ctx, giteaClient, redirectsCache) {
if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
@@ -138,12 +131,11 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
if optionsForNotFoundPages.Upstream(ctx, giteaClient, redirectsCache) {
if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
return true
}
}
}
return false
}
@@ -176,7 +168,7 @@ func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redi
ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
return true
}
if strings.HasSuffix(ctx.Path(), "/index.html") && !o.ServeRaw {
if strings.HasSuffix(ctx.Path(), "/index.html") {
ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
return true
}