s83/pages-server
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2024-11-23 06:18:59 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Add config file and rework cli parsing and passing of config values (#263)

Browse Source 
Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/263
Reviewed-by: 6543 <6543@obermui.de>
Co-authored-by: crapStone <me@crapstone.dev>
Co-committed-by: crapStone <me@crapstone.dev>
This commit is contained in:
crapStone 2024-02-15 16:08:29 +00:00 committed by 6543
parent c1fbe861fe
commit 7e80ade24b
37 changed files with 1270 additions and 344 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.env-dev Normal file
View File

@ -0,0 +1,11 @@
ACME_API=https://acme.mock.directory
ACME_ACCEPT_TERMS=true
PAGES_DOMAIN=localhost.mock.directory
RAW_DOMAIN=raw.localhost.mock.directory
PAGES_BRANCHES=pages,master,main
GITEA_ROOT=https://codeberg.org
PORT=4430
HTTP_PORT=8880
ENABLE_HTTP_SERVER=true
LOG_LEVEL=trace
ACME_ACCOUNT_CONFIG=integration/acme-account.json

26
.vscode/launch.json vendored Normal file
View File

@ -0,0 +1,26 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch PagesServer",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/main.go",
"args": ["sqlite", "sqlite_unlock_notify", "netgo"],
"envFile": "${workspaceFolder}/.env-dev"
},
{
"name": "Launch PagesServer integration test",
"type": "go",
"request": "launch",
"mode": "auto",
"program": "${workspaceFolder}/integration/main_test.go",
"args": ["codeberg.org/codeberg/pages/integration/..."],
"buildFlags": ["-tags", "'integration sqlite sqlite_unlock_notify netgo'"]
}
]
}

15
Justfile
View File

@ -4,14 +4,9 @@ TAGS := 'sqlite sqlite_unlock_notify netgo'
dev:
#!/usr/bin/env bash
set -euxo pipefail
export ACME_API=https://acme.mock.directory
export ACME_ACCEPT_TERMS=true
export PAGES_DOMAIN=localhost.mock.directory
export RAW_DOMAIN=raw.localhost.mock.directory
export PORT=4430
export HTTP_PORT=8880
export ENABLE_HTTP_SERVER=true
export LOG_LEVEL=trace
set -a # automatically export all variables
source .env-dev
set +a
go run -tags '{{TAGS}}' .
build:
@ -42,10 +37,10 @@ tool-gofumpt:
fi
test:
go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/...
test-run TEST:
go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/
go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/...
integration:
go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/...

6
cmd/certs.go → cli/certs.go
View File

@ -1,4 +1,4 @@
package cmd
package cli
import (
"fmt"
@ -26,7 +26,7 @@ var Certs = &cli.Command{
}
func listCerts(ctx *cli.Context) error {
certDB, closeFn, err := openCertDB(ctx)
certDB, closeFn, err := OpenCertDB(ctx)
if err != nil {
return err
}
@ -53,7 +53,7 @@ func removeCert(ctx *cli.Context) error {
domains := ctx.Args().Slice()
certDB, closeFn, err := openCertDB(ctx)
certDB, closeFn, err := OpenCertDB(ctx)
if err != nil {
return err
}

52
cmd/flags.go → cli/flags.go
View File

@ -1,4 +1,4 @@
package cmd
package cli
import (
"github.com/urfave/cli/v2"
@ -29,26 +29,35 @@ var (
Name: "gitea-root",
Usage: "specifies the root URL of the Gitea instance, without a trailing slash.",
EnvVars: []string{"GITEA_ROOT"},
Value: "https://codeberg.org",
},
// GiteaApiToken specifies an api token for the Gitea instance
&cli.StringFlag{
Name: "gitea-api-token",
Usage: "specifies an api token for the Gitea instance",
EnvVars: []string{"GITEA_API_TOKEN"},
Value: "",
},
&cli.BoolFlag{
Name: "enable-lfs-support",
Usage: "enable lfs support, require gitea >= v1.17.0 as backend",
EnvVars: []string{"ENABLE_LFS_SUPPORT"},
Value: true,
Value: false,
},
&cli.BoolFlag{
Name: "enable-symlink-support",
Usage: "follow symlinks if enabled, require gitea >= v1.18.0 as backend",
EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"},
Value: true,
Value: false,
},
&cli.StringFlag{
Name: "default-mime-type",
Usage: "specifies the default mime type for files that don't have a specific mime type.",
EnvVars: []string{"DEFAULT_MIME_TYPE"},
Value: "application/octet-stream",
},
&cli.StringSliceFlag{
Name: "forbidden-mime-types",
Usage: "specifies the forbidden mime types. Use this flag multiple times for multiple mime types.",
EnvVars: []string{"FORBIDDEN_MIME_TYPES"},
},
// ###########################
@ -61,7 +70,6 @@ var (
Name: "pages-domain",
Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages",
EnvVars: []string{"PAGES_DOMAIN"},
Value: "codeberg.page",
},
// RawDomain specifies the domain from which raw repository content shall be served in the following format:
// https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...}
@ -70,7 +78,6 @@ var (
Name: "raw-domain",
Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting",
EnvVars: []string{"RAW_DOMAIN"},
Value: "raw.codeberg.page",
},
// #########################
@ -98,19 +105,38 @@ var (
Name: "enable-http-server",
Usage: "start a http server to redirect to https and respond to http acme challenges",
EnvVars: []string{"ENABLE_HTTP_SERVER"},
Value: false,
},
// Default branches to fetch assets from
&cli.StringSliceFlag{
Name: "pages-branch",
Usage: "define a branch to fetch assets from. Use this flag multiple times for multiple branches.",
EnvVars: []string{"PAGES_BRANCHES"},
Value: cli.NewStringSlice("pages"),
},
&cli.StringSliceFlag{
Name: "allowed-cors-domains",
Usage: "specify allowed CORS domains. Use this flag multiple times for multiple domains.",
EnvVars: []string{"ALLOWED_CORS_DOMAINS"},
},
&cli.StringSliceFlag{
Name: "blacklisted-paths",
Usage: "return an error on these url paths.Use this flag multiple times for multiple paths.",
EnvVars: []string{"BLACKLISTED_PATHS"},
},
&cli.StringFlag{
Name: "log-level",
Value: "warn",
Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal",
EnvVars: []string{"LOG_LEVEL"},
},
// Default branches to fetch assets from
&cli.StringSliceFlag{
Name: "pages-branch",
Usage: "define a branch to fetch assets from",
EnvVars: []string{"PAGES_BRANCHES"},
Value: cli.NewStringSlice("pages"),
&cli.StringFlag{
Name: "config-file",
Usage: "specify the location of the config file",
Aliases: []string{"config"},
EnvVars: []string{"CONFIG_FILE"},
},
// ############################

39
cli/setup.go Normal file
View File

@ -0,0 +1,39 @@
package cli
import (
"fmt"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/database"
"codeberg.org/codeberg/pages/server/version"
)
func CreatePagesApp() *cli.App {
app := cli.NewApp()
app.Name = "pages-server"
app.Version = version.Version
app.Usage = "pages server"
app.Flags = ServerFlags
app.Commands = []*cli.Command{
Certs,
}
return app
}
func OpenCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return nil, nil, fmt.Errorf("could not connect to database: %w", err)
}
closeFn = func() {
if err := certDB.Close(); err != nil {
log.Error().Err(err)
}
}
return certDB, closeFn, nil
}

150
cmd/main.go
View File

@ -1,150 +0,0 @@
package cmd
import (
"context"
"crypto/tls"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/handler"
)
// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed.
// TODO: make it a flag
var AllowedCorsDomains = []string{
"fonts.codeberg.org",
"design.codeberg.org",
}
// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages.
// TODO: Make it a flag too
var BlacklistedPaths = []string{
"/.well-known/acme-challenge/",
}
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
// Initialize the logger.
logLevel, err := zerolog.ParseLevel(ctx.String("log-level"))
if err != nil {
return err
}
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
giteaRoot := ctx.String("gitea-root")
giteaAPIToken := ctx.String("gitea-api-token")
rawDomain := ctx.String("raw-domain")
defaultBranches := ctx.StringSlice("pages-branch")
mainDomainSuffix := ctx.String("pages-domain")
listeningHost := ctx.String("host")
listeningSSLPort := ctx.Uint("port")
listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort)
listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port"))
enableHTTPServer := ctx.Bool("enable-http-server")
allowedCorsDomains := AllowedCorsDomains
if rawDomain != "" {
allowedCorsDomains = append(allowedCorsDomains, rawDomain)
}
// Make sure MainDomain has a trailing dot
if !strings.HasPrefix(mainDomainSuffix, ".") {
mainDomainSuffix = "." + mainDomainSuffix
}
if len(defaultBranches) == 0 {
return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
}
// Init ssl cert database
certDB, closeFn, err := openCertDB(ctx)
if err != nil {
return err
}
defer closeFn()
keyCache := cache.NewKeyValueCache()
challengeCache := cache.NewKeyValueCache()
// canonicalDomainCache stores canonical domains
canonicalDomainCache := cache.NewKeyValueCache()
// dnsLookupCache stores DNS lookups for custom domains
dnsLookupCache := cache.NewKeyValueCache()
// redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewKeyValueCache()
// clientResponseCache stores responses from the Gitea server
clientResponseCache := cache.NewKeyValueCache()
giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support"))
if err != nil {
return fmt.Errorf("could not create new gitea client: %v", err)
}
acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache)
if err != nil {
return err
}
if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil {
return err
}
// Create listener for SSL connections
log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
listener, err := net.Listen("tcp", listeningSSLAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
}
// Setup listener for SSL connections
listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix,
giteaClient,
acmeClient,
defaultBranches[0],
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB))
interval := 12 * time.Hour
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
defer cancelCertMaintain()
go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB)
if enableHTTPServer {
// Create handler for http->https redirect and http acme challenges
httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort)
// Create listener for http and start listening
go func() {
log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
if err != nil {
log.Panic().Err(err).Msg("Couldn't start HTTP fastServer")
}
}()
}
// Create ssl handler based on settings
sslHandler := handler.Handler(mainDomainSuffix, rawDomain,
giteaClient,
BlacklistedPaths, allowedCorsDomains,
defaultBranches,
dnsLookupCache, canonicalDomainCache, redirectsCache)
// Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
if err := http.Serve(listener, sslHandler); err != nil {
log.Panic().Err(err).Msg("Couldn't start fastServer")
}
return nil
}

64
cmd/setup.go
View File

@ -1,64 +0,0 @@
package cmd
import (
"errors"
"fmt"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/database"
)
var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) {
certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn"))
if err != nil {
return nil, nil, fmt.Errorf("could not connect to database: %w", err)
}
closeFn = func() {
if err := certDB.Close(); err != nil {
log.Error().Err(err)
}
}
return certDB, closeFn, nil
}
func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) {
acmeAPI := ctx.String("acme-api-endpoint")
acmeMail := ctx.String("acme-email")
acmeEabHmac := ctx.String("acme-eab-hmac")
acmeEabKID := ctx.String("acme-eab-kid")
acmeAcceptTerms := ctx.Bool("acme-accept-terms")
dnsProvider := ctx.String("dns-provider")
acmeUseRateLimits := ctx.Bool("acme-use-rate-limits")
acmeAccountConf := ctx.String("acme-account-config")
// check config
if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" {
return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
}
if acmeEabHmac != "" && acmeEabKID == "" {
return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
} else if acmeEabHmac == "" && acmeEabKID != "" {
return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
}
return certificates.NewAcmeClient(
acmeAccountConf,
acmeAPI,
acmeMail,
acmeEabHmac,
acmeEabKID,
dnsProvider,
acmeAcceptTerms,
enableHTTPServer,
acmeUseRateLimits,
challengeCache,
)
}

33
config/assets/test_config.toml Normal file
View File

@ -0,0 +1,33 @@
logLevel = 'trace'
[server]
host = '127.0.0.1'
port = 443
httpPort = 80
httpServerEnabled = true
mainDomain = 'codeberg.page'
rawDomain = 'raw.codeberg.page'
allowedCorsDomains = ['fonts.codeberg.org', 'design.codeberg.org']
blacklistedPaths = ['do/not/use']
[gitea]
root = 'codeberg.org'
token = 'XXXXXXXX'
lfsEnabled = true
followSymlinks = true
defaultMimeType = "application/wasm"
forbiddenMimeTypes = ["text/html"]
[database]
type = 'sqlite'
conn = 'certs.sqlite'
[ACME]
email = 'a@b.c'
apiEndpoint = 'https://example.com'
acceptTerms = false
useRateLimits = true
eab_hmac = 'asdf'
eab_kid = 'qwer'
dnsProvider = 'cloudflare.com'
accountConfigFile = 'nope'

46
config/config.go Normal file
View File

@ -0,0 +1,46 @@
package config
type Config struct {
LogLevel string `default:"warn"`
Server ServerConfig
Gitea GiteaConfig
Database DatabaseConfig
ACME ACMEConfig
}
type ServerConfig struct {
Host string `default:"[::]"`
Port uint16 `default:"443"`
HttpPort uint16 `default:"80"`
HttpServerEnabled bool `default:"true"`
MainDomain string
RawDomain string
PagesBranches []string
AllowedCorsDomains []string
BlacklistedPaths []string
}
type GiteaConfig struct {
Root string
Token string
LFSEnabled bool `default:"false"`
FollowSymlinks bool `default:"false"`
DefaultMimeType string `default:"application/octet-stream"`
ForbiddenMimeTypes []string
}
type DatabaseConfig struct {
Type string `default:"sqlite3"`
Conn string `default:"certs.sqlite"`
}
type ACMEConfig struct {
Email string
APIEndpoint string `default:"https://acme-v02.api.letsencrypt.org/directory"`
AcceptTerms bool `default:"false"`
UseRateLimits bool `default:"true"`
EAB_HMAC string
EAB_KID string
DNSProvider string
AccountConfigFile string `default:"acme-account.json"`
}

147
config/setup.go Normal file
View File

@ -0,0 +1,147 @@
package config
import (
"os"
"path"
"github.com/creasty/defaults"
"github.com/pelletier/go-toml/v2"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
)
var ALWAYS_BLACKLISTED_PATHS = []string{
"/.well-known/acme-challenge/",
}
func NewDefaultConfig() Config {
config := Config{}
if err := defaults.Set(&config); err != nil {
panic(err)
}
// defaults does not support setting arrays from strings
config.Server.PagesBranches = []string{"main", "master", "pages"}
return config
}
func ReadConfig(ctx *cli.Context) (*Config, error) {
config := NewDefaultConfig()
// if config is not given as argument return empty config
if !ctx.IsSet("config-file") {
return &config, nil
}
configFile := path.Clean(ctx.String("config-file"))
log.Debug().Str("config-file", configFile).Msg("reading config file")
content, err := os.ReadFile(configFile)
if err != nil {
return nil, err
}
err = toml.Unmarshal(content, &config)
return &config, err
}
func MergeConfig(ctx *cli.Context, config *Config) {
if ctx.IsSet("log-level") {
config.LogLevel = ctx.String("log-level")
}
mergeServerConfig(ctx, &config.Server)
mergeGiteaConfig(ctx, &config.Gitea)
mergeDatabaseConfig(ctx, &config.Database)
mergeACMEConfig(ctx, &config.ACME)
}
func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
if ctx.IsSet("host") {
config.Host = ctx.String("host")
}
if ctx.IsSet("port") {
config.Port = uint16(ctx.Uint("port"))
}
if ctx.IsSet("http-port") {
config.HttpPort = uint16(ctx.Uint("http-port"))
}
if ctx.IsSet("enable-http-server") {
config.HttpServerEnabled = ctx.Bool("enable-http-server")
}
if ctx.IsSet("pages-domain") {
config.MainDomain = ctx.String("pages-domain")
}
if ctx.IsSet("raw-domain") {
config.RawDomain = ctx.String("raw-domain")
}
if ctx.IsSet("pages-branch") {
config.PagesBranches = ctx.StringSlice("pages-branch")
}
if ctx.IsSet("allowed-cors-domains") {
config.AllowedCorsDomains = ctx.StringSlice("allowed-cors-domains")
}
if ctx.IsSet("blacklisted-paths") {
config.BlacklistedPaths = ctx.StringSlice("blacklisted-paths")
}
// add the paths that should always be blacklisted
config.BlacklistedPaths = append(config.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
}
func mergeGiteaConfig(ctx *cli.Context, config *GiteaConfig) {
if ctx.IsSet("gitea-root") {
config.Root = ctx.String("gitea-root")
}
if ctx.IsSet("gitea-api-token") {
config.Token = ctx.String("gitea-api-token")
}
if ctx.IsSet("enable-lfs-support") {
config.LFSEnabled = ctx.Bool("enable-lfs-support")
}
if ctx.IsSet("enable-symlink-support") {
config.FollowSymlinks = ctx.Bool("enable-symlink-support")
}
if ctx.IsSet("default-mime-type") {
config.DefaultMimeType = ctx.String("default-mime-type")
}
if ctx.IsSet("forbidden-mime-types") {
config.ForbiddenMimeTypes = ctx.StringSlice("forbidden-mime-types")
}
}
func mergeDatabaseConfig(ctx *cli.Context, config *DatabaseConfig) {
if ctx.IsSet("db-type") {
config.Type = ctx.String("db-type")
}
if ctx.IsSet("db-conn") {
config.Conn = ctx.String("db-conn")
}
}
func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) {
if ctx.IsSet("acme-email") {
config.Email = ctx.String("acme-email")
}
if ctx.IsSet("acme-api-endpoint") {
config.APIEndpoint = ctx.String("acme-api-endpoint")
}
if ctx.IsSet("acme-accept-terms") {
config.AcceptTerms = ctx.Bool("acme-accept-terms")
}
if ctx.IsSet("acme-use-rate-limits") {
config.UseRateLimits = ctx.Bool("acme-use-rate-limits")
}
if ctx.IsSet("acme-eab-hmac") {
config.EAB_HMAC = ctx.String("acme-eab-hmac")
}
if ctx.IsSet("acme-eab-kid") {
config.EAB_KID = ctx.String("acme-eab-kid")
}
if ctx.IsSet("dns-provider") {
config.DNSProvider = ctx.String("dns-provider")
}
if ctx.IsSet("acme-account-config") {
config.AccountConfigFile = ctx.String("acme-account-config")
}
}

596
config/setup_test.go Normal file
View File

@ -0,0 +1,596 @@
package config
import (
"context"
"os"
"testing"
"github.com/pelletier/go-toml/v2"
"github.com/stretchr/testify/assert"
"github.com/urfave/cli/v2"
cmd "codeberg.org/codeberg/pages/cli"
)
func runApp(t *testing.T, fn func(*cli.Context) error, args []string) {
app := cmd.CreatePagesApp()
app.Action = fn
appCtx, appCancel := context.WithCancel(context.Background())
defer appCancel()
// os.Args always contains the binary name
args = append([]string{"testing"}, args...)
err := app.RunContext(appCtx, args)
assert.NoError(t, err)
}
// fixArrayFromCtx fixes the number of "changed" strings in a string slice according to the number of values in the context.
// This is a workaround because the cli library has a bug where the number of values in the context gets bigger the more tests are run.
func fixArrayFromCtx(ctx *cli.Context, key string, expected []string) []string {
if ctx.IsSet(key) {
ctxSlice := ctx.StringSlice(key)
if len(ctxSlice) > 1 {
for i := 1; i < len(ctxSlice); i++ {
expected = append([]string{"changed"}, expected...)
}
}
}
return expected
}
func readTestConfig() (*Config, error) {
content, err := os.ReadFile("assets/test_config.toml")
if err != nil {
return nil, err
}
expectedConfig := NewDefaultConfig()
err = toml.Unmarshal(content, &expectedConfig)
if err != nil {
return nil, err
}
return &expectedConfig, nil
}
func TestReadConfigShouldReturnEmptyConfigWhenConfigArgEmpty(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg, err := ReadConfig(ctx)
expected := NewDefaultConfig()
assert.Equal(t, &expected, cfg)
return err
},
[]string{},
)
}
func TestReadConfigShouldReturnConfigFromFileWhenConfigArgPresent(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg, err := ReadConfig(ctx)
if err != nil {
return err
}
expectedConfig, err := readTestConfig()
if err != nil {
return err
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{"--config-file", "assets/test_config.toml"},
)
}
func TestValuesReadFromConfigFileShouldBeOverwrittenByArgs(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg, err := ReadConfig(ctx)
if err != nil {
return err
}
MergeConfig(ctx, cfg)
expectedConfig, err := readTestConfig()
if err != nil {
return err
}
expectedConfig.LogLevel = "debug"
expectedConfig.Gitea.Root = "not-codeberg.org"
expectedConfig.ACME.AcceptTerms = true
expectedConfig.Server.Host = "172.17.0.2"
expectedConfig.Server.BlacklistedPaths = append(expectedConfig.Server.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--config-file", "assets/test_config.toml",
"--log-level", "debug",
"--gitea-root", "not-codeberg.org",
"--acme-accept-terms",
"--host", "172.17.0.2",
},
)
}
func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &Config{
LogLevel: "original",
Server: ServerConfig{
Host: "original",
Port: 8080,
HttpPort: 80,
HttpServerEnabled: false,
MainDomain: "original",
RawDomain: "original",
PagesBranches: []string{"original"},
AllowedCorsDomains: []string{"original"},
BlacklistedPaths: []string{"original"},
},
Gitea: GiteaConfig{
Root: "original",
Token: "original",
LFSEnabled: false,
FollowSymlinks: false,
DefaultMimeType: "original",
ForbiddenMimeTypes: []string{"original"},
},
Database: DatabaseConfig{
Type: "original",
Conn: "original",
},
ACME: ACMEConfig{
Email: "original",
APIEndpoint: "original",
AcceptTerms: false,
UseRateLimits: false,
EAB_HMAC: "original",
EAB_KID: "original",
DNSProvider: "original",
AccountConfigFile: "original",
},
}
MergeConfig(ctx, cfg)
expectedConfig := &Config{
LogLevel: "changed",
Server: ServerConfig{
Host: "changed",
Port: 8443,
HttpPort: 443,
HttpServerEnabled: true,
MainDomain: "changed",
RawDomain: "changed",
PagesBranches: []string{"changed"},
AllowedCorsDomains: []string{"changed"},
BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...),
},
Gitea: GiteaConfig{
Root: "changed",
Token: "changed",
LFSEnabled: true,
FollowSymlinks: true,
DefaultMimeType: "changed",
ForbiddenMimeTypes: []string{"changed"},
},
Database: DatabaseConfig{
Type: "changed",
Conn: "changed",
},
ACME: ACMEConfig{
Email: "changed",
APIEndpoint: "changed",
AcceptTerms: true,
UseRateLimits: true,
EAB_HMAC: "changed",
EAB_KID: "changed",
DNSProvider: "changed",
AccountConfigFile: "changed",
},
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--log-level", "changed",
// Server
"--pages-domain", "changed",
"--raw-domain", "changed",
"--allowed-cors-domains", "changed",
"--blacklisted-paths", "changed",
"--pages-branch", "changed",
"--host", "changed",
"--port", "8443",
"--http-port", "443",
"--enable-http-server",
// Gitea
"--gitea-root", "changed",
"--gitea-api-token", "changed",
"--enable-lfs-support",
"--enable-symlink-support",
"--default-mime-type", "changed",
"--forbidden-mime-types", "changed",
// Database
"--db-type", "changed",
"--db-conn", "changed",
// ACME
"--acme-email", "changed",
"--acme-api-endpoint", "changed",
"--acme-accept-terms",
"--acme-use-rate-limits",
"--acme-eab-hmac", "changed",
"--acme-eab-kid", "changed",
"--dns-provider", "changed",
"--acme-account-config", "changed",
},
)
}
func TestMergeServerConfigShouldAddDefaultBlacklistedPathsToBlacklistedPaths(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &ServerConfig{}
mergeServerConfig(ctx, cfg)
expected := ALWAYS_BLACKLISTED_PATHS
assert.Equal(t, expected, cfg.BlacklistedPaths)
return nil
},
[]string{},
)
}
func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
for range []uint8{0, 1} {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &ServerConfig{
Host: "original",
Port: 8080,
HttpPort: 80,
HttpServerEnabled: false,
MainDomain: "original",
RawDomain: "original",
AllowedCorsDomains: []string{"original"},
BlacklistedPaths: []string{"original"},
}
mergeServerConfig(ctx, cfg)
expectedConfig := &ServerConfig{
Host: "changed",
Port: 8443,
HttpPort: 443,
HttpServerEnabled: true,
MainDomain: "changed",
RawDomain: "changed",
AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}),
BlacklistedPaths: fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)),
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--pages-domain", "changed",
"--raw-domain", "changed",
"--allowed-cors-domains", "changed",
"--blacklisted-paths", "changed",
"--host", "changed",
"--port", "8443",
"--http-port", "443",
"--enable-http-server",
},
)
}
}
func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
type testValuePair struct {
args []string
callback func(*ServerConfig)
}
testValuePairs := []testValuePair{
{args: []string{"--host", "changed"}, callback: func(sc *ServerConfig) { sc.Host = "changed" }},
{args: []string{"--port", "8443"}, callback: func(sc *ServerConfig) { sc.Port = 8443 }},
{args: []string{"--http-port", "443"}, callback: func(sc *ServerConfig) { sc.HttpPort = 443 }},
{args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }},
{args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }},
{args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }},
{args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }},
{args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }},
{args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }},
}
for _, pair := range testValuePairs {
runApp(
t,
func(ctx *cli.Context) error {
cfg := ServerConfig{
Host: "original",
Port: 8080,
HttpPort: 80,
HttpServerEnabled: false,
MainDomain: "original",
RawDomain: "original",
PagesBranches: []string{"original"},
AllowedCorsDomains: []string{"original"},
BlacklistedPaths: []string{"original"},
}
expectedConfig := cfg
pair.callback(&expectedConfig)
expectedConfig.BlacklistedPaths = append(expectedConfig.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...)
expectedConfig.PagesBranches = fixArrayFromCtx(ctx, "pages-branch", expectedConfig.PagesBranches)
expectedConfig.AllowedCorsDomains = fixArrayFromCtx(ctx, "allowed-cors-domains", expectedConfig.AllowedCorsDomains)
expectedConfig.BlacklistedPaths = fixArrayFromCtx(ctx, "blacklisted-paths", expectedConfig.BlacklistedPaths)
mergeServerConfig(ctx, &cfg)
assert.Equal(t, expectedConfig, cfg)
return nil
},
pair.args,
)
}
}
func TestMergeGiteaConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &GiteaConfig{
Root: "original",
Token: "original",
LFSEnabled: false,
FollowSymlinks: false,
DefaultMimeType: "original",
ForbiddenMimeTypes: []string{"original"},
}
mergeGiteaConfig(ctx, cfg)
expectedConfig := &GiteaConfig{
Root: "changed",
Token: "changed",
LFSEnabled: true,
FollowSymlinks: true,
DefaultMimeType: "changed",
ForbiddenMimeTypes: fixArrayFromCtx(ctx, "forbidden-mime-types", []string{"changed"}),
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--gitea-root", "changed",
"--gitea-api-token", "changed",
"--enable-lfs-support",
"--enable-symlink-support",
"--default-mime-type", "changed",
"--forbidden-mime-types", "changed",
},
)
}
func TestMergeGiteaConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
type testValuePair struct {
args []string
callback func(*GiteaConfig)
}
testValuePairs := []testValuePair{
{args: []string{"--gitea-root", "changed"}, callback: func(gc *GiteaConfig) { gc.Root = "changed" }},
{args: []string{"--gitea-api-token", "changed"}, callback: func(gc *GiteaConfig) { gc.Token = "changed" }},
{args: []string{"--enable-lfs-support"}, callback: func(gc *GiteaConfig) { gc.LFSEnabled = true }},
{args: []string{"--enable-symlink-support"}, callback: func(gc *GiteaConfig) { gc.FollowSymlinks = true }},
{args: []string{"--default-mime-type", "changed"}, callback: func(gc *GiteaConfig) { gc.DefaultMimeType = "changed" }},
{args: []string{"--forbidden-mime-types", "changed"}, callback: func(gc *GiteaConfig) { gc.ForbiddenMimeTypes = []string{"changed"} }},
}
for _, pair := range testValuePairs {
runApp(
t,
func(ctx *cli.Context) error {
cfg := GiteaConfig{
Root: "original",
Token: "original",
LFSEnabled: false,
FollowSymlinks: false,
DefaultMimeType: "original",
ForbiddenMimeTypes: []string{"original"},
}
expectedConfig := cfg
pair.callback(&expectedConfig)
mergeGiteaConfig(ctx, &cfg)
expectedConfig.ForbiddenMimeTypes = fixArrayFromCtx(ctx, "forbidden-mime-types", expectedConfig.ForbiddenMimeTypes)
assert.Equal(t, expectedConfig, cfg)
return nil
},
pair.args,
)
}
}
func TestMergeDatabaseConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &DatabaseConfig{
Type: "original",
Conn: "original",
}
mergeDatabaseConfig(ctx, cfg)
expectedConfig := &DatabaseConfig{
Type: "changed",
Conn: "changed",
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--db-type", "changed",
"--db-conn", "changed",
},
)
}
func TestMergeDatabaseConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
type testValuePair struct {
args []string
callback func(*DatabaseConfig)
}
testValuePairs := []testValuePair{
{args: []string{"--db-type", "changed"}, callback: func(gc *DatabaseConfig) { gc.Type = "changed" }},
{args: []string{"--db-conn", "changed"}, callback: func(gc *DatabaseConfig) { gc.Conn = "changed" }},
}
for _, pair := range testValuePairs {
runApp(
t,
func(ctx *cli.Context) error {
cfg := DatabaseConfig{
Type: "original",
Conn: "original",
}
expectedConfig := cfg
pair.callback(&expectedConfig)
mergeDatabaseConfig(ctx, &cfg)
assert.Equal(t, expectedConfig, cfg)
return nil
},
pair.args,
)
}
}
func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) {
runApp(
t,
func(ctx *cli.Context) error {
cfg := &ACMEConfig{
Email: "original",
APIEndpoint: "original",
AcceptTerms: false,
UseRateLimits: false,
EAB_HMAC: "original",
EAB_KID: "original",
DNSProvider: "original",
AccountConfigFile: "original",
}
mergeACMEConfig(ctx, cfg)
expectedConfig := &ACMEConfig{
Email: "changed",
APIEndpoint: "changed",
AcceptTerms: true,
UseRateLimits: true,
EAB_HMAC: "changed",
EAB_KID: "changed",
DNSProvider: "changed",
AccountConfigFile: "changed",
}
assert.Equal(t, expectedConfig, cfg)
return nil
},
[]string{
"--acme-email", "changed",
"--acme-api-endpoint", "changed",
"--acme-accept-terms",
"--acme-use-rate-limits",
"--acme-eab-hmac", "changed",
"--acme-eab-kid", "changed",
"--dns-provider", "changed",
"--acme-account-config", "changed",
},
)
}
func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) {
type testValuePair struct {
args []string
callback func(*ACMEConfig)
}
testValuePairs := []testValuePair{
{args: []string{"--acme-email", "changed"}, callback: func(gc *ACMEConfig) { gc.Email = "changed" }},
{args: []string{"--acme-api-endpoint", "changed"}, callback: func(gc *ACMEConfig) { gc.APIEndpoint = "changed" }},
{args: []string{"--acme-accept-terms"}, callback: func(gc *ACMEConfig) { gc.AcceptTerms = true }},
{args: []string{"--acme-use-rate-limits"}, callback: func(gc *ACMEConfig) { gc.UseRateLimits = true }},
{args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }},
{args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }},
{args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }},
{args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }},
}
for _, pair := range testValuePairs {
runApp(
t,
func(ctx *cli.Context) error {
cfg := ACMEConfig{
Email: "original",
APIEndpoint: "original",
AcceptTerms: false,
UseRateLimits: false,
EAB_HMAC: "original",
EAB_KID: "original",
DNSProvider: "original",
AccountConfigFile: "original",
}
expectedConfig := cfg
pair.callback(&expectedConfig)
mergeACMEConfig(ctx, &cfg)
assert.Equal(t, expectedConfig, cfg)
return nil
},
pair.args,
)
}
}

32
example_config.toml Normal file
View File

@ -0,0 +1,32 @@
logLevel = 'debug'
[server]
host = '[::]'
port = 443
httpPort = 80
httpServerEnabled = true
mainDomain = 'codeberg.page'
rawDomain = 'raw.codeberg.page'
pagesBranches = ["pages"]
allowedCorsDomains = []
blacklistedPaths = []
[gitea]
root = 'https://codeberg.org'
token = 'ASDF1234'
lfsEnabled = true
followSymlinks = true
[database]
type = 'sqlite'
conn = 'certs.sqlite'
[ACME]
email = 'noreply@example.email'
apiEndpoint = 'https://acme-v02.api.letsencrypt.org/directory'
acceptTerms = false
useRateLimits = false
eab_hmac = ''
eab_kid = ''
dnsProvider = ''
accountConfigFile = 'acme-account.json'

8
go.mod
View File

@ -13,9 +13,10 @@ require (
github.com/lib/pq v1.10.7
github.com/mattn/go-sqlite3 v1.14.16
github.com/microcosm-cc/bluemonday v1.0.26
github.com/pelletier/go-toml/v2 v2.1.0
github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad
github.com/rs/zerolog v1.27.0
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.3.0
golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb
xorm.io/xorm v1.3.2
@ -44,6 +45,7 @@ require (
github.com/cloudflare/cloudflare-go v0.20.0 // indirect
github.com/cpu/goacmedns v0.1.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect
github.com/creasty/defaults v1.7.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/davidmz/go-pageant v1.0.2 // indirect
github.com/deepmap/oapi-codegen v1.6.1 // indirect
@ -113,7 +115,7 @@ require (
github.com/softlayer/softlayer-go v1.0.3 // indirect
github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/stretchr/objx v0.3.0 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/transip/gotransip/v6 v6.6.1 // indirect
github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect
@ -135,6 +137,6 @@ require (
gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect
gopkg.in/square/go-jose.v2 v2.6.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
xorm.io/builder v0.3.12 // indirect
)

16
go.sum
View File

@ -131,6 +131,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma
github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA=
github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM=
github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@ -570,6 +572,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc=
github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
@ -680,14 +684,19 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As=
github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
@ -1079,8 +1088,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

17
integration/main_test.go
View File

@ -10,9 +10,10 @@ import (
"testing"
"time"
"codeberg.org/codeberg/pages/cmd"
"github.com/urfave/cli/v2"
cmd "codeberg.org/codeberg/pages/cli"
"codeberg.org/codeberg/pages/server"
)
func TestMain(m *testing.M) {
@ -32,10 +33,7 @@ func TestMain(m *testing.M) {
}
func startServer(ctx context.Context) error {
args := []string{
"--verbose",
"--acme-accept-terms", "true",
}
args := []string{"integration"}
setEnvIfNotSet("ACME_API", "https://acme.mock.directory")
setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory")
setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory")
@ -44,10 +42,15 @@ func startServer(ctx context.Context) error {
setEnvIfNotSet("HTTP_PORT", "8880")
setEnvIfNotSet("ENABLE_HTTP_SERVER", "true")
setEnvIfNotSet("DB_TYPE", "sqlite3")
setEnvIfNotSet("GITEA_ROOT", "https://codeberg.org")
setEnvIfNotSet("LOG_LEVEL", "trace")
setEnvIfNotSet("ENABLE_LFS_SUPPORT", "true")
setEnvIfNotSet("ENABLE_SYMLINK_SUPPORT", "true")
setEnvIfNotSet("ACME_ACCOUNT_CONFIG", "integration/acme-account.json")
app := cli.NewApp()
app.Name = "pages-server"
app.Action = cmd.Serve
app.Action = server.Serve
app.Flags = cmd.ServerFlags
go func() {

20
main.go
View File

@ -1,29 +1,21 @@
package main
import (
"fmt"
"os"
_ "github.com/joho/godotenv/autoload"
"github.com/urfave/cli/v2"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/cmd"
"codeberg.org/codeberg/pages/server/version"
"codeberg.org/codeberg/pages/cli"
"codeberg.org/codeberg/pages/server"
)
func main() {
app := cli.NewApp()
app.Name = "pages-server"
app.Version = version.Version
app.Usage = "pages server"
app.Action = cmd.Serve
app.Flags = cmd.ServerFlags
app.Commands = []*cli.Command{
cmd.Certs,
}
app := cli.CreatePagesApp()
app.Action = server.Serve
if err := app.Run(os.Args); err != nil {
_, _ = fmt.Fprintln(os.Stderr, err)
log.Error().Err(err).Msg("A fatal error occurred")
os.Exit(1)
}
}

26
server/acme/client.go Normal file
View File

@ -0,0 +1,26 @@
package acme
import (
"errors"
"fmt"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
)
var ErrAcmeMissConfig = errors.New("ACME client has wrong config")
func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) {
// check config
if (!cfg.AcceptTerms || cfg.DNSProvider == "") && cfg.APIEndpoint != "https://acme.mock.directory" {
return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig)
}
if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" {
return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig)
} else if cfg.EAB_HMAC == "" && cfg.EAB_KID != "" {
return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig)
}
return certificates.NewAcmeClient(cfg, enableHTTPServer, challengeCache)
}

3
server/cache/interface.go vendored
View File

@ -2,7 +2,8 @@ package cache
import "time"
type SetGetKey interface {
// ICache is an interface that defines how the pages server interacts with the cache.
type ICache interface {
Set(key string, value interface{}, ttl time.Duration) error
Get(key string) (interface{}, bool)
Remove(key string)

2
server/cache/setup.go → server/cache/memory.go vendored
View File

@ -2,6 +2,6 @@ package cache
import "github.com/OrlovEvgeny/go-mcache"
func NewKeyValueCache() SetGetKey {
func NewInMemoryCache() ICache {
return mcache.New()
}

11
server/certificates/acme_client.go
View File

@ -10,6 +10,7 @@ import (
"github.com/reugn/equalizer"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
)
@ -28,8 +29,8 @@ type AcmeClient struct {
acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket
}
func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) {
acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms)
func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*AcmeClient, error) {
acmeConfig, err := setupAcmeConfig(cfg)
if err != nil {
return nil, err
}
@ -54,7 +55,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID,
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
if dnsProvider == "" {
if cfg.DNSProvider == "" {
// using mock server, don't use wildcard certs
err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache})
if err != nil {
@ -62,7 +63,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID,
}
} else {
// use DNS-Challenge https://go-acme.github.io/lego/dns/
provider, err := dns.NewDNSChallengeProviderByName(dnsProvider)
provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider)
if err != nil {
return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err)
}
@ -76,7 +77,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID,
legoClient: acmeClient,
dnsChallengerLegoClient: mainDomainAcmeClient,
acmeUseRateLimits: acmeUseRateLimits,
acmeUseRateLimits: cfg.UseRateLimits,
obtainLocks: sync.Map{},

34
server/certificates/acme_config.go
View File

@ -8,6 +8,7 @@ import (
"fmt"
"os"
"codeberg.org/codeberg/pages/config"
"github.com/go-acme/lego/v4/certcrypto"
"github.com/go-acme/lego/v4/lego"
"github.com/go-acme/lego/v4/registration"
@ -16,21 +17,27 @@ import (
const challengePath = "/.well-known/acme-challenge/"
func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) {
func setupAcmeConfig(cfg config.ACMEConfig) (*lego.Config, error) {
var myAcmeAccount AcmeAccount
var myAcmeConfig *lego.Config
if account, err := os.ReadFile(configFile); err == nil {
log.Info().Msgf("found existing acme account config file '%s'", configFile)
if cfg.AccountConfigFile == "" {
return nil, fmt.Errorf("invalid acme config file: '%s'", cfg.AccountConfigFile)
}
if account, err := os.ReadFile(cfg.AccountConfigFile); err == nil {
log.Info().Msgf("found existing acme account config file '%s'", cfg.AccountConfigFile)
if err := json.Unmarshal(account, &myAcmeAccount); err != nil {
return nil, err
}
myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM))
if err != nil {
return nil, err
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.CADirURL = cfg.APIEndpoint
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
// Validate Config
@ -39,6 +46,7 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate")
return nil, fmt.Errorf("acme config validation failed: %w", err)
}
return myAcmeConfig, nil
} else if !os.IsNotExist(err) {
return nil, err
@ -51,20 +59,20 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
return nil, err
}
myAcmeAccount = AcmeAccount{
Email: acmeMail,
Email: cfg.Email,
Key: privateKey,
KeyPEM: string(certcrypto.PEMEncode(privateKey)),
}
myAcmeConfig = lego.NewConfig(&myAcmeAccount)
myAcmeConfig.CADirURL = acmeAPI
myAcmeConfig.CADirURL = cfg.APIEndpoint
myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048
tempClient, err := lego.NewClient(myAcmeConfig)
if err != nil {
log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only")
} else {
// accept terms & log in to EAB
if acmeEabKID == "" || acmeEabHmac == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms})
if cfg.EAB_KID == "" || cfg.EAB_HMAC == "" {
reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: cfg.AcceptTerms})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
} else {
@ -72,9 +80,9 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
}
} else {
reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{
TermsOfServiceAgreed: acmeAcceptTerms,
Kid: acmeEabKID,
HmacEncoded: acmeEabHmac,
TermsOfServiceAgreed: cfg.AcceptTerms,
Kid: cfg.EAB_KID,
HmacEncoded: cfg.EAB_HMAC,
})
if err != nil {
log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only")
@ -89,8 +97,8 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri
log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits")
select {}
}
log.Info().Msgf("new acme account created. write to config file '%s'", configFile)
err = os.WriteFile(configFile, acmeAccountJSON, 0o600)
log.Info().Msgf("new acme account created. write to config file '%s'", cfg.AccountConfigFile)
err = os.WriteFile(cfg.AccountConfigFile, acmeAccountJSON, 0o600)
if err != nil {
log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits")
select {}

6
server/certificates/cached_challengers.go
View File

@ -15,7 +15,7 @@ import (
)
type AcmeTLSChallengeProvider struct {
challengeCache cache.SetGetKey
challengeCache cache.ICache
}
// make sure AcmeTLSChallengeProvider match Provider interface
@ -31,7 +31,7 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error {
}
type AcmeHTTPChallengeProvider struct {
challengeCache cache.SetGetKey
challengeCache cache.ICache
}
// make sure AcmeHTTPChallengeProvider match Provider interface
@ -46,7 +46,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
return nil
}
func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc {
func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc {
// handle custom-ssl-ports to be added on https redirects
portPart := ""
if sslPort != 443 {

2
server/certificates/certificates.go
View File

@ -31,7 +31,7 @@ func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client,
acmeClient *AcmeClient,
firstDefaultBranch string,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey,
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache,
certDB database.CertDB,
) *tls.Config {
return &tls.Config{

2
server/dns/dns.go
View File

@ -15,7 +15,7 @@ var defaultPagesRepo = "pages"
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) {
// Get CNAME or TXT
var cname string
var err error

4
server/gitea/cache.go
View File

@ -74,7 +74,7 @@ type writeCacheReader struct {
buffer *bytes.Buffer
rileResponse *FileResponse
cacheKey string
cache cache.SetGetKey
cache cache.ICache
hasError bool
}
@ -99,7 +99,7 @@ func (t *writeCacheReader) Close() error {
return t.originalReader.Close()
}
func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser {
if r == nil || cache == nil || cacheKey == "" {
log.Error().Msg("could not create CacheReader")
return nil

28
server/gitea/client.go
View File

@ -16,6 +16,7 @@ import (
"code.gitea.io/sdk/gitea"
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/version"
)
@ -44,7 +45,7 @@ const (
type Client struct {
sdkClient *gitea.Client
responseCache cache.SetGetKey
responseCache cache.ICache
giteaRoot string
@ -55,24 +56,21 @@ type Client struct {
defaultMimeType string
}
func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
rootURL, err := url.Parse(giteaRoot)
func NewClient(cfg config.GiteaConfig, respCache cache.ICache) (*Client, error) {
rootURL, err := url.Parse(cfg.Root)
if err != nil {
return nil, err
}
giteaRoot = strings.Trim(rootURL.String(), "/")
giteaRoot := strings.Trim(rootURL.String(), "/")
stdClient := http.Client{Timeout: 10 * time.Second}
// TODO: pass down
var (
forbiddenMimeTypes map[string]bool
defaultMimeType string
)
if forbiddenMimeTypes == nil {
forbiddenMimeTypes = make(map[string]bool)
forbiddenMimeTypes := make(map[string]bool, len(cfg.ForbiddenMimeTypes))
for _, mimeType := range cfg.ForbiddenMimeTypes {
forbiddenMimeTypes[mimeType] = true
}
defaultMimeType := cfg.DefaultMimeType
if defaultMimeType == "" {
defaultMimeType = "application/octet-stream"
}
@ -80,7 +78,7 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
sdk, err := gitea.NewClient(
giteaRoot,
gitea.SetHTTPClient(&stdClient),
gitea.SetToken(giteaAPIToken),
gitea.SetToken(cfg.Token),
gitea.SetUserAgent("pages-server/"+version.Version),
)
@ -90,8 +88,8 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo
giteaRoot: giteaRoot,
followSymlinks: followSymlinks,
supportLFS: supportLFS,
followSymlinks: cfg.FollowSymlinks,
supportLFS: cfg.LFSEnabled,
forbiddenMimeTypes: forbiddenMimeTypes,
defaultMimeType: defaultMimeType,

30
server/handler/handler.go
View File

@ -6,6 +6,7 @@ import (
"github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/context"
@ -19,11 +20,10 @@ const (
)
// Handler handles a single HTTP request to the web server.
func Handler(mainDomainSuffix, rawDomain string,
func Handler(
cfg config.ServerConfig,
giteaClient *gitea.Client,
blacklistedPaths, allowedCorsDomains []string,
defaultPagesBranches []string,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
@ -39,8 +39,8 @@ func Handler(mainDomainSuffix, rawDomain string,
trimmedHost := ctx.TrimHostPort()
// Add HSTS for RawDomain and MainDomainSuffix
if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
// Add HSTS for RawDomain and MainDomain
if hsts := getHSTSHeader(trimmedHost, cfg.MainDomain, cfg.RawDomain); hsts != "" {
ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
}
@ -62,7 +62,7 @@ func Handler(mainDomainSuffix, rawDomain string,
}
// Block blacklisted paths (like ACME challenges)
for _, blacklistedPath := range blacklistedPaths {
for _, blacklistedPath := range cfg.BlacklistedPaths {
if strings.HasPrefix(ctx.Path(), blacklistedPath) {
html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden)
return
@ -71,7 +71,7 @@ func Handler(mainDomainSuffix, rawDomain string,
// Allow CORS for specified domains
allowCors := false
for _, allowedCorsDomain := range allowedCorsDomains {
for _, allowedCorsDomain := range cfg.AllowedCorsDomains {
if strings.EqualFold(trimmedHost, allowedCorsDomain) {
allowCors = true
break
@ -85,28 +85,28 @@ func Handler(mainDomainSuffix, rawDomain string,
// Prepare request information to Gitea
pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
if cfg.RawDomain != "" && strings.EqualFold(trimmedHost, cfg.RawDomain) {
log.Debug().Msg("raw domain request detected")
handleRaw(log, ctx, giteaClient,
mainDomainSuffix,
cfg.MainDomain,
trimmedHost,
pathElements,
canonicalDomainCache, redirectsCache)
} else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
} else if strings.HasSuffix(trimmedHost, cfg.MainDomain) {
log.Debug().Msg("subdomain request detected")
handleSubDomain(log, ctx, giteaClient,
mainDomainSuffix,
defaultPagesBranches,
cfg.MainDomain,
cfg.PagesBranches,
trimmedHost,
pathElements,
canonicalDomainCache, redirectsCache)
} else {
log.Debug().Msg("custom domain request detected")
handleCustomDomain(log, ctx, giteaClient,
mainDomainSuffix,
cfg.MainDomain,
trimmedHost,
pathElements,
defaultPagesBranches[0],
cfg.PagesBranches[0],
dnsLookupCache, canonicalDomainCache, redirectsCache)
}
}

2
server/handler/handler_custom_domain.go
View File

@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g
trimmedHost string,
pathElements []string,
firstDefaultBranch string,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey,
dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve pages from custom domains
targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache)

2
server/handler/handler_raw_domain.go
View File

@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie
mainDomainSuffix string,
trimmedHost string,
pathElements []string,
canonicalDomainCache, redirectsCache cache.SetGetKey,
canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve raw content from RawDomain
log.Debug().Msg("raw domain")

2
server/handler/handler_sub_domain.go
View File

@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite
defaultPagesBranches []string,
trimmedHost string,
pathElements []string,
canonicalDomainCache, redirectsCache cache.SetGetKey,
canonicalDomainCache, redirectsCache cache.ICache,
) {
// Serve pages from subdomains of MainDomainSuffix
log.Debug().Msg("main domain suffix")

29
server/handler/handler_test.go
View File

@ -6,23 +6,30 @@ import (
"testing"
"time"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
)
func TestHandlerPerformance(t *testing.T) {
giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false)
testHandler := Handler(
"codeberg.page", "raw.codeberg.org",
giteaClient,
[]string{"/.well-known/acme-challenge/"},
[]string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
[]string{"pages"},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
)
cfg := config.GiteaConfig{
Root: "https://codeberg.org",
Token: "",
LFSEnabled: false,
FollowSymlinks: false,
}
giteaClient, _ := gitea.NewClient(cfg, cache.NewInMemoryCache())
serverCfg := config.ServerConfig{
MainDomain: "codeberg.page",
RawDomain: "raw.codeberg.page",
BlacklistedPaths: []string{
"/.well-known/acme-challenge/",
},
AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
PagesBranches: []string{"pages"},
}
testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), cache.NewInMemoryCache())
testCase := func(uri string, status int) {
t.Run(uri, func(t *testing.T) {

4
server/handler/try.go
View File

@ -17,8 +17,8 @@ import (
func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
mainDomainSuffix, trimmedHost string,
options *upstream.Options,
canonicalDomainCache cache.SetGetKey,
redirectsCache cache.SetGetKey,
canonicalDomainCache cache.ICache,
redirectsCache cache.ICache,
) {
// check if a canonical domain exists on a request on MainDomain
if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw {

141
server/startup.go Normal file
View File

@ -0,0 +1,141 @@
package server
import (
"context"
"crypto/tls"
"encoding/json"
"fmt"
"net"
"net/http"
"os"
"strings"
"time"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
"github.com/urfave/cli/v2"
cmd "codeberg.org/codeberg/pages/cli"
"codeberg.org/codeberg/pages/config"
"codeberg.org/codeberg/pages/server/acme"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/certificates"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/handler"
)
// Serve sets up and starts the web server.
func Serve(ctx *cli.Context) error {
// initialize logger with Trace, overridden later with actual level
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(zerolog.TraceLevel)
cfg, err := config.ReadConfig(ctx)
if err != nil {
log.Error().Err(err).Msg("could not read config")
}
config.MergeConfig(ctx, cfg)
// Initialize the logger.
logLevel, err := zerolog.ParseLevel(cfg.LogLevel)
if err != nil {
return err
}
log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel)
foo, _ := json.Marshal(cfg)
log.Trace().RawJSON("config", foo).Msg("starting server with config")
listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port)
listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort)
if cfg.Server.RawDomain != "" {
cfg.Server.AllowedCorsDomains = append(cfg.Server.AllowedCorsDomains, cfg.Server.RawDomain)
}
// Make sure MainDomain has a leading dot
if !strings.HasPrefix(cfg.Server.MainDomain, ".") {
// TODO make this better
cfg.Server.MainDomain = "." + cfg.Server.MainDomain
}
if len(cfg.Server.PagesBranches) == 0 {
return fmt.Errorf("no default branches set (PAGES_BRANCHES)")
}
// Init ssl cert database
certDB, closeFn, err := cmd.OpenCertDB(ctx)
if err != nil {
return err
}
defer closeFn()
keyCache := cache.NewInMemoryCache()
challengeCache := cache.NewInMemoryCache()
// canonicalDomainCache stores canonical domains
canonicalDomainCache := cache.NewInMemoryCache()
// dnsLookupCache stores DNS lookups for custom domains
dnsLookupCache := cache.NewInMemoryCache()
// redirectsCache stores redirects in _redirects files
redirectsCache := cache.NewInMemoryCache()
// clientResponseCache stores responses from the Gitea server
clientResponseCache := cache.NewInMemoryCache()
giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache)
if err != nil {
return fmt.Errorf("could not create new gitea client: %v", err)
}
acmeClient, err := acme.CreateAcmeClient(cfg.ACME, cfg.Server.HttpServerEnabled, challengeCache)
if err != nil {
return err
}
if err := certificates.SetupMainDomainCertificates(cfg.Server.MainDomain, acmeClient, certDB); err != nil {
return err
}
// Create listener for SSL connections
log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress)
listener, err := net.Listen("tcp", listeningSSLAddress)
if err != nil {
return fmt.Errorf("couldn't create listener: %v", err)
}
// Setup listener for SSL connections
listener = tls.NewListener(listener, certificates.TLSConfig(
cfg.Server.MainDomain,
giteaClient,
acmeClient,
cfg.Server.PagesBranches[0],
keyCache, challengeCache, dnsLookupCache, canonicalDomainCache,
certDB,
))
interval := 12 * time.Hour
certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background())
defer cancelCertMaintain()
go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, cfg.Server.MainDomain, certDB)
if cfg.Server.HttpServerEnabled {
// Create handler for http->https redirect and http acme challenges
httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, uint(cfg.Server.Port))
// Create listener for http and start listening
go func() {
log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress)
err := http.ListenAndServe(listeningHTTPAddress, httpHandler)
if err != nil {
log.Error().Err(err).Msg("Couldn't start HTTP server")
}
}()
}
// Create ssl handler based on settings
sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache)
// Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())
return http.Serve(listener, sslHandler)
}

2
server/upstream/domains.go
View File

@ -17,7 +17,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute
const canonicalDomainConfig = ".domains"
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) {
func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) {
// Check if this request is cached.
if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok {
domains := cachedValue.([]string)

4
server/upstream/redirects.go
View File

@ -23,7 +23,7 @@ var redirectsCacheTimeout = 10 * time.Minute
const redirectsConfig = "_redirects"
// getRedirects returns redirects specified in the _redirects file.
func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect {
func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect {
var redirects []Redirect
cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch
@ -63,7 +63,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.S
return redirects
}
func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) {
func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) {
if len(redirects) > 0 {
for _, redirect := range redirects {
reqUrl := ctx.Req.RequestURI

2
server/upstream/upstream.go
View File

@ -53,7 +53,7 @@ type Options struct {
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool {
func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool {
log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
if o.TargetOwner == "" || o.TargetRepo == "" {