diff --git a/.env-dev b/.env-dev new file mode 100644 index 0000000..2286005 --- /dev/null +++ b/.env-dev @@ -0,0 +1,11 @@ +ACME_API=https://acme.mock.directory +ACME_ACCEPT_TERMS=true +PAGES_DOMAIN=localhost.mock.directory +RAW_DOMAIN=raw.localhost.mock.directory +PAGES_BRANCHES=pages,master,main +GITEA_ROOT=https://codeberg.org +PORT=4430 +HTTP_PORT=8880 +ENABLE_HTTP_SERVER=true +LOG_LEVEL=trace +ACME_ACCOUNT_CONFIG=integration/acme-account.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..d2cc8d1 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,26 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch PagesServer", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/main.go", + "args": ["sqlite", "sqlite_unlock_notify", "netgo"], + "envFile": "${workspaceFolder}/.env-dev" + }, + { + "name": "Launch PagesServer integration test", + "type": "go", + "request": "launch", + "mode": "auto", + "program": "${workspaceFolder}/integration/main_test.go", + "args": ["codeberg.org/codeberg/pages/integration/..."], + "buildFlags": ["-tags", "'integration sqlite sqlite_unlock_notify netgo'"] + } + ] +} diff --git a/Justfile b/Justfile index 0b8f814..0bf38a3 100644 --- a/Justfile +++ b/Justfile @@ -4,14 +4,9 @@ TAGS := 'sqlite sqlite_unlock_notify netgo' dev: #!/usr/bin/env bash set -euxo pipefail - export ACME_API=https://acme.mock.directory - export ACME_ACCEPT_TERMS=true - export PAGES_DOMAIN=localhost.mock.directory - export RAW_DOMAIN=raw.localhost.mock.directory - export PORT=4430 - export HTTP_PORT=8880 - export ENABLE_HTTP_SERVER=true - export LOG_LEVEL=trace + set -a # automatically export all variables + source .env-dev + set +a go run -tags '{{TAGS}}' . build: @@ -42,10 +37,10 @@ tool-gofumpt: fi test: - go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -cover -tags '{{TAGS}}' codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/... test-run TEST: - go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/server/... codeberg.org/codeberg/pages/html/ + go test -race -tags '{{TAGS}}' -run "^{{TEST}}$" codeberg.org/codeberg/pages/config/ codeberg.org/codeberg/pages/html/ codeberg.org/codeberg/pages/server/... integration: go test -race -tags 'integration {{TAGS}}' codeberg.org/codeberg/pages/integration/... diff --git a/cmd/certs.go b/cli/certs.go similarity index 92% rename from cmd/certs.go rename to cli/certs.go index 6012b6e..76df7f0 100644 --- a/cmd/certs.go +++ b/cli/certs.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "fmt" @@ -26,7 +26,7 @@ var Certs = &cli.Command{ } func listCerts(ctx *cli.Context) error { - certDB, closeFn, err := openCertDB(ctx) + certDB, closeFn, err := OpenCertDB(ctx) if err != nil { return err } @@ -53,7 +53,7 @@ func removeCert(ctx *cli.Context) error { domains := ctx.Args().Slice() - certDB, closeFn, err := openCertDB(ctx) + certDB, closeFn, err := OpenCertDB(ctx) if err != nil { return err } diff --git a/cmd/flags.go b/cli/flags.go similarity index 80% rename from cmd/flags.go rename to cli/flags.go index 7ac94e6..097cf4f 100644 --- a/cmd/flags.go +++ b/cli/flags.go @@ -1,4 +1,4 @@ -package cmd +package cli import ( "github.com/urfave/cli/v2" @@ -29,26 +29,35 @@ var ( Name: "gitea-root", Usage: "specifies the root URL of the Gitea instance, without a trailing slash.", EnvVars: []string{"GITEA_ROOT"}, - Value: "https://codeberg.org", }, // GiteaApiToken specifies an api token for the Gitea instance &cli.StringFlag{ Name: "gitea-api-token", Usage: "specifies an api token for the Gitea instance", EnvVars: []string{"GITEA_API_TOKEN"}, - Value: "", }, &cli.BoolFlag{ Name: "enable-lfs-support", Usage: "enable lfs support, require gitea >= v1.17.0 as backend", EnvVars: []string{"ENABLE_LFS_SUPPORT"}, - Value: true, + Value: false, }, &cli.BoolFlag{ Name: "enable-symlink-support", Usage: "follow symlinks if enabled, require gitea >= v1.18.0 as backend", EnvVars: []string{"ENABLE_SYMLINK_SUPPORT"}, - Value: true, + Value: false, + }, + &cli.StringFlag{ + Name: "default-mime-type", + Usage: "specifies the default mime type for files that don't have a specific mime type.", + EnvVars: []string{"DEFAULT_MIME_TYPE"}, + Value: "application/octet-stream", + }, + &cli.StringSliceFlag{ + Name: "forbidden-mime-types", + Usage: "specifies the forbidden mime types. Use this flag multiple times for multiple mime types.", + EnvVars: []string{"FORBIDDEN_MIME_TYPES"}, }, // ########################### @@ -61,7 +70,6 @@ var ( Name: "pages-domain", Usage: "specifies the main domain (starting with a dot) for which subdomains shall be served as static pages", EnvVars: []string{"PAGES_DOMAIN"}, - Value: "codeberg.page", }, // RawDomain specifies the domain from which raw repository content shall be served in the following format: // https://{RawDomain}/{owner}/{repo}[/{branch|tag|commit}/{version}]/{filepath...} @@ -70,7 +78,6 @@ var ( Name: "raw-domain", Usage: "specifies the domain from which raw repository content shall be served, not set disable raw content hosting", EnvVars: []string{"RAW_DOMAIN"}, - Value: "raw.codeberg.page", }, // ######################### @@ -98,19 +105,38 @@ var ( Name: "enable-http-server", Usage: "start a http server to redirect to https and respond to http acme challenges", EnvVars: []string{"ENABLE_HTTP_SERVER"}, + Value: false, }, + // Default branches to fetch assets from + &cli.StringSliceFlag{ + Name: "pages-branch", + Usage: "define a branch to fetch assets from. Use this flag multiple times for multiple branches.", + EnvVars: []string{"PAGES_BRANCHES"}, + Value: cli.NewStringSlice("pages"), + }, + + &cli.StringSliceFlag{ + Name: "allowed-cors-domains", + Usage: "specify allowed CORS domains. Use this flag multiple times for multiple domains.", + EnvVars: []string{"ALLOWED_CORS_DOMAINS"}, + }, + &cli.StringSliceFlag{ + Name: "blacklisted-paths", + Usage: "return an error on these url paths.Use this flag multiple times for multiple paths.", + EnvVars: []string{"BLACKLISTED_PATHS"}, + }, + &cli.StringFlag{ Name: "log-level", Value: "warn", Usage: "specify at which log level should be logged. Possible options: info, warn, error, fatal", EnvVars: []string{"LOG_LEVEL"}, }, - // Default branches to fetch assets from - &cli.StringSliceFlag{ - Name: "pages-branch", - Usage: "define a branch to fetch assets from", - EnvVars: []string{"PAGES_BRANCHES"}, - Value: cli.NewStringSlice("pages"), + &cli.StringFlag{ + Name: "config-file", + Usage: "specify the location of the config file", + Aliases: []string{"config"}, + EnvVars: []string{"CONFIG_FILE"}, }, // ############################ diff --git a/cli/setup.go b/cli/setup.go new file mode 100644 index 0000000..6bbff40 --- /dev/null +++ b/cli/setup.go @@ -0,0 +1,39 @@ +package cli + +import ( + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + "codeberg.org/codeberg/pages/server/database" + "codeberg.org/codeberg/pages/server/version" +) + +func CreatePagesApp() *cli.App { + app := cli.NewApp() + app.Name = "pages-server" + app.Version = version.Version + app.Usage = "pages server" + app.Flags = ServerFlags + app.Commands = []*cli.Command{ + Certs, + } + + return app +} + +func OpenCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { + certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) + if err != nil { + return nil, nil, fmt.Errorf("could not connect to database: %w", err) + } + + closeFn = func() { + if err := certDB.Close(); err != nil { + log.Error().Err(err) + } + } + + return certDB, closeFn, nil +} diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 683e859..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,150 +0,0 @@ -package cmd - -import ( - "context" - "crypto/tls" - "fmt" - "net" - "net/http" - "os" - "strings" - "time" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/gitea" - "codeberg.org/codeberg/pages/server/handler" -) - -// AllowedCorsDomains lists the domains for which Cross-Origin Resource Sharing is allowed. -// TODO: make it a flag -var AllowedCorsDomains = []string{ - "fonts.codeberg.org", - "design.codeberg.org", -} - -// BlacklistedPaths specifies forbidden path prefixes for all Codeberg Pages. -// TODO: Make it a flag too -var BlacklistedPaths = []string{ - "/.well-known/acme-challenge/", -} - -// Serve sets up and starts the web server. -func Serve(ctx *cli.Context) error { - // Initialize the logger. - logLevel, err := zerolog.ParseLevel(ctx.String("log-level")) - if err != nil { - return err - } - log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) - - giteaRoot := ctx.String("gitea-root") - giteaAPIToken := ctx.String("gitea-api-token") - rawDomain := ctx.String("raw-domain") - defaultBranches := ctx.StringSlice("pages-branch") - mainDomainSuffix := ctx.String("pages-domain") - listeningHost := ctx.String("host") - listeningSSLPort := ctx.Uint("port") - listeningSSLAddress := fmt.Sprintf("%s:%d", listeningHost, listeningSSLPort) - listeningHTTPAddress := fmt.Sprintf("%s:%d", listeningHost, ctx.Uint("http-port")) - enableHTTPServer := ctx.Bool("enable-http-server") - - allowedCorsDomains := AllowedCorsDomains - if rawDomain != "" { - allowedCorsDomains = append(allowedCorsDomains, rawDomain) - } - - // Make sure MainDomain has a trailing dot - if !strings.HasPrefix(mainDomainSuffix, ".") { - mainDomainSuffix = "." + mainDomainSuffix - } - - if len(defaultBranches) == 0 { - return fmt.Errorf("no default branches set (PAGES_BRANCHES)") - } - - // Init ssl cert database - certDB, closeFn, err := openCertDB(ctx) - if err != nil { - return err - } - defer closeFn() - - keyCache := cache.NewKeyValueCache() - challengeCache := cache.NewKeyValueCache() - // canonicalDomainCache stores canonical domains - canonicalDomainCache := cache.NewKeyValueCache() - // dnsLookupCache stores DNS lookups for custom domains - dnsLookupCache := cache.NewKeyValueCache() - // redirectsCache stores redirects in _redirects files - redirectsCache := cache.NewKeyValueCache() - // clientResponseCache stores responses from the Gitea server - clientResponseCache := cache.NewKeyValueCache() - - giteaClient, err := gitea.NewClient(giteaRoot, giteaAPIToken, clientResponseCache, ctx.Bool("enable-symlink-support"), ctx.Bool("enable-lfs-support")) - if err != nil { - return fmt.Errorf("could not create new gitea client: %v", err) - } - - acmeClient, err := createAcmeClient(ctx, enableHTTPServer, challengeCache) - if err != nil { - return err - } - - if err := certificates.SetupMainDomainCertificates(mainDomainSuffix, acmeClient, certDB); err != nil { - return err - } - - // Create listener for SSL connections - log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) - listener, err := net.Listen("tcp", listeningSSLAddress) - if err != nil { - return fmt.Errorf("couldn't create listener: %v", err) - } - - // Setup listener for SSL connections - listener = tls.NewListener(listener, certificates.TLSConfig(mainDomainSuffix, - giteaClient, - acmeClient, - defaultBranches[0], - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, - certDB)) - - interval := 12 * time.Hour - certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) - defer cancelCertMaintain() - go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, mainDomainSuffix, certDB) - - if enableHTTPServer { - // Create handler for http->https redirect and http acme challenges - httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, listeningSSLPort) - - // Create listener for http and start listening - go func() { - log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) - err := http.ListenAndServe(listeningHTTPAddress, httpHandler) - if err != nil { - log.Panic().Err(err).Msg("Couldn't start HTTP fastServer") - } - }() - } - - // Create ssl handler based on settings - sslHandler := handler.Handler(mainDomainSuffix, rawDomain, - giteaClient, - BlacklistedPaths, allowedCorsDomains, - defaultBranches, - dnsLookupCache, canonicalDomainCache, redirectsCache) - - // Start the ssl listener - log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) - if err := http.Serve(listener, sslHandler); err != nil { - log.Panic().Err(err).Msg("Couldn't start fastServer") - } - - return nil -} diff --git a/cmd/setup.go b/cmd/setup.go deleted file mode 100644 index cde4bc9..0000000 --- a/cmd/setup.go +++ /dev/null @@ -1,64 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - - "codeberg.org/codeberg/pages/server/cache" - "codeberg.org/codeberg/pages/server/certificates" - "codeberg.org/codeberg/pages/server/database" -) - -var ErrAcmeMissConfig = errors.New("ACME client has wrong config") - -func openCertDB(ctx *cli.Context) (certDB database.CertDB, closeFn func(), err error) { - certDB, err = database.NewXormDB(ctx.String("db-type"), ctx.String("db-conn")) - if err != nil { - return nil, nil, fmt.Errorf("could not connect to database: %w", err) - } - - closeFn = func() { - if err := certDB.Close(); err != nil { - log.Error().Err(err) - } - } - - return certDB, closeFn, nil -} - -func createAcmeClient(ctx *cli.Context, enableHTTPServer bool, challengeCache cache.SetGetKey) (*certificates.AcmeClient, error) { - acmeAPI := ctx.String("acme-api-endpoint") - acmeMail := ctx.String("acme-email") - acmeEabHmac := ctx.String("acme-eab-hmac") - acmeEabKID := ctx.String("acme-eab-kid") - acmeAcceptTerms := ctx.Bool("acme-accept-terms") - dnsProvider := ctx.String("dns-provider") - acmeUseRateLimits := ctx.Bool("acme-use-rate-limits") - acmeAccountConf := ctx.String("acme-account-config") - - // check config - if (!acmeAcceptTerms || dnsProvider == "") && acmeAPI != "https://acme.mock.directory" { - return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) - } - if acmeEabHmac != "" && acmeEabKID == "" { - return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) - } else if acmeEabHmac == "" && acmeEabKID != "" { - return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) - } - - return certificates.NewAcmeClient( - acmeAccountConf, - acmeAPI, - acmeMail, - acmeEabHmac, - acmeEabKID, - dnsProvider, - acmeAcceptTerms, - enableHTTPServer, - acmeUseRateLimits, - challengeCache, - ) -} diff --git a/config/assets/test_config.toml b/config/assets/test_config.toml new file mode 100644 index 0000000..6a2f0d0 --- /dev/null +++ b/config/assets/test_config.toml @@ -0,0 +1,33 @@ +logLevel = 'trace' + +[server] +host = '127.0.0.1' +port = 443 +httpPort = 80 +httpServerEnabled = true +mainDomain = 'codeberg.page' +rawDomain = 'raw.codeberg.page' +allowedCorsDomains = ['fonts.codeberg.org', 'design.codeberg.org'] +blacklistedPaths = ['do/not/use'] + +[gitea] +root = 'codeberg.org' +token = 'XXXXXXXX' +lfsEnabled = true +followSymlinks = true +defaultMimeType = "application/wasm" +forbiddenMimeTypes = ["text/html"] + +[database] +type = 'sqlite' +conn = 'certs.sqlite' + +[ACME] +email = 'a@b.c' +apiEndpoint = 'https://example.com' +acceptTerms = false +useRateLimits = true +eab_hmac = 'asdf' +eab_kid = 'qwer' +dnsProvider = 'cloudflare.com' +accountConfigFile = 'nope' diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..6cb972b --- /dev/null +++ b/config/config.go @@ -0,0 +1,46 @@ +package config + +type Config struct { + LogLevel string `default:"warn"` + Server ServerConfig + Gitea GiteaConfig + Database DatabaseConfig + ACME ACMEConfig +} + +type ServerConfig struct { + Host string `default:"[::]"` + Port uint16 `default:"443"` + HttpPort uint16 `default:"80"` + HttpServerEnabled bool `default:"true"` + MainDomain string + RawDomain string + PagesBranches []string + AllowedCorsDomains []string + BlacklistedPaths []string +} + +type GiteaConfig struct { + Root string + Token string + LFSEnabled bool `default:"false"` + FollowSymlinks bool `default:"false"` + DefaultMimeType string `default:"application/octet-stream"` + ForbiddenMimeTypes []string +} + +type DatabaseConfig struct { + Type string `default:"sqlite3"` + Conn string `default:"certs.sqlite"` +} + +type ACMEConfig struct { + Email string + APIEndpoint string `default:"https://acme-v02.api.letsencrypt.org/directory"` + AcceptTerms bool `default:"false"` + UseRateLimits bool `default:"true"` + EAB_HMAC string + EAB_KID string + DNSProvider string + AccountConfigFile string `default:"acme-account.json"` +} diff --git a/config/setup.go b/config/setup.go new file mode 100644 index 0000000..e774084 --- /dev/null +++ b/config/setup.go @@ -0,0 +1,147 @@ +package config + +import ( + "os" + "path" + + "github.com/creasty/defaults" + "github.com/pelletier/go-toml/v2" + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" +) + +var ALWAYS_BLACKLISTED_PATHS = []string{ + "/.well-known/acme-challenge/", +} + +func NewDefaultConfig() Config { + config := Config{} + if err := defaults.Set(&config); err != nil { + panic(err) + } + + // defaults does not support setting arrays from strings + config.Server.PagesBranches = []string{"main", "master", "pages"} + + return config +} + +func ReadConfig(ctx *cli.Context) (*Config, error) { + config := NewDefaultConfig() + // if config is not given as argument return empty config + if !ctx.IsSet("config-file") { + return &config, nil + } + + configFile := path.Clean(ctx.String("config-file")) + + log.Debug().Str("config-file", configFile).Msg("reading config file") + content, err := os.ReadFile(configFile) + if err != nil { + return nil, err + } + + err = toml.Unmarshal(content, &config) + return &config, err +} + +func MergeConfig(ctx *cli.Context, config *Config) { + if ctx.IsSet("log-level") { + config.LogLevel = ctx.String("log-level") + } + + mergeServerConfig(ctx, &config.Server) + mergeGiteaConfig(ctx, &config.Gitea) + mergeDatabaseConfig(ctx, &config.Database) + mergeACMEConfig(ctx, &config.ACME) +} + +func mergeServerConfig(ctx *cli.Context, config *ServerConfig) { + if ctx.IsSet("host") { + config.Host = ctx.String("host") + } + if ctx.IsSet("port") { + config.Port = uint16(ctx.Uint("port")) + } + if ctx.IsSet("http-port") { + config.HttpPort = uint16(ctx.Uint("http-port")) + } + if ctx.IsSet("enable-http-server") { + config.HttpServerEnabled = ctx.Bool("enable-http-server") + } + if ctx.IsSet("pages-domain") { + config.MainDomain = ctx.String("pages-domain") + } + if ctx.IsSet("raw-domain") { + config.RawDomain = ctx.String("raw-domain") + } + if ctx.IsSet("pages-branch") { + config.PagesBranches = ctx.StringSlice("pages-branch") + } + if ctx.IsSet("allowed-cors-domains") { + config.AllowedCorsDomains = ctx.StringSlice("allowed-cors-domains") + } + if ctx.IsSet("blacklisted-paths") { + config.BlacklistedPaths = ctx.StringSlice("blacklisted-paths") + } + + // add the paths that should always be blacklisted + config.BlacklistedPaths = append(config.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) +} + +func mergeGiteaConfig(ctx *cli.Context, config *GiteaConfig) { + if ctx.IsSet("gitea-root") { + config.Root = ctx.String("gitea-root") + } + if ctx.IsSet("gitea-api-token") { + config.Token = ctx.String("gitea-api-token") + } + if ctx.IsSet("enable-lfs-support") { + config.LFSEnabled = ctx.Bool("enable-lfs-support") + } + if ctx.IsSet("enable-symlink-support") { + config.FollowSymlinks = ctx.Bool("enable-symlink-support") + } + if ctx.IsSet("default-mime-type") { + config.DefaultMimeType = ctx.String("default-mime-type") + } + if ctx.IsSet("forbidden-mime-types") { + config.ForbiddenMimeTypes = ctx.StringSlice("forbidden-mime-types") + } +} + +func mergeDatabaseConfig(ctx *cli.Context, config *DatabaseConfig) { + if ctx.IsSet("db-type") { + config.Type = ctx.String("db-type") + } + if ctx.IsSet("db-conn") { + config.Conn = ctx.String("db-conn") + } +} + +func mergeACMEConfig(ctx *cli.Context, config *ACMEConfig) { + if ctx.IsSet("acme-email") { + config.Email = ctx.String("acme-email") + } + if ctx.IsSet("acme-api-endpoint") { + config.APIEndpoint = ctx.String("acme-api-endpoint") + } + if ctx.IsSet("acme-accept-terms") { + config.AcceptTerms = ctx.Bool("acme-accept-terms") + } + if ctx.IsSet("acme-use-rate-limits") { + config.UseRateLimits = ctx.Bool("acme-use-rate-limits") + } + if ctx.IsSet("acme-eab-hmac") { + config.EAB_HMAC = ctx.String("acme-eab-hmac") + } + if ctx.IsSet("acme-eab-kid") { + config.EAB_KID = ctx.String("acme-eab-kid") + } + if ctx.IsSet("dns-provider") { + config.DNSProvider = ctx.String("dns-provider") + } + if ctx.IsSet("acme-account-config") { + config.AccountConfigFile = ctx.String("acme-account-config") + } +} diff --git a/config/setup_test.go b/config/setup_test.go new file mode 100644 index 0000000..a863e2f --- /dev/null +++ b/config/setup_test.go @@ -0,0 +1,596 @@ +package config + +import ( + "context" + "os" + "testing" + + "github.com/pelletier/go-toml/v2" + "github.com/stretchr/testify/assert" + "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" +) + +func runApp(t *testing.T, fn func(*cli.Context) error, args []string) { + app := cmd.CreatePagesApp() + app.Action = fn + + appCtx, appCancel := context.WithCancel(context.Background()) + defer appCancel() + + // os.Args always contains the binary name + args = append([]string{"testing"}, args...) + + err := app.RunContext(appCtx, args) + assert.NoError(t, err) +} + +// fixArrayFromCtx fixes the number of "changed" strings in a string slice according to the number of values in the context. +// This is a workaround because the cli library has a bug where the number of values in the context gets bigger the more tests are run. +func fixArrayFromCtx(ctx *cli.Context, key string, expected []string) []string { + if ctx.IsSet(key) { + ctxSlice := ctx.StringSlice(key) + + if len(ctxSlice) > 1 { + for i := 1; i < len(ctxSlice); i++ { + expected = append([]string{"changed"}, expected...) + } + } + } + + return expected +} + +func readTestConfig() (*Config, error) { + content, err := os.ReadFile("assets/test_config.toml") + if err != nil { + return nil, err + } + + expectedConfig := NewDefaultConfig() + err = toml.Unmarshal(content, &expectedConfig) + if err != nil { + return nil, err + } + + return &expectedConfig, nil +} + +func TestReadConfigShouldReturnEmptyConfigWhenConfigArgEmpty(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + expected := NewDefaultConfig() + assert.Equal(t, &expected, cfg) + + return err + }, + []string{}, + ) +} + +func TestReadConfigShouldReturnConfigFromFileWhenConfigArgPresent(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + if err != nil { + return err + } + + expectedConfig, err := readTestConfig() + if err != nil { + return err + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{"--config-file", "assets/test_config.toml"}, + ) +} + +func TestValuesReadFromConfigFileShouldBeOverwrittenByArgs(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg, err := ReadConfig(ctx) + if err != nil { + return err + } + + MergeConfig(ctx, cfg) + + expectedConfig, err := readTestConfig() + if err != nil { + return err + } + + expectedConfig.LogLevel = "debug" + expectedConfig.Gitea.Root = "not-codeberg.org" + expectedConfig.ACME.AcceptTerms = true + expectedConfig.Server.Host = "172.17.0.2" + expectedConfig.Server.BlacklistedPaths = append(expectedConfig.Server.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--config-file", "assets/test_config.toml", + "--log-level", "debug", + "--gitea-root", "not-codeberg.org", + "--acme-accept-terms", + "--host", "172.17.0.2", + }, + ) +} + +func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &Config{ + LogLevel: "original", + Server: ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + }, + Gitea: GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + }, + Database: DatabaseConfig{ + Type: "original", + Conn: "original", + }, + ACME: ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + }, + } + + MergeConfig(ctx, cfg) + + expectedConfig := &Config{ + LogLevel: "changed", + Server: ServerConfig{ + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + PagesBranches: []string{"changed"}, + AllowedCorsDomains: []string{"changed"}, + BlacklistedPaths: append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...), + }, + Gitea: GiteaConfig{ + Root: "changed", + Token: "changed", + LFSEnabled: true, + FollowSymlinks: true, + DefaultMimeType: "changed", + ForbiddenMimeTypes: []string{"changed"}, + }, + Database: DatabaseConfig{ + Type: "changed", + Conn: "changed", + }, + ACME: ACMEConfig{ + Email: "changed", + APIEndpoint: "changed", + AcceptTerms: true, + UseRateLimits: true, + EAB_HMAC: "changed", + EAB_KID: "changed", + DNSProvider: "changed", + AccountConfigFile: "changed", + }, + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--log-level", "changed", + // Server + "--pages-domain", "changed", + "--raw-domain", "changed", + "--allowed-cors-domains", "changed", + "--blacklisted-paths", "changed", + "--pages-branch", "changed", + "--host", "changed", + "--port", "8443", + "--http-port", "443", + "--enable-http-server", + // Gitea + "--gitea-root", "changed", + "--gitea-api-token", "changed", + "--enable-lfs-support", + "--enable-symlink-support", + "--default-mime-type", "changed", + "--forbidden-mime-types", "changed", + // Database + "--db-type", "changed", + "--db-conn", "changed", + // ACME + "--acme-email", "changed", + "--acme-api-endpoint", "changed", + "--acme-accept-terms", + "--acme-use-rate-limits", + "--acme-eab-hmac", "changed", + "--acme-eab-kid", "changed", + "--dns-provider", "changed", + "--acme-account-config", "changed", + }, + ) +} + +func TestMergeServerConfigShouldAddDefaultBlacklistedPathsToBlacklistedPaths(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ServerConfig{} + mergeServerConfig(ctx, cfg) + + expected := ALWAYS_BLACKLISTED_PATHS + assert.Equal(t, expected, cfg.BlacklistedPaths) + + return nil + }, + []string{}, + ) +} + +func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + for range []uint8{0, 1} { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + } + + mergeServerConfig(ctx, cfg) + + expectedConfig := &ServerConfig{ + Host: "changed", + Port: 8443, + HttpPort: 443, + HttpServerEnabled: true, + MainDomain: "changed", + RawDomain: "changed", + AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}), + BlacklistedPaths: fixArrayFromCtx(ctx, "blacklisted-paths", append([]string{"changed"}, ALWAYS_BLACKLISTED_PATHS...)), + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--pages-domain", "changed", + "--raw-domain", "changed", + "--allowed-cors-domains", "changed", + "--blacklisted-paths", "changed", + "--host", "changed", + "--port", "8443", + "--http-port", "443", + "--enable-http-server", + }, + ) + } +} + +func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*ServerConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--host", "changed"}, callback: func(sc *ServerConfig) { sc.Host = "changed" }}, + {args: []string{"--port", "8443"}, callback: func(sc *ServerConfig) { sc.Port = 8443 }}, + {args: []string{"--http-port", "443"}, callback: func(sc *ServerConfig) { sc.HttpPort = 443 }}, + {args: []string{"--enable-http-server"}, callback: func(sc *ServerConfig) { sc.HttpServerEnabled = true }}, + {args: []string{"--pages-domain", "changed"}, callback: func(sc *ServerConfig) { sc.MainDomain = "changed" }}, + {args: []string{"--raw-domain", "changed"}, callback: func(sc *ServerConfig) { sc.RawDomain = "changed" }}, + {args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }}, + {args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }}, + {args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := ServerConfig{ + Host: "original", + Port: 8080, + HttpPort: 80, + HttpServerEnabled: false, + MainDomain: "original", + RawDomain: "original", + PagesBranches: []string{"original"}, + AllowedCorsDomains: []string{"original"}, + BlacklistedPaths: []string{"original"}, + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + expectedConfig.BlacklistedPaths = append(expectedConfig.BlacklistedPaths, ALWAYS_BLACKLISTED_PATHS...) + + expectedConfig.PagesBranches = fixArrayFromCtx(ctx, "pages-branch", expectedConfig.PagesBranches) + expectedConfig.AllowedCorsDomains = fixArrayFromCtx(ctx, "allowed-cors-domains", expectedConfig.AllowedCorsDomains) + expectedConfig.BlacklistedPaths = fixArrayFromCtx(ctx, "blacklisted-paths", expectedConfig.BlacklistedPaths) + + mergeServerConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeGiteaConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + } + + mergeGiteaConfig(ctx, cfg) + + expectedConfig := &GiteaConfig{ + Root: "changed", + Token: "changed", + LFSEnabled: true, + FollowSymlinks: true, + DefaultMimeType: "changed", + ForbiddenMimeTypes: fixArrayFromCtx(ctx, "forbidden-mime-types", []string{"changed"}), + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--gitea-root", "changed", + "--gitea-api-token", "changed", + "--enable-lfs-support", + "--enable-symlink-support", + "--default-mime-type", "changed", + "--forbidden-mime-types", "changed", + }, + ) +} + +func TestMergeGiteaConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*GiteaConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--gitea-root", "changed"}, callback: func(gc *GiteaConfig) { gc.Root = "changed" }}, + {args: []string{"--gitea-api-token", "changed"}, callback: func(gc *GiteaConfig) { gc.Token = "changed" }}, + {args: []string{"--enable-lfs-support"}, callback: func(gc *GiteaConfig) { gc.LFSEnabled = true }}, + {args: []string{"--enable-symlink-support"}, callback: func(gc *GiteaConfig) { gc.FollowSymlinks = true }}, + {args: []string{"--default-mime-type", "changed"}, callback: func(gc *GiteaConfig) { gc.DefaultMimeType = "changed" }}, + {args: []string{"--forbidden-mime-types", "changed"}, callback: func(gc *GiteaConfig) { gc.ForbiddenMimeTypes = []string{"changed"} }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := GiteaConfig{ + Root: "original", + Token: "original", + LFSEnabled: false, + FollowSymlinks: false, + DefaultMimeType: "original", + ForbiddenMimeTypes: []string{"original"}, + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeGiteaConfig(ctx, &cfg) + + expectedConfig.ForbiddenMimeTypes = fixArrayFromCtx(ctx, "forbidden-mime-types", expectedConfig.ForbiddenMimeTypes) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeDatabaseConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &DatabaseConfig{ + Type: "original", + Conn: "original", + } + + mergeDatabaseConfig(ctx, cfg) + + expectedConfig := &DatabaseConfig{ + Type: "changed", + Conn: "changed", + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--db-type", "changed", + "--db-conn", "changed", + }, + ) +} + +func TestMergeDatabaseConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*DatabaseConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--db-type", "changed"}, callback: func(gc *DatabaseConfig) { gc.Type = "changed" }}, + {args: []string{"--db-conn", "changed"}, callback: func(gc *DatabaseConfig) { gc.Conn = "changed" }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := DatabaseConfig{ + Type: "original", + Conn: "original", + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeDatabaseConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} + +func TestMergeACMEConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T) { + runApp( + t, + func(ctx *cli.Context) error { + cfg := &ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + } + + mergeACMEConfig(ctx, cfg) + + expectedConfig := &ACMEConfig{ + Email: "changed", + APIEndpoint: "changed", + AcceptTerms: true, + UseRateLimits: true, + EAB_HMAC: "changed", + EAB_KID: "changed", + DNSProvider: "changed", + AccountConfigFile: "changed", + } + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + []string{ + "--acme-email", "changed", + "--acme-api-endpoint", "changed", + "--acme-accept-terms", + "--acme-use-rate-limits", + "--acme-eab-hmac", "changed", + "--acme-eab-kid", "changed", + "--dns-provider", "changed", + "--acme-account-config", "changed", + }, + ) +} + +func TestMergeACMEConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgExists(t *testing.T) { + type testValuePair struct { + args []string + callback func(*ACMEConfig) + } + testValuePairs := []testValuePair{ + {args: []string{"--acme-email", "changed"}, callback: func(gc *ACMEConfig) { gc.Email = "changed" }}, + {args: []string{"--acme-api-endpoint", "changed"}, callback: func(gc *ACMEConfig) { gc.APIEndpoint = "changed" }}, + {args: []string{"--acme-accept-terms"}, callback: func(gc *ACMEConfig) { gc.AcceptTerms = true }}, + {args: []string{"--acme-use-rate-limits"}, callback: func(gc *ACMEConfig) { gc.UseRateLimits = true }}, + {args: []string{"--acme-eab-hmac", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_HMAC = "changed" }}, + {args: []string{"--acme-eab-kid", "changed"}, callback: func(gc *ACMEConfig) { gc.EAB_KID = "changed" }}, + {args: []string{"--dns-provider", "changed"}, callback: func(gc *ACMEConfig) { gc.DNSProvider = "changed" }}, + {args: []string{"--acme-account-config", "changed"}, callback: func(gc *ACMEConfig) { gc.AccountConfigFile = "changed" }}, + } + + for _, pair := range testValuePairs { + runApp( + t, + func(ctx *cli.Context) error { + cfg := ACMEConfig{ + Email: "original", + APIEndpoint: "original", + AcceptTerms: false, + UseRateLimits: false, + EAB_HMAC: "original", + EAB_KID: "original", + DNSProvider: "original", + AccountConfigFile: "original", + } + + expectedConfig := cfg + pair.callback(&expectedConfig) + + mergeACMEConfig(ctx, &cfg) + + assert.Equal(t, expectedConfig, cfg) + + return nil + }, + pair.args, + ) + } +} diff --git a/example_config.toml b/example_config.toml new file mode 100644 index 0000000..30e77c4 --- /dev/null +++ b/example_config.toml @@ -0,0 +1,32 @@ +logLevel = 'debug' + +[server] +host = '[::]' +port = 443 +httpPort = 80 +httpServerEnabled = true +mainDomain = 'codeberg.page' +rawDomain = 'raw.codeberg.page' +pagesBranches = ["pages"] +allowedCorsDomains = [] +blacklistedPaths = [] + +[gitea] +root = 'https://codeberg.org' +token = 'ASDF1234' +lfsEnabled = true +followSymlinks = true + +[database] +type = 'sqlite' +conn = 'certs.sqlite' + +[ACME] +email = 'noreply@example.email' +apiEndpoint = 'https://acme-v02.api.letsencrypt.org/directory' +acceptTerms = false +useRateLimits = false +eab_hmac = '' +eab_kid = '' +dnsProvider = '' +accountConfigFile = 'acme-account.json' diff --git a/go.mod b/go.mod index eba292e..47e1e71 100644 --- a/go.mod +++ b/go.mod @@ -13,9 +13,10 @@ require ( github.com/lib/pq v1.10.7 github.com/mattn/go-sqlite3 v1.14.16 github.com/microcosm-cc/bluemonday v1.0.26 + github.com/pelletier/go-toml/v2 v2.1.0 github.com/reugn/equalizer v0.0.0-20210216135016-a959c509d7ad github.com/rs/zerolog v1.27.0 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.8.4 github.com/urfave/cli/v2 v2.3.0 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb xorm.io/xorm v1.3.2 @@ -44,6 +45,7 @@ require ( github.com/cloudflare/cloudflare-go v0.20.0 // indirect github.com/cpu/goacmedns v0.1.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/creasty/defaults v1.7.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidmz/go-pageant v1.0.2 // indirect github.com/deepmap/oapi-codegen v1.6.1 // indirect @@ -113,7 +115,7 @@ require ( github.com/softlayer/softlayer-go v1.0.3 // indirect github.com/softlayer/xmlrpc v0.0.0-20200409220501-5f089df7cb7e // indirect github.com/spf13/cast v1.3.1 // indirect - github.com/stretchr/objx v0.3.0 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/syndtr/goleveldb v1.0.0 // indirect github.com/transip/gotransip/v6 v6.6.1 // indirect github.com/vinyldns/go-vinyldns v0.0.0-20200917153823-148a5f6b8f14 // indirect @@ -135,6 +137,6 @@ require ( gopkg.in/ns1/ns1-go.v2 v2.6.2 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect xorm.io/builder v0.3.12 // indirect ) diff --git a/go.sum b/go.sum index 7ea8b78..1a10599 100644 --- a/go.sum +++ b/go.sum @@ -131,6 +131,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creasty/defaults v1.7.0 h1:eNdqZvc5B509z18lD8yc212CAqJNvfT1Jq6L8WowdBA= +github.com/creasty/defaults v1.7.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -570,6 +572,8 @@ github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTK github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -680,14 +684,19 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.3.0 h1:NGXK3lHquSN08v5vWalVI/L8XU9hdzE/G6xsrze47As= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= @@ -1079,8 +1088,9 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integration/main_test.go b/integration/main_test.go index 6566f78..86fd9d3 100644 --- a/integration/main_test.go +++ b/integration/main_test.go @@ -10,9 +10,10 @@ import ( "testing" "time" - "codeberg.org/codeberg/pages/cmd" - "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/server" ) func TestMain(m *testing.M) { @@ -32,10 +33,7 @@ func TestMain(m *testing.M) { } func startServer(ctx context.Context) error { - args := []string{ - "--verbose", - "--acme-accept-terms", "true", - } + args := []string{"integration"} setEnvIfNotSet("ACME_API", "https://acme.mock.directory") setEnvIfNotSet("PAGES_DOMAIN", "localhost.mock.directory") setEnvIfNotSet("RAW_DOMAIN", "raw.localhost.mock.directory") @@ -44,10 +42,15 @@ func startServer(ctx context.Context) error { setEnvIfNotSet("HTTP_PORT", "8880") setEnvIfNotSet("ENABLE_HTTP_SERVER", "true") setEnvIfNotSet("DB_TYPE", "sqlite3") + setEnvIfNotSet("GITEA_ROOT", "https://codeberg.org") + setEnvIfNotSet("LOG_LEVEL", "trace") + setEnvIfNotSet("ENABLE_LFS_SUPPORT", "true") + setEnvIfNotSet("ENABLE_SYMLINK_SUPPORT", "true") + setEnvIfNotSet("ACME_ACCOUNT_CONFIG", "integration/acme-account.json") app := cli.NewApp() app.Name = "pages-server" - app.Action = cmd.Serve + app.Action = server.Serve app.Flags = cmd.ServerFlags go func() { diff --git a/main.go b/main.go index 6c1d0cc..87e21f3 100644 --- a/main.go +++ b/main.go @@ -1,29 +1,21 @@ package main import ( - "fmt" "os" _ "github.com/joho/godotenv/autoload" - "github.com/urfave/cli/v2" + "github.com/rs/zerolog/log" - "codeberg.org/codeberg/pages/cmd" - "codeberg.org/codeberg/pages/server/version" + "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/server" ) func main() { - app := cli.NewApp() - app.Name = "pages-server" - app.Version = version.Version - app.Usage = "pages server" - app.Action = cmd.Serve - app.Flags = cmd.ServerFlags - app.Commands = []*cli.Command{ - cmd.Certs, - } + app := cli.CreatePagesApp() + app.Action = server.Serve if err := app.Run(os.Args); err != nil { - _, _ = fmt.Fprintln(os.Stderr, err) + log.Error().Err(err).Msg("A fatal error occurred") os.Exit(1) } } diff --git a/server/acme/client.go b/server/acme/client.go new file mode 100644 index 0000000..38e2785 --- /dev/null +++ b/server/acme/client.go @@ -0,0 +1,26 @@ +package acme + +import ( + "errors" + "fmt" + + "codeberg.org/codeberg/pages/config" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" +) + +var ErrAcmeMissConfig = errors.New("ACME client has wrong config") + +func CreateAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*certificates.AcmeClient, error) { + // check config + if (!cfg.AcceptTerms || cfg.DNSProvider == "") && cfg.APIEndpoint != "https://acme.mock.directory" { + return nil, fmt.Errorf("%w: you must set $ACME_ACCEPT_TERMS and $DNS_PROVIDER, unless $ACME_API is set to https://acme.mock.directory", ErrAcmeMissConfig) + } + if cfg.EAB_HMAC != "" && cfg.EAB_KID == "" { + return nil, fmt.Errorf("%w: ACME_EAB_HMAC also needs ACME_EAB_KID to be set", ErrAcmeMissConfig) + } else if cfg.EAB_HMAC == "" && cfg.EAB_KID != "" { + return nil, fmt.Errorf("%w: ACME_EAB_KID also needs ACME_EAB_HMAC to be set", ErrAcmeMissConfig) + } + + return certificates.NewAcmeClient(cfg, enableHTTPServer, challengeCache) +} diff --git a/server/cache/interface.go b/server/cache/interface.go index 2952b29..b3412cc 100644 --- a/server/cache/interface.go +++ b/server/cache/interface.go @@ -2,7 +2,8 @@ package cache import "time" -type SetGetKey interface { +// ICache is an interface that defines how the pages server interacts with the cache. +type ICache interface { Set(key string, value interface{}, ttl time.Duration) error Get(key string) (interface{}, bool) Remove(key string) diff --git a/server/cache/setup.go b/server/cache/memory.go similarity index 69% rename from server/cache/setup.go rename to server/cache/memory.go index a5928b0..093696f 100644 --- a/server/cache/setup.go +++ b/server/cache/memory.go @@ -2,6 +2,6 @@ package cache import "github.com/OrlovEvgeny/go-mcache" -func NewKeyValueCache() SetGetKey { +func NewInMemoryCache() ICache { return mcache.New() } diff --git a/server/certificates/acme_client.go b/server/certificates/acme_client.go index ba83e50..d53e854 100644 --- a/server/certificates/acme_client.go +++ b/server/certificates/acme_client.go @@ -10,6 +10,7 @@ import ( "github.com/reugn/equalizer" "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" ) @@ -28,8 +29,8 @@ type AcmeClient struct { acmeClientCertificateLimitPerUser map[string]*equalizer.TokenBucket } -func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, dnsProvider string, acmeAcceptTerms, enableHTTPServer, acmeUseRateLimits bool, challengeCache cache.SetGetKey) (*AcmeClient, error) { - acmeConfig, err := setupAcmeConfig(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, acmeAcceptTerms) +func NewAcmeClient(cfg config.ACMEConfig, enableHTTPServer bool, challengeCache cache.ICache) (*AcmeClient, error) { + acmeConfig, err := setupAcmeConfig(cfg) if err != nil { return nil, err } @@ -54,7 +55,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, if err != nil { log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { - if dnsProvider == "" { + if cfg.DNSProvider == "" { // using mock server, don't use wildcard certs err := mainDomainAcmeClient.Challenge.SetTLSALPN01Provider(AcmeTLSChallengeProvider{challengeCache}) if err != nil { @@ -62,7 +63,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, } } else { // use DNS-Challenge https://go-acme.github.io/lego/dns/ - provider, err := dns.NewDNSChallengeProviderByName(dnsProvider) + provider, err := dns.NewDNSChallengeProviderByName(cfg.DNSProvider) if err != nil { return nil, fmt.Errorf("can not create DNS Challenge provider: %w", err) } @@ -76,7 +77,7 @@ func NewAcmeClient(acmeAccountConf, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID, legoClient: acmeClient, dnsChallengerLegoClient: mainDomainAcmeClient, - acmeUseRateLimits: acmeUseRateLimits, + acmeUseRateLimits: cfg.UseRateLimits, obtainLocks: sync.Map{}, diff --git a/server/certificates/acme_config.go b/server/certificates/acme_config.go index 12ad7c6..2b5151d 100644 --- a/server/certificates/acme_config.go +++ b/server/certificates/acme_config.go @@ -8,6 +8,7 @@ import ( "fmt" "os" + "codeberg.org/codeberg/pages/config" "github.com/go-acme/lego/v4/certcrypto" "github.com/go-acme/lego/v4/lego" "github.com/go-acme/lego/v4/registration" @@ -16,21 +17,27 @@ import ( const challengePath = "/.well-known/acme-challenge/" -func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcceptTerms bool) (*lego.Config, error) { +func setupAcmeConfig(cfg config.ACMEConfig) (*lego.Config, error) { var myAcmeAccount AcmeAccount var myAcmeConfig *lego.Config - if account, err := os.ReadFile(configFile); err == nil { - log.Info().Msgf("found existing acme account config file '%s'", configFile) + if cfg.AccountConfigFile == "" { + return nil, fmt.Errorf("invalid acme config file: '%s'", cfg.AccountConfigFile) + } + + if account, err := os.ReadFile(cfg.AccountConfigFile); err == nil { + log.Info().Msgf("found existing acme account config file '%s'", cfg.AccountConfigFile) if err := json.Unmarshal(account, &myAcmeAccount); err != nil { return nil, err } + myAcmeAccount.Key, err = certcrypto.ParsePEMPrivateKey([]byte(myAcmeAccount.KeyPEM)) if err != nil { return nil, err } + myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.CADirURL = cfg.APIEndpoint myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 // Validate Config @@ -39,6 +46,7 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri log.Info().Err(err).Msg("config validation failed, you might just delete the config file and let it recreate") return nil, fmt.Errorf("acme config validation failed: %w", err) } + return myAcmeConfig, nil } else if !os.IsNotExist(err) { return nil, err @@ -51,20 +59,20 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri return nil, err } myAcmeAccount = AcmeAccount{ - Email: acmeMail, + Email: cfg.Email, Key: privateKey, KeyPEM: string(certcrypto.PEMEncode(privateKey)), } myAcmeConfig = lego.NewConfig(&myAcmeAccount) - myAcmeConfig.CADirURL = acmeAPI + myAcmeConfig.CADirURL = cfg.APIEndpoint myAcmeConfig.Certificate.KeyType = certcrypto.RSA2048 tempClient, err := lego.NewClient(myAcmeConfig) if err != nil { log.Error().Err(err).Msg("Can't create ACME client, continuing with mock certs only") } else { // accept terms & log in to EAB - if acmeEabKID == "" || acmeEabHmac == "" { - reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: acmeAcceptTerms}) + if cfg.EAB_KID == "" || cfg.EAB_HMAC == "" { + reg, err := tempClient.Registration.Register(registration.RegisterOptions{TermsOfServiceAgreed: cfg.AcceptTerms}) if err != nil { log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") } else { @@ -72,9 +80,9 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri } } else { reg, err := tempClient.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ - TermsOfServiceAgreed: acmeAcceptTerms, - Kid: acmeEabKID, - HmacEncoded: acmeEabHmac, + TermsOfServiceAgreed: cfg.AcceptTerms, + Kid: cfg.EAB_KID, + HmacEncoded: cfg.EAB_HMAC, }) if err != nil { log.Error().Err(err).Msg("Can't register ACME account, continuing with mock certs only") @@ -89,8 +97,8 @@ func setupAcmeConfig(configFile, acmeAPI, acmeMail, acmeEabHmac, acmeEabKID stri log.Error().Err(err).Msg("json.Marshalfailed, waiting for manual restart to avoid rate limits") select {} } - log.Info().Msgf("new acme account created. write to config file '%s'", configFile) - err = os.WriteFile(configFile, acmeAccountJSON, 0o600) + log.Info().Msgf("new acme account created. write to config file '%s'", cfg.AccountConfigFile) + err = os.WriteFile(cfg.AccountConfigFile, acmeAccountJSON, 0o600) if err != nil { log.Error().Err(err).Msg("os.WriteFile failed, waiting for manual restart to avoid rate limits") select {} diff --git a/server/certificates/cached_challengers.go b/server/certificates/cached_challengers.go index bc9ea67..39439fb 100644 --- a/server/certificates/cached_challengers.go +++ b/server/certificates/cached_challengers.go @@ -15,7 +15,7 @@ import ( ) type AcmeTLSChallengeProvider struct { - challengeCache cache.SetGetKey + challengeCache cache.ICache } // make sure AcmeTLSChallengeProvider match Provider interface @@ -31,7 +31,7 @@ func (a AcmeTLSChallengeProvider) CleanUp(domain, _, _ string) error { } type AcmeHTTPChallengeProvider struct { - challengeCache cache.SetGetKey + challengeCache cache.ICache } // make sure AcmeHTTPChallengeProvider match Provider interface @@ -46,7 +46,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error { return nil } -func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey, sslPort uint) http.HandlerFunc { +func SetupHTTPACMEChallengeServer(challengeCache cache.ICache, sslPort uint) http.HandlerFunc { // handle custom-ssl-ports to be added on https redirects portPart := "" if sslPort != 443 { diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go index 4fb99fa..b638755 100644 --- a/server/certificates/certificates.go +++ b/server/certificates/certificates.go @@ -31,7 +31,7 @@ func TLSConfig(mainDomainSuffix string, giteaClient *gitea.Client, acmeClient *AcmeClient, firstDefaultBranch string, - keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.SetGetKey, + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache cache.ICache, certDB database.CertDB, ) *tls.Config { return &tls.Config{ diff --git a/server/dns/dns.go b/server/dns/dns.go index c11b278..970f0c0 100644 --- a/server/dns/dns.go +++ b/server/dns/dns.go @@ -15,7 +15,7 @@ var defaultPagesRepo = "pages" // GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix. // If everything is fine, it returns the target data. -func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) { +func GetTargetFromDNS(domain, mainDomainSuffix, firstDefaultBranch string, dnsLookupCache cache.ICache) (targetOwner, targetRepo, targetBranch string) { // Get CNAME or TXT var cname string var err error diff --git a/server/gitea/cache.go b/server/gitea/cache.go index af61edf..267c3d8 100644 --- a/server/gitea/cache.go +++ b/server/gitea/cache.go @@ -74,7 +74,7 @@ type writeCacheReader struct { buffer *bytes.Buffer rileResponse *FileResponse cacheKey string - cache cache.SetGetKey + cache cache.ICache hasError bool } @@ -99,7 +99,7 @@ func (t *writeCacheReader) Close() error { return t.originalReader.Close() } -func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser { +func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.ICache, cacheKey string) io.ReadCloser { if r == nil || cache == nil || cacheKey == "" { log.Error().Msg("could not create CacheReader") return nil diff --git a/server/gitea/client.go b/server/gitea/client.go index f3bda54..42cf065 100644 --- a/server/gitea/client.go +++ b/server/gitea/client.go @@ -16,6 +16,7 @@ import ( "code.gitea.io/sdk/gitea" "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/version" ) @@ -44,7 +45,7 @@ const ( type Client struct { sdkClient *gitea.Client - responseCache cache.SetGetKey + responseCache cache.ICache giteaRoot string @@ -55,24 +56,21 @@ type Client struct { defaultMimeType string } -func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) { - rootURL, err := url.Parse(giteaRoot) +func NewClient(cfg config.GiteaConfig, respCache cache.ICache) (*Client, error) { + rootURL, err := url.Parse(cfg.Root) if err != nil { return nil, err } - giteaRoot = strings.Trim(rootURL.String(), "/") + giteaRoot := strings.Trim(rootURL.String(), "/") stdClient := http.Client{Timeout: 10 * time.Second} - // TODO: pass down - var ( - forbiddenMimeTypes map[string]bool - defaultMimeType string - ) - - if forbiddenMimeTypes == nil { - forbiddenMimeTypes = make(map[string]bool) + forbiddenMimeTypes := make(map[string]bool, len(cfg.ForbiddenMimeTypes)) + for _, mimeType := range cfg.ForbiddenMimeTypes { + forbiddenMimeTypes[mimeType] = true } + + defaultMimeType := cfg.DefaultMimeType if defaultMimeType == "" { defaultMimeType = "application/octet-stream" } @@ -80,7 +78,7 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo sdk, err := gitea.NewClient( giteaRoot, gitea.SetHTTPClient(&stdClient), - gitea.SetToken(giteaAPIToken), + gitea.SetToken(cfg.Token), gitea.SetUserAgent("pages-server/"+version.Version), ) @@ -90,8 +88,8 @@ func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, follo giteaRoot: giteaRoot, - followSymlinks: followSymlinks, - supportLFS: supportLFS, + followSymlinks: cfg.FollowSymlinks, + supportLFS: cfg.LFSEnabled, forbiddenMimeTypes: forbiddenMimeTypes, defaultMimeType: defaultMimeType, diff --git a/server/handler/handler.go b/server/handler/handler.go index 7da5d39..96788e4 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -6,6 +6,7 @@ import ( "github.com/rs/zerolog/log" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/html" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/context" @@ -19,11 +20,10 @@ const ( ) // Handler handles a single HTTP request to the web server. -func Handler(mainDomainSuffix, rawDomain string, +func Handler( + cfg config.ServerConfig, giteaClient *gitea.Client, - blacklistedPaths, allowedCorsDomains []string, - defaultPagesBranches []string, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { log := log.With().Strs("Handler", []string{req.Host, req.RequestURI}).Logger() @@ -39,8 +39,8 @@ func Handler(mainDomainSuffix, rawDomain string, trimmedHost := ctx.TrimHostPort() - // Add HSTS for RawDomain and MainDomainSuffix - if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" { + // Add HSTS for RawDomain and MainDomain + if hsts := getHSTSHeader(trimmedHost, cfg.MainDomain, cfg.RawDomain); hsts != "" { ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts) } @@ -62,7 +62,7 @@ func Handler(mainDomainSuffix, rawDomain string, } // Block blacklisted paths (like ACME challenges) - for _, blacklistedPath := range blacklistedPaths { + for _, blacklistedPath := range cfg.BlacklistedPaths { if strings.HasPrefix(ctx.Path(), blacklistedPath) { html.ReturnErrorPage(ctx, "requested path is blacklisted", http.StatusForbidden) return @@ -71,7 +71,7 @@ func Handler(mainDomainSuffix, rawDomain string, // Allow CORS for specified domains allowCors := false - for _, allowedCorsDomain := range allowedCorsDomains { + for _, allowedCorsDomain := range cfg.AllowedCorsDomains { if strings.EqualFold(trimmedHost, allowedCorsDomain) { allowCors = true break @@ -85,28 +85,28 @@ func Handler(mainDomainSuffix, rawDomain string, // Prepare request information to Gitea pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/") - if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) { + if cfg.RawDomain != "" && strings.EqualFold(trimmedHost, cfg.RawDomain) { log.Debug().Msg("raw domain request detected") handleRaw(log, ctx, giteaClient, - mainDomainSuffix, + cfg.MainDomain, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) - } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) { + } else if strings.HasSuffix(trimmedHost, cfg.MainDomain) { log.Debug().Msg("subdomain request detected") handleSubDomain(log, ctx, giteaClient, - mainDomainSuffix, - defaultPagesBranches, + cfg.MainDomain, + cfg.PagesBranches, trimmedHost, pathElements, canonicalDomainCache, redirectsCache) } else { log.Debug().Msg("custom domain request detected") handleCustomDomain(log, ctx, giteaClient, - mainDomainSuffix, + cfg.MainDomain, trimmedHost, pathElements, - defaultPagesBranches[0], + cfg.PagesBranches[0], dnsLookupCache, canonicalDomainCache, redirectsCache) } } diff --git a/server/handler/handler_custom_domain.go b/server/handler/handler_custom_domain.go index 8742be4..82953f9 100644 --- a/server/handler/handler_custom_domain.go +++ b/server/handler/handler_custom_domain.go @@ -19,7 +19,7 @@ func handleCustomDomain(log zerolog.Logger, ctx *context.Context, giteaClient *g trimmedHost string, pathElements []string, firstDefaultBranch string, - dnsLookupCache, canonicalDomainCache, redirectsCache cache.SetGetKey, + dnsLookupCache, canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from custom domains targetOwner, targetRepo, targetBranch := dns.GetTargetFromDNS(trimmedHost, mainDomainSuffix, firstDefaultBranch, dnsLookupCache) diff --git a/server/handler/handler_raw_domain.go b/server/handler/handler_raw_domain.go index caa8209..f48e8e4 100644 --- a/server/handler/handler_raw_domain.go +++ b/server/handler/handler_raw_domain.go @@ -19,7 +19,7 @@ func handleRaw(log zerolog.Logger, ctx *context.Context, giteaClient *gitea.Clie mainDomainSuffix string, trimmedHost string, pathElements []string, - canonicalDomainCache, redirectsCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve raw content from RawDomain log.Debug().Msg("raw domain") diff --git a/server/handler/handler_sub_domain.go b/server/handler/handler_sub_domain.go index 6c14393..806fe7f 100644 --- a/server/handler/handler_sub_domain.go +++ b/server/handler/handler_sub_domain.go @@ -21,7 +21,7 @@ func handleSubDomain(log zerolog.Logger, ctx *context.Context, giteaClient *gite defaultPagesBranches []string, trimmedHost string, pathElements []string, - canonicalDomainCache, redirectsCache cache.SetGetKey, + canonicalDomainCache, redirectsCache cache.ICache, ) { // Serve pages from subdomains of MainDomainSuffix log.Debug().Msg("main domain suffix") diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index d04ebda..4cb859a 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -6,23 +6,30 @@ import ( "testing" "time" + "codeberg.org/codeberg/pages/config" "codeberg.org/codeberg/pages/server/cache" "codeberg.org/codeberg/pages/server/gitea" "github.com/rs/zerolog/log" ) func TestHandlerPerformance(t *testing.T) { - giteaClient, _ := gitea.NewClient("https://codeberg.org", "", cache.NewKeyValueCache(), false, false) - testHandler := Handler( - "codeberg.page", "raw.codeberg.org", - giteaClient, - []string{"/.well-known/acme-challenge/"}, - []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, - []string{"pages"}, - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - cache.NewKeyValueCache(), - ) + cfg := config.GiteaConfig{ + Root: "https://codeberg.org", + Token: "", + LFSEnabled: false, + FollowSymlinks: false, + } + giteaClient, _ := gitea.NewClient(cfg, cache.NewInMemoryCache()) + serverCfg := config.ServerConfig{ + MainDomain: "codeberg.page", + RawDomain: "raw.codeberg.page", + BlacklistedPaths: []string{ + "/.well-known/acme-challenge/", + }, + AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, + PagesBranches: []string{"pages"}, + } + testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), cache.NewInMemoryCache()) testCase := func(uri string, status int) { t.Run(uri, func(t *testing.T) { diff --git a/server/handler/try.go b/server/handler/try.go index bb79dce..145b1a9 100644 --- a/server/handler/try.go +++ b/server/handler/try.go @@ -17,8 +17,8 @@ import ( func tryUpstream(ctx *context.Context, giteaClient *gitea.Client, mainDomainSuffix, trimmedHost string, options *upstream.Options, - canonicalDomainCache cache.SetGetKey, - redirectsCache cache.SetGetKey, + canonicalDomainCache cache.ICache, + redirectsCache cache.ICache, ) { // check if a canonical domain exists on a request on MainDomain if strings.HasSuffix(trimmedHost, mainDomainSuffix) && !options.ServeRaw { diff --git a/server/startup.go b/server/startup.go new file mode 100644 index 0000000..ffdabb7 --- /dev/null +++ b/server/startup.go @@ -0,0 +1,141 @@ +package server + +import ( + "context" + "crypto/tls" + "encoding/json" + "fmt" + "net" + "net/http" + "os" + "strings" + "time" + + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + + cmd "codeberg.org/codeberg/pages/cli" + "codeberg.org/codeberg/pages/config" + "codeberg.org/codeberg/pages/server/acme" + "codeberg.org/codeberg/pages/server/cache" + "codeberg.org/codeberg/pages/server/certificates" + "codeberg.org/codeberg/pages/server/gitea" + "codeberg.org/codeberg/pages/server/handler" +) + +// Serve sets up and starts the web server. +func Serve(ctx *cli.Context) error { + // initialize logger with Trace, overridden later with actual level + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(zerolog.TraceLevel) + + cfg, err := config.ReadConfig(ctx) + if err != nil { + log.Error().Err(err).Msg("could not read config") + } + + config.MergeConfig(ctx, cfg) + + // Initialize the logger. + logLevel, err := zerolog.ParseLevel(cfg.LogLevel) + if err != nil { + return err + } + log.Logger = zerolog.New(zerolog.ConsoleWriter{Out: os.Stderr}).With().Timestamp().Logger().Level(logLevel) + + foo, _ := json.Marshal(cfg) + log.Trace().RawJSON("config", foo).Msg("starting server with config") + + listeningSSLAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.Port) + listeningHTTPAddress := fmt.Sprintf("%s:%d", cfg.Server.Host, cfg.Server.HttpPort) + + if cfg.Server.RawDomain != "" { + cfg.Server.AllowedCorsDomains = append(cfg.Server.AllowedCorsDomains, cfg.Server.RawDomain) + } + + // Make sure MainDomain has a leading dot + if !strings.HasPrefix(cfg.Server.MainDomain, ".") { + // TODO make this better + cfg.Server.MainDomain = "." + cfg.Server.MainDomain + } + + if len(cfg.Server.PagesBranches) == 0 { + return fmt.Errorf("no default branches set (PAGES_BRANCHES)") + } + + // Init ssl cert database + certDB, closeFn, err := cmd.OpenCertDB(ctx) + if err != nil { + return err + } + defer closeFn() + + keyCache := cache.NewInMemoryCache() + challengeCache := cache.NewInMemoryCache() + // canonicalDomainCache stores canonical domains + canonicalDomainCache := cache.NewInMemoryCache() + // dnsLookupCache stores DNS lookups for custom domains + dnsLookupCache := cache.NewInMemoryCache() + // redirectsCache stores redirects in _redirects files + redirectsCache := cache.NewInMemoryCache() + // clientResponseCache stores responses from the Gitea server + clientResponseCache := cache.NewInMemoryCache() + + giteaClient, err := gitea.NewClient(cfg.Gitea, clientResponseCache) + if err != nil { + return fmt.Errorf("could not create new gitea client: %v", err) + } + + acmeClient, err := acme.CreateAcmeClient(cfg.ACME, cfg.Server.HttpServerEnabled, challengeCache) + if err != nil { + return err + } + + if err := certificates.SetupMainDomainCertificates(cfg.Server.MainDomain, acmeClient, certDB); err != nil { + return err + } + + // Create listener for SSL connections + log.Info().Msgf("Create TCP listener for SSL on %s", listeningSSLAddress) + listener, err := net.Listen("tcp", listeningSSLAddress) + if err != nil { + return fmt.Errorf("couldn't create listener: %v", err) + } + + // Setup listener for SSL connections + listener = tls.NewListener(listener, certificates.TLSConfig( + cfg.Server.MainDomain, + giteaClient, + acmeClient, + cfg.Server.PagesBranches[0], + keyCache, challengeCache, dnsLookupCache, canonicalDomainCache, + certDB, + )) + + interval := 12 * time.Hour + certMaintainCtx, cancelCertMaintain := context.WithCancel(context.Background()) + defer cancelCertMaintain() + go certificates.MaintainCertDB(certMaintainCtx, interval, acmeClient, cfg.Server.MainDomain, certDB) + + if cfg.Server.HttpServerEnabled { + // Create handler for http->https redirect and http acme challenges + httpHandler := certificates.SetupHTTPACMEChallengeServer(challengeCache, uint(cfg.Server.Port)) + + // Create listener for http and start listening + go func() { + log.Info().Msgf("Start HTTP server listening on %s", listeningHTTPAddress) + err := http.ListenAndServe(listeningHTTPAddress, httpHandler) + if err != nil { + log.Error().Err(err).Msg("Couldn't start HTTP server") + } + }() + } + + // Create ssl handler based on settings + sslHandler := handler.Handler(cfg.Server, giteaClient, dnsLookupCache, canonicalDomainCache, redirectsCache) + + // Start the ssl listener + log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr()) + + return http.Serve(listener, sslHandler) +} diff --git a/server/upstream/domains.go b/server/upstream/domains.go index 230f268..d53d586 100644 --- a/server/upstream/domains.go +++ b/server/upstream/domains.go @@ -17,7 +17,7 @@ var canonicalDomainCacheTimeout = 15 * time.Minute const canonicalDomainConfig = ".domains" // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file). -func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (domain string, valid bool) { +func (o *Options) CheckCanonicalDomain(giteaClient *gitea.Client, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.ICache) (domain string, valid bool) { // Check if this request is cached. if cachedValue, ok := canonicalDomainCache.Get(o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch); ok { domains := cachedValue.([]string) diff --git a/server/upstream/redirects.go b/server/upstream/redirects.go index ab6c971..dd36a84 100644 --- a/server/upstream/redirects.go +++ b/server/upstream/redirects.go @@ -23,7 +23,7 @@ var redirectsCacheTimeout = 10 * time.Minute const redirectsConfig = "_redirects" // getRedirects returns redirects specified in the _redirects file. -func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.SetGetKey) []Redirect { +func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.ICache) []Redirect { var redirects []Redirect cacheKey := o.TargetOwner + "/" + o.TargetRepo + "/" + o.TargetBranch @@ -63,7 +63,7 @@ func (o *Options) getRedirects(giteaClient *gitea.Client, redirectsCache cache.S return redirects } -func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.SetGetKey) (final bool) { +func (o *Options) matchRedirects(ctx *context.Context, giteaClient *gitea.Client, redirects []Redirect, redirectsCache cache.ICache) (final bool) { if len(redirects) > 0 { for _, redirect := range redirects { reqUrl := ctx.Req.RequestURI diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go index 4a85503..07b6ad2 100644 --- a/server/upstream/upstream.go +++ b/server/upstream/upstream.go @@ -53,7 +53,7 @@ type Options struct { } // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context. -func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.SetGetKey) bool { +func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client, redirectsCache cache.ICache) bool { log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger() if o.TargetOwner == "" || o.TargetRepo == "" {