- Sorry, but this page couldn't be found or is inaccessible (%status).
- We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
+ Sorry, but this page couldn't be found or is inaccessible (%status%).
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
diff --git a/html/error.go b/html/error.go
index 325dada..826c42b 100644
--- a/html/error.go
+++ b/html/error.go
@@ -1,24 +1,45 @@
package html
import (
- "bytes"
+ "net/http"
"strconv"
+ "strings"
- "github.com/valyala/fasthttp"
+ "codeberg.org/codeberg/pages/server/context"
)
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
-// with the provided status code.
-func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
- ctx.Response.SetStatusCode(code)
- ctx.Response.Header.SetContentType("text/html; charset=utf-8")
- message := fasthttp.StatusMessage(code)
- if code == fasthttp.StatusMisdirectedRequest {
- message += " - domain not specified in .domains file"
+// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body,
+// with "%status%" and %message% replaced with the provided statusCode and msg
+func ReturnErrorPage(ctx *context.Context, msg string, statusCode int) {
+ ctx.RespWriter.Header().Set("Content-Type", "text/html; charset=utf-8")
+ ctx.RespWriter.WriteHeader(statusCode)
+
+ if msg == "" {
+ msg = errorBody(statusCode)
+ } else {
+ // TODO: use template engine
+ msg = strings.ReplaceAll(strings.ReplaceAll(ErrorPage, "%message%", msg), "%status%", http.StatusText(statusCode))
}
- if code == fasthttp.StatusFailedDependency {
+
+ _, _ = ctx.RespWriter.Write([]byte(msg))
+}
+
+func errorMessage(statusCode int) string {
+ message := http.StatusText(statusCode)
+
+ switch statusCode {
+ case http.StatusMisdirectedRequest:
+ message += " - domain not specified in .domains file"
+ case http.StatusFailedDependency:
message += " - target repo/branch doesn't exist or is private"
}
- // TODO: use template engine?
- ctx.Response.SetBody(bytes.ReplaceAll(NotFoundPage, []byte("%status"), []byte(strconv.Itoa(code)+" "+message)))
+
+ return message
+}
+
+// TODO: use template engine
+func errorBody(statusCode int) string {
+ return strings.ReplaceAll(NotFoundPage,
+ "%status%",
+ strconv.Itoa(statusCode)+" "+errorMessage(statusCode))
}
diff --git a/html/error.html b/html/error.html
new file mode 100644
index 0000000..f1975f7
--- /dev/null
+++ b/html/error.html
@@ -0,0 +1,38 @@
+
+
+
+
+
+ %status%
+
+
+
+
+
+
+
+
+
+
+ %status%!
+
+
+ Sorry, but this page couldn't be served.
+ We got an "%message%"
+ We hope this isn't a problem on our end ;) - Make sure to check the troubleshooting section in the Docs!
+
+
+
+ Static pages made easy - Codeberg Pages
+
+
+
diff --git a/html/html.go b/html/html.go
index d223e15..a76ce59 100644
--- a/html/html.go
+++ b/html/html.go
@@ -3,4 +3,7 @@ package html
import _ "embed"
//go:embed 404.html
-var NotFoundPage []byte
+var NotFoundPage string
+
+//go:embed error.html
+var ErrorPage string
diff --git a/integration/get_test.go b/integration/get_test.go
index 6054e17..8794651 100644
--- a/integration/get_test.go
+++ b/integration/get_test.go
@@ -25,7 +25,7 @@ func TestGetRedirect(t *testing.T) {
t.FailNow()
}
assert.EqualValues(t, "https://www.cabr2.de/", resp.Header.Get("Location"))
- assert.EqualValues(t, 0, getSize(resp.Body))
+ assert.EqualValues(t, `Temporary Redirect.`, strings.TrimSpace(string(getBytes(resp.Body))))
}
func TestGetContent(t *testing.T) {
@@ -44,12 +44,13 @@ func TestGetContent(t *testing.T) {
// specify branch
resp, err = getTestHTTPSClient().Get("https://momar.localhost.mock.directory:4430/pag/@master/")
assert.NoError(t, err)
- if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+ if !assert.NotNil(t, resp) {
t.FailNow()
}
+ assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.True(t, getSize(resp.Body) > 1000)
- assert.Len(t, resp.Header.Get("ETag"), 42)
+ assert.Len(t, resp.Header.Get("ETag"), 44)
// access branch name contains '/'
resp, err = getTestHTTPSClient().Get("https://blumia.localhost.mock.directory:4430/pages-server-integration-tests/@docs~main/")
@@ -59,7 +60,7 @@ func TestGetContent(t *testing.T) {
}
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.True(t, getSize(resp.Body) > 100)
- assert.Len(t, resp.Header.Get("ETag"), 42)
+ assert.Len(t, resp.Header.Get("ETag"), 44)
// TODO: test get of non cachable content (content size > fileCacheSizeLimit)
}
@@ -68,9 +69,10 @@ func TestCustomDomain(t *testing.T) {
log.Println("=== TestCustomDomain ===")
resp, err := getTestHTTPSClient().Get("https://mock-pages.codeberg-test.org:4430/README.md")
assert.NoError(t, err)
- if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+ if !assert.NotNil(t, resp) {
t.FailNow()
}
+ assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "text/markdown; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "106", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 106, getSize(resp.Body))
@@ -81,9 +83,10 @@ func TestGetNotFound(t *testing.T) {
// test custom not found pages
resp, err := getTestHTTPSClient().Get("https://crystal.localhost.mock.directory:4430/pages-404-demo/blah")
assert.NoError(t, err)
- if !assert.EqualValues(t, http.StatusNotFound, resp.StatusCode) {
+ if !assert.NotNil(t, resp) {
t.FailNow()
}
+ assert.EqualValues(t, http.StatusNotFound, resp.StatusCode)
assert.EqualValues(t, "text/html; charset=utf-8", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "37", resp.Header.Get("Content-Length"))
assert.EqualValues(t, 37, getSize(resp.Body))
@@ -94,9 +97,10 @@ func TestFollowSymlink(t *testing.T) {
resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/link")
assert.NoError(t, err)
- if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+ if !assert.NotNil(t, resp) {
t.FailNow()
}
+ assert.EqualValues(t, http.StatusOK, resp.StatusCode)
assert.EqualValues(t, "application/octet-stream", resp.Header.Get("Content-Type"))
assert.EqualValues(t, "4", resp.Header.Get("Content-Length"))
body := getBytes(resp.Body)
@@ -109,14 +113,27 @@ func TestLFSSupport(t *testing.T) {
resp, err := getTestHTTPSClient().Get("https://6543.localhost.mock.directory:4430/tests_for_pages-server/@main/lfs.txt")
assert.NoError(t, err)
- if !assert.EqualValues(t, http.StatusOK, resp.StatusCode) {
+ if !assert.NotNil(t, resp) {
t.FailNow()
}
+ assert.EqualValues(t, http.StatusOK, resp.StatusCode)
body := strings.TrimSpace(string(getBytes(resp.Body)))
assert.EqualValues(t, 12, len(body))
assert.EqualValues(t, "actual value", body)
}
+func TestGetOptions(t *testing.T) {
+ log.Println("=== TestGetOptions ===")
+ req, _ := http.NewRequest(http.MethodOptions, "https://mock-pages.codeberg-test.org:4430/README.md", nil)
+ resp, err := getTestHTTPSClient().Do(req)
+ assert.NoError(t, err)
+ if !assert.NotNil(t, resp) {
+ t.FailNow()
+ }
+ assert.EqualValues(t, http.StatusNoContent, resp.StatusCode)
+ assert.EqualValues(t, "GET, HEAD, OPTIONS", resp.Header.Get("Allow"))
+}
+
func getTestHTTPSClient() *http.Client {
cookieJar, _ := cookiejar.New(nil)
return &http.Client{
diff --git a/server/certificates/certificates.go b/server/certificates/certificates.go
index 2f59fb4..429ab23 100644
--- a/server/certificates/certificates.go
+++ b/server/certificates/certificates.go
@@ -36,7 +36,7 @@ import (
)
// TLSConfig returns the configuration for generating, serving and cleaning up Let's Encrypt certificates.
-func TLSConfig(mainDomainSuffix []byte,
+func TLSConfig(mainDomainSuffix string,
giteaClient *gitea.Client,
dnsProvider string,
acmeUseRateLimits bool,
@@ -47,7 +47,6 @@ func TLSConfig(mainDomainSuffix []byte,
// check DNS name & get certificate from Let's Encrypt
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
sni := strings.ToLower(strings.TrimSpace(info.ServerName))
- sniBytes := []byte(sni)
if len(sni) < 1 {
return nil, errors.New("missing sni")
}
@@ -69,23 +68,20 @@ func TLSConfig(mainDomainSuffix []byte,
}
targetOwner := ""
- if bytes.HasSuffix(sniBytes, mainDomainSuffix) || bytes.Equal(sniBytes, mainDomainSuffix[1:]) {
+ if strings.HasSuffix(sni, mainDomainSuffix) || strings.EqualFold(sni, mainDomainSuffix[1:]) {
// deliver default certificate for the main domain (*.codeberg.page)
- sniBytes = mainDomainSuffix
- sni = string(sniBytes)
+ sni = mainDomainSuffix
} else {
var targetRepo, targetBranch string
- targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, string(mainDomainSuffix), dnsLookupCache)
+ targetOwner, targetRepo, targetBranch = dnsutils.GetTargetFromDNS(sni, mainDomainSuffix, dnsLookupCache)
if targetOwner == "" {
// DNS not set up, return main certificate to redirect to the docs
- sniBytes = mainDomainSuffix
- sni = string(sniBytes)
+ sni = mainDomainSuffix
} else {
_, _ = targetRepo, targetBranch
- _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, string(mainDomainSuffix), canonicalDomainCache)
+ _, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, sni, mainDomainSuffix, canonicalDomainCache)
if !valid {
- sniBytes = mainDomainSuffix
- sni = string(sniBytes)
+ sni = mainDomainSuffix
}
}
}
@@ -98,9 +94,9 @@ func TLSConfig(mainDomainSuffix []byte,
var tlsCertificate tls.Certificate
var err error
var ok bool
- if tlsCertificate, ok = retrieveCertFromDB(sniBytes, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
+ if tlsCertificate, ok = retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider, acmeUseRateLimits, certDB); !ok {
// request a new certificate
- if bytes.Equal(sniBytes, mainDomainSuffix) {
+ if strings.EqualFold(sni, mainDomainSuffix) {
return nil, errors.New("won't request certificate for main domain, something really bad has happened")
}
@@ -192,7 +188,7 @@ func (a AcmeHTTPChallengeProvider) CleanUp(domain, token, _ string) error {
return nil
}
-func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
+func retrieveCertFromDB(sni, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) (tls.Certificate, bool) {
// parse certificate from database
res, err := certDB.Get(string(sni))
if err != nil {
@@ -208,7 +204,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
}
// TODO: document & put into own function
- if !bytes.Equal(sni, mainDomainSuffix) {
+ if !strings.EqualFold(sni, mainDomainSuffix) {
tlsCertificate.Leaf, err = x509.ParseCertificate(tlsCertificate.Certificate[0])
if err != nil {
panic(err)
@@ -239,7 +235,7 @@ func retrieveCertFromDB(sni, mainDomainSuffix []byte, dnsProvider string, acmeUs
var obtainLocks = sync.Map{}
-func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider string, mainDomainSuffix []byte, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
+func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Resource, user, dnsProvider, mainDomainSuffix string, acmeUseRateLimits bool, keyDatabase database.CertDB) (tls.Certificate, error) {
name := strings.TrimPrefix(domains[0], "*")
if dnsProvider == "" && len(domains[0]) > 0 && domains[0][0] == '*' {
domains = domains[1:]
@@ -252,7 +248,7 @@ func obtainCert(acmeClient *lego.Client, domains []string, renew *certificate.Re
time.Sleep(100 * time.Millisecond)
_, working = obtainLocks.Load(name)
}
- cert, ok := retrieveCertFromDB([]byte(name), mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
+ cert, ok := retrieveCertFromDB(name, mainDomainSuffix, dnsProvider, acmeUseRateLimits, keyDatabase)
if !ok {
return tls.Certificate{}, errors.New("certificate failed in synchronous request")
}
@@ -405,7 +401,7 @@ func SetupAcmeConfig(acmeAPI, acmeMail, acmeEabHmac, acmeEabKID string, acmeAcce
return myAcmeConfig, nil
}
-func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
+func SetupCertificates(mainDomainSuffix, dnsProvider string, acmeConfig *lego.Config, acmeUseRateLimits, enableHTTPServer bool, challengeCache cache.SetGetKey, certDB database.CertDB) error {
// getting main cert before ACME account so that we can fail here without hitting rate limits
mainCertBytes, err := certDB.Get(string(mainDomainSuffix))
if err != nil {
@@ -460,7 +456,7 @@ func SetupCertificates(mainDomainSuffix []byte, dnsProvider string, acmeConfig *
return nil
}
-func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix []byte, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
+func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffix, dnsProvider string, acmeUseRateLimits bool, certDB database.CertDB) {
for {
// clean up expired certs
now := time.Now()
@@ -468,7 +464,7 @@ func MaintainCertDB(ctx context.Context, interval time.Duration, mainDomainSuffi
keyDatabaseIterator := certDB.Items()
key, resBytes, err := keyDatabaseIterator.Next()
for err == nil {
- if !bytes.Equal(key, mainDomainSuffix) {
+ if !strings.EqualFold(string(key), mainDomainSuffix) {
resGob := bytes.NewBuffer(resBytes)
resDec := gob.NewDecoder(resGob)
res := &certificate.Resource{}
diff --git a/server/context/context.go b/server/context/context.go
new file mode 100644
index 0000000..be01df0
--- /dev/null
+++ b/server/context/context.go
@@ -0,0 +1,62 @@
+package context
+
+import (
+ stdContext "context"
+ "net/http"
+)
+
+type Context struct {
+ RespWriter http.ResponseWriter
+ Req *http.Request
+ StatusCode int
+}
+
+func New(w http.ResponseWriter, r *http.Request) *Context {
+ return &Context{
+ RespWriter: w,
+ Req: r,
+ StatusCode: http.StatusOK,
+ }
+}
+
+func (c *Context) Context() stdContext.Context {
+ if c.Req != nil {
+ return c.Req.Context()
+ }
+ return stdContext.Background()
+}
+
+func (c *Context) Response() *http.Response {
+ if c.Req != nil && c.Req.Response != nil {
+ return c.Req.Response
+ }
+ return nil
+}
+
+func (c *Context) String(raw string, status ...int) {
+ code := http.StatusOK
+ if len(status) != 0 {
+ code = status[0]
+ }
+ c.RespWriter.WriteHeader(code)
+ _, _ = c.RespWriter.Write([]byte(raw))
+}
+
+func (c *Context) IsMethod(m string) bool {
+ return c.Req.Method == m
+}
+
+func (c *Context) Redirect(uri string, statusCode int) {
+ http.Redirect(c.RespWriter, c.Req, uri, statusCode)
+}
+
+// Path returns requested path.
+//
+// The returned bytes are valid until your request handler returns.
+func (c *Context) Path() string {
+ return c.Req.URL.Path
+}
+
+func (c *Context) Host() string {
+ return c.Req.URL.Host
+}
diff --git a/server/database/mock.go b/server/database/mock.go
index e6c1b5a..dfe2316 100644
--- a/server/database/mock.go
+++ b/server/database/mock.go
@@ -28,7 +28,7 @@ func (p tmpDB) Put(name string, cert *certificate.Resource) error {
func (p tmpDB) Get(name string) (*certificate.Resource, error) {
cert, has := p.intern.Get(name)
if !has {
- return nil, fmt.Errorf("cert for '%s' not found", name)
+ return nil, fmt.Errorf("cert for %q not found", name)
}
return cert.(*certificate.Resource), nil
}
diff --git a/server/dns/const.go b/server/dns/const.go
deleted file mode 100644
index bb2413b..0000000
--- a/server/dns/const.go
+++ /dev/null
@@ -1,6 +0,0 @@
-package dns
-
-import "time"
-
-// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
-var lookupCacheTimeout = 15 * time.Minute
diff --git a/server/dns/dns.go b/server/dns/dns.go
index dc759b0..818e29a 100644
--- a/server/dns/dns.go
+++ b/server/dns/dns.go
@@ -3,10 +3,14 @@ package dns
import (
"net"
"strings"
+ "time"
"codeberg.org/codeberg/pages/server/cache"
)
+// lookupCacheTimeout specifies the timeout for the DNS lookup cache.
+var lookupCacheTimeout = 15 * time.Minute
+
// GetTargetFromDNS searches for CNAME or TXT entries on the request domain ending with MainDomainSuffix.
// If everything is fine, it returns the target data.
func GetTargetFromDNS(domain, mainDomainSuffix string, dnsLookupCache cache.SetGetKey) (targetOwner, targetRepo, targetBranch string) {
diff --git a/server/gitea/cache.go b/server/gitea/cache.go
index 932ff3c..b11a370 100644
--- a/server/gitea/cache.go
+++ b/server/gitea/cache.go
@@ -1,12 +1,116 @@
package gitea
+import (
+ "bytes"
+ "fmt"
+ "io"
+ "net/http"
+ "time"
+
+ "github.com/rs/zerolog/log"
+
+ "codeberg.org/codeberg/pages/server/cache"
+)
+
+const (
+ // defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
+ defaultBranchCacheTimeout = 15 * time.Minute
+
+ // branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
+ // than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
+ // picked up faster, while still allowing the content to be cached longer if nothing changes.
+ branchExistenceCacheTimeout = 5 * time.Minute
+
+ // fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
+ // on your available memory.
+ // TODO: move as option into cache interface
+ fileCacheTimeout = 5 * time.Minute
+
+ // fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
+ fileCacheSizeLimit = int64(1000 * 1000)
+)
+
type FileResponse struct {
- Exists bool
- ETag []byte
- MimeType string
- Body []byte
+ Exists bool
+ IsSymlink bool
+ ETag string
+ MimeType string
+ Body []byte
}
func (f FileResponse) IsEmpty() bool {
return len(f.Body) != 0
}
+
+func (f FileResponse) createHttpResponse(cacheKey string) (http.Header, int) {
+ header := make(http.Header)
+ var statusCode int
+
+ if f.Exists {
+ statusCode = http.StatusOK
+ } else {
+ statusCode = http.StatusNotFound
+ }
+
+ if f.IsSymlink {
+ header.Set(giteaObjectTypeHeader, objTypeSymlink)
+ }
+ header.Set(ETagHeader, f.ETag)
+ header.Set(ContentTypeHeader, f.MimeType)
+ header.Set(ContentLengthHeader, fmt.Sprintf("%d", len(f.Body)))
+ header.Set(PagesCacheIndicatorHeader, "true")
+
+ log.Trace().Msgf("fileCache for %q used", cacheKey)
+ return header, statusCode
+}
+
+type BranchTimestamp struct {
+ Branch string
+ Timestamp time.Time
+ notFound bool
+}
+
+type writeCacheReader struct {
+ originalReader io.ReadCloser
+ buffer *bytes.Buffer
+ rileResponse *FileResponse
+ cacheKey string
+ cache cache.SetGetKey
+ hasError bool
+}
+
+func (t *writeCacheReader) Read(p []byte) (n int, err error) {
+ n, err = t.originalReader.Read(p)
+ if err != nil {
+ log.Trace().Err(err).Msgf("[cache] original reader for %q has returned an error", t.cacheKey)
+ t.hasError = true
+ } else if n > 0 {
+ _, _ = t.buffer.Write(p[:n])
+ }
+ return
+}
+
+func (t *writeCacheReader) Close() error {
+ if !t.hasError {
+ fc := *t.rileResponse
+ fc.Body = t.buffer.Bytes()
+ _ = t.cache.Set(t.cacheKey, fc, fileCacheTimeout)
+ }
+ log.Trace().Msgf("cacheReader for %q saved=%t closed", t.cacheKey, !t.hasError)
+ return t.originalReader.Close()
+}
+
+func (f FileResponse) CreateCacheReader(r io.ReadCloser, cache cache.SetGetKey, cacheKey string) io.ReadCloser {
+ if r == nil || cache == nil || cacheKey == "" {
+ log.Error().Msg("could not create CacheReader")
+ return nil
+ }
+
+ return &writeCacheReader{
+ originalReader: r,
+ buffer: bytes.NewBuffer(make([]byte, 0)),
+ rileResponse: &f,
+ cache: cache,
+ cacheKey: cacheKey,
+ }
+}
diff --git a/server/gitea/client.go b/server/gitea/client.go
index 16cba84..c63ee21 100644
--- a/server/gitea/client.go
+++ b/server/gitea/client.go
@@ -1,142 +1,276 @@
package gitea
import (
+ "bytes"
"errors"
"fmt"
+ "io"
+ "mime"
+ "net/http"
"net/url"
+ "path"
+ "strconv"
"strings"
"time"
+ "code.gitea.io/sdk/gitea"
"github.com/rs/zerolog/log"
- "github.com/valyala/fasthttp"
- "github.com/valyala/fastjson"
-)
-const (
- giteaAPIRepos = "/api/v1/repos/"
- giteaObjectTypeHeader = "X-Gitea-Object-Type"
+ "codeberg.org/codeberg/pages/server/cache"
)
var ErrorNotFound = errors.New("not found")
+const (
+ // cache key prefixe
+ branchTimestampCacheKeyPrefix = "branchTime"
+ defaultBranchCacheKeyPrefix = "defaultBranch"
+ rawContentCacheKeyPrefix = "rawContent"
+
+ // pages server
+ PagesCacheIndicatorHeader = "X-Pages-Cache"
+ symlinkReadLimit = 10000
+
+ // gitea
+ giteaObjectTypeHeader = "X-Gitea-Object-Type"
+ objTypeSymlink = "symlink"
+
+ // std
+ ETagHeader = "ETag"
+ ContentTypeHeader = "Content-Type"
+ ContentLengthHeader = "Content-Length"
+)
+
type Client struct {
- giteaRoot string
- giteaAPIToken string
- fastClient *fasthttp.Client
- infoTimeout time.Duration
- contentTimeout time.Duration
+ sdkClient *gitea.Client
+ responseCache cache.SetGetKey
followSymlinks bool
supportLFS bool
+
+ forbiddenMimeTypes map[string]bool
+ defaultMimeType string
}
-// TODO: once golang v1.19 is min requirement, we can switch to 'JoinPath()' of 'net/url' package
-func joinURL(baseURL string, paths ...string) string {
- p := make([]string, 0, len(paths))
- for i := range paths {
- path := strings.TrimSpace(paths[i])
- path = strings.Trim(path, "/")
- if len(path) != 0 {
- p = append(p, path)
- }
- }
-
- return baseURL + "/" + strings.Join(p, "/")
-}
-
-func NewClient(giteaRoot, giteaAPIToken string, followSymlinks, supportLFS bool) (*Client, error) {
+func NewClient(giteaRoot, giteaAPIToken string, respCache cache.SetGetKey, followSymlinks, supportLFS bool) (*Client, error) {
rootURL, err := url.Parse(giteaRoot)
+ if err != nil {
+ return nil, err
+ }
giteaRoot = strings.Trim(rootURL.String(), "/")
+ stdClient := http.Client{Timeout: 10 * time.Second}
+
+ // TODO: pass down
+ var (
+ forbiddenMimeTypes map[string]bool
+ defaultMimeType string
+ )
+
+ if forbiddenMimeTypes == nil {
+ forbiddenMimeTypes = make(map[string]bool)
+ }
+ if defaultMimeType == "" {
+ defaultMimeType = "application/octet-stream"
+ }
+
+ sdk, err := gitea.NewClient(giteaRoot, gitea.SetHTTPClient(&stdClient), gitea.SetToken(giteaAPIToken))
return &Client{
- giteaRoot: giteaRoot,
- giteaAPIToken: giteaAPIToken,
- infoTimeout: 5 * time.Second,
- contentTimeout: 10 * time.Second,
- fastClient: getFastHTTPClient(),
+ sdkClient: sdk,
+ responseCache: respCache,
followSymlinks: followSymlinks,
supportLFS: supportLFS,
+
+ forbiddenMimeTypes: forbiddenMimeTypes,
+ defaultMimeType: defaultMimeType,
}, err
}
func (client *Client) GiteaRawContent(targetOwner, targetRepo, ref, resource string) ([]byte, error) {
- resp, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
+ reader, _, _, err := client.ServeRawContent(targetOwner, targetRepo, ref, resource)
if err != nil {
return nil, err
}
- return resp.Body(), nil
+ defer reader.Close()
+ return io.ReadAll(reader)
}
-func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (*fasthttp.Response, error) {
- var apiURL string
- if client.supportLFS {
- apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "media", resource+"?ref="+url.QueryEscape(ref))
- } else {
- apiURL = joinURL(client.giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref))
- }
- resp, err := client.do(client.contentTimeout, apiURL)
- if err != nil {
- return nil, err
- }
+func (client *Client) ServeRawContent(targetOwner, targetRepo, ref, resource string) (io.ReadCloser, http.Header, int, error) {
+ cacheKey := fmt.Sprintf("%s/%s/%s|%s|%s", rawContentCacheKeyPrefix, targetOwner, targetRepo, ref, resource)
+ log := log.With().Str("cache_key", cacheKey).Logger()
- if err != nil {
- return nil, err
- }
-
- switch resp.StatusCode() {
- case fasthttp.StatusOK:
- objType := string(resp.Header.Peek(giteaObjectTypeHeader))
- log.Trace().Msgf("server raw content object: %s", objType)
- if client.followSymlinks && objType == "symlink" {
- // TODO: limit to 1000 chars if we switched to std
- linkDest := strings.TrimSpace(string(resp.Body()))
- log.Debug().Msgf("follow symlink from '%s' to '%s'", resource, linkDest)
- return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+ // handle if cache entry exist
+ if cache, ok := client.responseCache.Get(cacheKey); ok {
+ cache := cache.(FileResponse)
+ cachedHeader, cachedStatusCode := cache.createHttpResponse(cacheKey)
+ // TODO: check against some timestamp missmatch?!?
+ if cache.Exists {
+ if cache.IsSymlink {
+ linkDest := string(cache.Body)
+ log.Debug().Msgf("[cache] follow symlink from %q to %q", resource, linkDest)
+ return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+ } else {
+ log.Debug().Msg("[cache] return bytes")
+ return io.NopCloser(bytes.NewReader(cache.Body)), cachedHeader, cachedStatusCode, nil
+ }
+ } else {
+ return nil, cachedHeader, cachedStatusCode, ErrorNotFound
}
-
- return resp, nil
-
- case fasthttp.StatusNotFound:
- return nil, ErrorNotFound
-
- default:
- return nil, fmt.Errorf("unexpected status code '%d'", resp.StatusCode())
}
+
+ // not in cache, open reader via gitea api
+ reader, resp, err := client.sdkClient.GetFileReader(targetOwner, targetRepo, ref, resource, client.supportLFS)
+ if resp != nil {
+ switch resp.StatusCode {
+ case http.StatusOK:
+ // first handle symlinks
+ {
+ objType := resp.Header.Get(giteaObjectTypeHeader)
+ log.Trace().Msgf("server raw content object %q", objType)
+ if client.followSymlinks && objType == objTypeSymlink {
+ defer reader.Close()
+ // read limited chars for symlink
+ linkDestBytes, err := io.ReadAll(io.LimitReader(reader, symlinkReadLimit))
+ if err != nil {
+ return nil, nil, http.StatusInternalServerError, err
+ }
+ linkDest := strings.TrimSpace(string(linkDestBytes))
+
+ // we store symlink not content to reduce duplicates in cache
+ if err := client.responseCache.Set(cacheKey, FileResponse{
+ Exists: true,
+ IsSymlink: true,
+ Body: []byte(linkDest),
+ ETag: resp.Header.Get(ETagHeader),
+ }, fileCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+
+ log.Debug().Msgf("follow symlink from %q to %q", resource, linkDest)
+ return client.ServeRawContent(targetOwner, targetRepo, ref, linkDest)
+ }
+ }
+
+ // now we are sure it's content so set the MIME type
+ mimeType := client.getMimeTypeByExtension(resource)
+ resp.Response.Header.Set(ContentTypeHeader, mimeType)
+
+ if !shouldRespBeSavedToCache(resp.Response) {
+ return reader, resp.Response.Header, resp.StatusCode, err
+ }
+
+ // now we write to cache and respond at the sime time
+ fileResp := FileResponse{
+ Exists: true,
+ ETag: resp.Header.Get(ETagHeader),
+ MimeType: mimeType,
+ }
+ return fileResp.CreateCacheReader(reader, client.responseCache, cacheKey), resp.Response.Header, resp.StatusCode, nil
+
+ case http.StatusNotFound:
+ if err := client.responseCache.Set(cacheKey, FileResponse{
+ Exists: false,
+ ETag: resp.Header.Get(ETagHeader),
+ }, fileCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+
+ return nil, resp.Response.Header, http.StatusNotFound, ErrorNotFound
+ default:
+ return nil, resp.Response.Header, resp.StatusCode, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
+ }
+ }
+ return nil, nil, http.StatusInternalServerError, err
}
-func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (time.Time, error) {
- url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName)
- res, err := client.do(client.infoTimeout, url)
+func (client *Client) GiteaGetRepoBranchTimestamp(repoOwner, repoName, branchName string) (*BranchTimestamp, error) {
+ cacheKey := fmt.Sprintf("%s/%s/%s/%s", branchTimestampCacheKeyPrefix, repoOwner, repoName, branchName)
+
+ if stamp, ok := client.responseCache.Get(cacheKey); ok && stamp != nil {
+ branchTimeStamp := stamp.(*BranchTimestamp)
+ if branchTimeStamp.notFound {
+ log.Trace().Msgf("[cache] use branch %q not found", branchName)
+ return &BranchTimestamp{}, ErrorNotFound
+ }
+ log.Trace().Msgf("[cache] use branch %q exist", branchName)
+ return branchTimeStamp, nil
+ }
+
+ branch, resp, err := client.sdkClient.GetRepoBranch(repoOwner, repoName, branchName)
if err != nil {
- return time.Time{}, err
+ if resp != nil && resp.StatusCode == http.StatusNotFound {
+ log.Trace().Msgf("[cache] set cache branch %q not found", branchName)
+ if err := client.responseCache.Set(cacheKey, &BranchTimestamp{Branch: branchName, notFound: true}, branchExistenceCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+ return &BranchTimestamp{}, ErrorNotFound
+ }
+ return &BranchTimestamp{}, err
}
- if res.StatusCode() != fasthttp.StatusOK {
- return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+ if resp.StatusCode != http.StatusOK {
+ return &BranchTimestamp{}, fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
}
- return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+
+ stamp := &BranchTimestamp{
+ Branch: branch.Name,
+ Timestamp: branch.Commit.Timestamp,
+ }
+
+ log.Trace().Msgf("set cache branch [%s] exist", branchName)
+ if err := client.responseCache.Set(cacheKey, stamp, branchExistenceCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+ return stamp, nil
}
func (client *Client) GiteaGetRepoDefaultBranch(repoOwner, repoName string) (string, error) {
- url := joinURL(client.giteaRoot, giteaAPIRepos, repoOwner, repoName)
- res, err := client.do(client.infoTimeout, url)
+ cacheKey := fmt.Sprintf("%s/%s/%s", defaultBranchCacheKeyPrefix, repoOwner, repoName)
+
+ if branch, ok := client.responseCache.Get(cacheKey); ok && branch != nil {
+ return branch.(string), nil
+ }
+
+ repo, resp, err := client.sdkClient.GetRepo(repoOwner, repoName)
if err != nil {
return "", err
}
- if res.StatusCode() != fasthttp.StatusOK {
- return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+ if resp.StatusCode != http.StatusOK {
+ return "", fmt.Errorf("unexpected status code '%d'", resp.StatusCode)
}
- return fastjson.GetString(res.Body(), "default_branch"), nil
+
+ branch := repo.DefaultBranch
+ if err := client.responseCache.Set(cacheKey, branch, defaultBranchCacheTimeout); err != nil {
+ log.Error().Err(err).Msg("[cache] error on cache write")
+ }
+ return branch, nil
}
-func (client *Client) do(timeout time.Duration, url string) (*fasthttp.Response, error) {
- req := fasthttp.AcquireRequest()
-
- req.SetRequestURI(url)
- req.Header.Set(fasthttp.HeaderAuthorization, "token "+client.giteaAPIToken)
- res := fasthttp.AcquireResponse()
-
- err := client.fastClient.DoTimeout(req, res, timeout)
-
- return res, err
+func (client *Client) getMimeTypeByExtension(resource string) string {
+ mimeType := mime.TypeByExtension(path.Ext(resource))
+ mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
+ if client.forbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
+ mimeType = client.defaultMimeType
+ }
+ log.Trace().Msgf("probe mime of %q is %q", resource, mimeType)
+ return mimeType
+}
+
+func shouldRespBeSavedToCache(resp *http.Response) bool {
+ if resp == nil {
+ return false
+ }
+
+ contentLengthRaw := resp.Header.Get(ContentLengthHeader)
+ if contentLengthRaw == "" {
+ return false
+ }
+
+ contentLeng, err := strconv.ParseInt(contentLengthRaw, 10, 64)
+ if err != nil {
+ log.Error().Err(err).Msg("could not parse content length")
+ }
+
+ // if content to big or could not be determined we not cache it
+ return contentLeng > 0 && contentLeng < fileCacheSizeLimit
}
diff --git a/server/gitea/client_test.go b/server/gitea/client_test.go
deleted file mode 100644
index 7dbad68..0000000
--- a/server/gitea/client_test.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package gitea
-
-import (
- "net/url"
- "testing"
-
- "github.com/stretchr/testify/assert"
-)
-
-func TestJoinURL(t *testing.T) {
- baseURL := ""
- assert.EqualValues(t, "/", joinURL(baseURL))
- assert.EqualValues(t, "/", joinURL(baseURL, "", ""))
-
- baseURL = "http://wwow.url.com"
- assert.EqualValues(t, "http://wwow.url.com/a/b/c/d", joinURL(baseURL, "a", "b/c/", "d"))
-
- baseURL = "http://wow.url.com/subpath/2"
- assert.EqualValues(t, "http://wow.url.com/subpath/2/content.pdf", joinURL(baseURL, "/content.pdf"))
- assert.EqualValues(t, "http://wow.url.com/subpath/2/wonderful.jpg", joinURL(baseURL, "wonderful.jpg"))
- assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg?ref=main", joinURL(baseURL, "raw", "wonderful.jpg"+"?ref="+url.QueryEscape("main")))
- assert.EqualValues(t, "http://wow.url.com/subpath/2/raw/wonderful.jpg%3Fref=main", joinURL(baseURL, "raw", "wonderful.jpg%3Fref=main"))
-}
diff --git a/server/gitea/fasthttp.go b/server/gitea/fasthttp.go
deleted file mode 100644
index 4ff0f4a..0000000
--- a/server/gitea/fasthttp.go
+++ /dev/null
@@ -1,15 +0,0 @@
-package gitea
-
-import (
- "time"
-
- "github.com/valyala/fasthttp"
-)
-
-func getFastHTTPClient() *fasthttp.Client {
- return &fasthttp.Client{
- MaxConnDuration: 60 * time.Second,
- MaxConnWaitTimeout: 1000 * time.Millisecond,
- MaxConnsPerHost: 128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
- }
-}
diff --git a/server/handler.go b/server/handler.go
index fb8b419..894cd25 100644
--- a/server/handler.go
+++ b/server/handler.go
@@ -1,15 +1,17 @@
package server
import (
- "bytes"
+ "fmt"
+ "net/http"
+ "path"
"strings"
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
- "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/dns"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
@@ -17,42 +19,48 @@ import (
"codeberg.org/codeberg/pages/server/version"
)
+const (
+ headerAccessControlAllowOrigin = "Access-Control-Allow-Origin"
+ headerAccessControlAllowMethods = "Access-Control-Allow-Methods"
+)
+
// Handler handles a single HTTP request to the web server.
-func Handler(mainDomainSuffix, rawDomain []byte,
+func Handler(mainDomainSuffix, rawDomain string,
giteaClient *gitea.Client,
giteaRoot, rawInfoPage string,
- blacklistedPaths, allowedCorsDomains [][]byte,
- dnsLookupCache, canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
-) func(ctx *fasthttp.RequestCtx) {
- return func(ctx *fasthttp.RequestCtx) {
- log := log.With().Strs("Handler", []string{string(ctx.Request.Host()), string(ctx.Request.Header.RequestURI())}).Logger()
+ blacklistedPaths, allowedCorsDomains []string,
+ dnsLookupCache, canonicalDomainCache cache.SetGetKey,
+) http.HandlerFunc {
+ return func(w http.ResponseWriter, req *http.Request) {
+ log := log.With().Strs("Handler", []string{string(req.Host), req.RequestURI}).Logger()
+ ctx := context.New(w, req)
- ctx.Response.Header.Set("Server", "CodebergPages/"+version.Version)
+ ctx.RespWriter.Header().Set("Server", "CodebergPages/"+version.Version)
// Force new default from specification (since November 2020) - see https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#strict-origin-when-cross-origin
- ctx.Response.Header.Set("Referrer-Policy", "strict-origin-when-cross-origin")
+ ctx.RespWriter.Header().Set("Referrer-Policy", "strict-origin-when-cross-origin")
// Enable browser caching for up to 10 minutes
- ctx.Response.Header.Set("Cache-Control", "public, max-age=600")
+ ctx.RespWriter.Header().Set("Cache-Control", "public, max-age=600")
- trimmedHost := utils.TrimHostPort(ctx.Request.Host())
+ trimmedHost := utils.TrimHostPort(req.Host)
// Add HSTS for RawDomain and MainDomainSuffix
- if hsts := GetHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
- ctx.Response.Header.Set("Strict-Transport-Security", hsts)
+ if hsts := getHSTSHeader(trimmedHost, mainDomainSuffix, rawDomain); hsts != "" {
+ ctx.RespWriter.Header().Set("Strict-Transport-Security", hsts)
}
// Block all methods not required for static pages
- if !ctx.IsGet() && !ctx.IsHead() && !ctx.IsOptions() {
- ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
- ctx.Error("Method not allowed", fasthttp.StatusMethodNotAllowed)
+ if !ctx.IsMethod(http.MethodGet) && !ctx.IsMethod(http.MethodHead) && !ctx.IsMethod(http.MethodOptions) {
+ ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1
+ ctx.String("Method not allowed", http.StatusMethodNotAllowed)
return
}
// Block blacklisted paths (like ACME challenges)
for _, blacklistedPath := range blacklistedPaths {
- if bytes.HasPrefix(ctx.Path(), blacklistedPath) {
- html.ReturnErrorPage(ctx, fasthttp.StatusForbidden)
+ if strings.HasPrefix(ctx.Path(), blacklistedPath) {
+ html.ReturnErrorPage(ctx, "requested blacklisted path", http.StatusForbidden)
return
}
}
@@ -60,18 +68,19 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Allow CORS for specified domains
allowCors := false
for _, allowedCorsDomain := range allowedCorsDomains {
- if bytes.Equal(trimmedHost, allowedCorsDomain) {
+ if strings.EqualFold(trimmedHost, allowedCorsDomain) {
allowCors = true
break
}
}
if allowCors {
- ctx.Response.Header.Set("Access-Control-Allow-Origin", "*")
- ctx.Response.Header.Set("Access-Control-Allow-Methods", "GET, HEAD")
+ ctx.RespWriter.Header().Set(headerAccessControlAllowOrigin, "*")
+ ctx.RespWriter.Header().Set(headerAccessControlAllowMethods, http.MethodGet+", "+http.MethodHead)
}
- ctx.Response.Header.Set("Allow", "GET, HEAD, OPTIONS")
- if ctx.IsOptions() {
- ctx.Response.Header.SetStatusCode(fasthttp.StatusNoContent)
+
+ ctx.RespWriter.Header().Set("Allow", http.MethodGet+", "+http.MethodHead+", "+http.MethodOptions) // duplic 1
+ if ctx.IsMethod(http.MethodOptions) {
+ ctx.RespWriter.WriteHeader(http.StatusNoContent)
return
}
@@ -83,9 +92,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// tryBranch checks if a branch exists and populates the target variables. If canonicalLink is non-empty, it will
// also disallow search indexing and add a Link header to the canonical URL.
- tryBranch := func(log zerolog.Logger, repo, branch string, path []string, canonicalLink string) bool {
+ // TODO: move into external func to not alert vars indirectly
+ tryBranch := func(log zerolog.Logger, repo, branch string, _path []string, canonicalLink string) bool {
if repo == "" {
- log.Warn().Msg("tryBranch: repo is empty")
+ log.Debug().Msg("tryBranch: repo is empty")
return false
}
@@ -94,23 +104,23 @@ func Handler(mainDomainSuffix, rawDomain []byte,
branch = strings.ReplaceAll(branch, "~", "/")
// Check if the branch exists, otherwise treat it as a file path
- branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch, branchTimestampCache)
+ branchTimestampResult := upstream.GetBranchTimestamp(giteaClient, targetOwner, repo, branch)
if branchTimestampResult == nil {
- log.Warn().Msg("tryBranch: branch doesn't exist")
+ log.Debug().Msg("tryBranch: branch doesn't exist")
return false
}
// Branch exists, use it
targetRepo = repo
- targetPath = strings.Trim(strings.Join(path, "/"), "/")
+ targetPath = path.Join(_path...)
targetBranch = branchTimestampResult.Branch
targetOptions.BranchTimestamp = branchTimestampResult.Timestamp
if canonicalLink != "" {
// Hide from search machines & add canonical link
- ctx.Response.Header.Set("X-Robots-Tag", "noarchive, noindex")
- ctx.Response.Header.Set("Link",
+ ctx.RespWriter.Header().Set("X-Robots-Tag", "noarchive, noindex")
+ ctx.RespWriter.Header().Set("Link",
strings.NewReplacer("%b", targetBranch, "%p", targetPath).Replace(canonicalLink)+
"; rel=\"canonical\"",
)
@@ -120,22 +130,18 @@ func Handler(mainDomainSuffix, rawDomain []byte,
return true
}
- log.Debug().Msg("Preparing")
- if rawDomain != nil && bytes.Equal(trimmedHost, rawDomain) {
+ log.Debug().Msg("preparations")
+ if rawDomain != "" && strings.EqualFold(trimmedHost, rawDomain) {
// Serve raw content from RawDomain
- log.Debug().Msg("Serving raw domain")
+ log.Debug().Msg("raw domain")
targetOptions.TryIndexPages = false
- if targetOptions.ForbiddenMimeTypes == nil {
- targetOptions.ForbiddenMimeTypes = make(map[string]bool)
- }
- targetOptions.ForbiddenMimeTypes["text/html"] = true
- targetOptions.DefaultMimeType = "text/plain; charset=utf-8"
+ targetOptions.ServeRaw = true
- pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+ pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
if len(pathElements) < 2 {
// https://{RawDomain}/{owner}/{repo}[/@{branch}]/{path} is required
- ctx.Redirect(rawInfoPage, fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect(rawInfoPage, http.StatusTemporaryRedirect)
return
}
targetOwner = pathElements[0]
@@ -143,45 +149,45 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// raw.codeberg.org/example/myrepo/@main/index.html
if len(pathElements) > 2 && strings.HasPrefix(pathElements[2], "@") {
- log.Debug().Msg("Preparing raw domain, now trying with specified branch")
+ log.Debug().Msg("raw domain preparations, now trying with specified branch")
if tryBranch(log,
targetRepo, pathElements[2][1:], pathElements[3:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
) {
- log.Info().Msg("tryBranch, now trying upstream 1")
+ log.Debug().Msg("tryBranch, now trying upstream 1")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
return
}
- log.Warn().Msg("Path missed a branch")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ log.Debug().Msg("missing branch info")
+ html.ReturnErrorPage(ctx, "missing branch info", http.StatusFailedDependency)
return
}
- log.Debug().Msg("Preparing raw domain, now trying with default branch")
+ log.Debug().Msg("raw domain preparations, now trying with default branch")
tryBranch(log,
targetRepo, "", pathElements[2:],
giteaRoot+"/"+targetOwner+"/"+targetRepo+"/src/branch/%b/%p",
)
- log.Info().Msg("tryBranch, now trying upstream 2")
+ log.Debug().Msg("tryBranch, now trying upstream 2")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
return
- } else if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+ } else if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
// Serve pages from subdomains of MainDomainSuffix
- log.Info().Msg("Serve pages from main domain suffix")
+ log.Debug().Msg("main domain suffix")
- pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
- targetOwner = string(bytes.TrimSuffix(trimmedHost, mainDomainSuffix))
+ pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
+ targetOwner = strings.TrimSuffix(trimmedHost, mainDomainSuffix)
targetRepo = pathElements[0]
targetPath = strings.Trim(strings.Join(pathElements[1:], "/"), "/")
if targetOwner == "www" {
// www.codeberg.page redirects to codeberg.page // TODO: rm hardcoded - use cname?
- ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), fasthttp.StatusPermanentRedirect)
+ ctx.Redirect("https://"+string(mainDomainSuffix[1:])+string(ctx.Path()), http.StatusPermanentRedirect)
return
}
@@ -190,22 +196,24 @@ func Handler(mainDomainSuffix, rawDomain []byte,
if len(pathElements) > 1 && strings.HasPrefix(pathElements[1], "@") {
if targetRepo == "pages" {
// example.codeberg.org/pages/@... redirects to example.codeberg.org/@...
- ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect("/"+strings.Join(pathElements[1:], "/"), http.StatusTemporaryRedirect)
return
}
- log.Debug().Msg("Preparing main domain, now trying with specified repo & branch")
+ log.Debug().Msg("main domain preparations, now trying with specified repo & branch")
+ branch := pathElements[1][1:]
if tryBranch(log,
- pathElements[0], pathElements[1][1:], pathElements[2:],
+ pathElements[0], branch, pathElements[2:],
"/"+pathElements[0]+"/%p",
) {
- log.Info().Msg("tryBranch, now trying upstream 3")
+ log.Debug().Msg("tryBranch, now trying upstream 3")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
} else {
- log.Warn().Msg("tryBranch: upstream 3 failed")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, targetRepo),
+ http.StatusFailedDependency)
}
return
}
@@ -213,16 +221,18 @@ func Handler(mainDomainSuffix, rawDomain []byte,
// Check if the first directory is a branch for the "pages" repo
// example.codeberg.page/@main/index.html
if strings.HasPrefix(pathElements[0], "@") {
- log.Debug().Msg("Preparing main domain, now trying with specified branch")
+ log.Debug().Msg("main domain preparations, now trying with specified branch")
+ branch := pathElements[0][1:]
if tryBranch(log,
- "pages", pathElements[0][1:], pathElements[1:], "/%p") {
- log.Info().Msg("tryBranch, now trying upstream 4")
+ "pages", branch, pathElements[1:], "/%p") {
+ log.Debug().Msg("tryBranch, now trying upstream 4")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
- targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ targetOptions, targetOwner, "pages", targetBranch, targetPath,
+ canonicalDomainCache)
} else {
- log.Warn().Msg("tryBranch: upstream 4 failed")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("explizite set branch %q do not exist at '%s/%s'", branch, targetOwner, "pages"),
+ http.StatusFailedDependency)
}
return
}
@@ -233,10 +243,10 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with specified repo")
if pathElements[0] != "pages" && tryBranch(log,
pathElements[0], "pages", pathElements[1:], "") {
- log.Info().Msg("tryBranch, now trying upstream 5")
+ log.Debug().Msg("tryBranch, now trying upstream 5")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
return
}
@@ -245,28 +255,31 @@ func Handler(mainDomainSuffix, rawDomain []byte,
log.Debug().Msg("main domain preparations, now trying with default repo/branch")
if tryBranch(log,
"pages", "", pathElements, "") {
- log.Info().Msg("tryBranch, now trying upstream 6")
+ log.Debug().Msg("tryBranch, now trying upstream 6")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
return
}
// Couldn't find a valid repo/branch
-
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("couldn't find a valid repo[%s]/branch[%s]", targetRepo, targetBranch),
+ http.StatusFailedDependency)
return
} else {
trimmedHostStr := string(trimmedHost)
- // Serve pages from external domains
+ // Serve pages from custom domains
targetOwner, targetRepo, targetBranch = dns.GetTargetFromDNS(trimmedHostStr, string(mainDomainSuffix), dnsLookupCache)
if targetOwner == "" {
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx,
+ "could not obtain repo owner from custom domain",
+ http.StatusFailedDependency)
return
}
- pathElements := strings.Split(string(bytes.Trim(ctx.Request.URI().Path(), "/")), "/")
+ pathElements := strings.Split(strings.Trim(ctx.Path(), "/"), "/")
canonicalLink := ""
if strings.HasPrefix(pathElements[0], "@") {
targetBranch = pathElements[0][1:]
@@ -275,36 +288,33 @@ func Handler(mainDomainSuffix, rawDomain []byte,
}
// Try to use the given repo on the given branch or the default branch
- log.Debug().Msg("Preparing custom domain, now trying with details from DNS")
+ log.Debug().Msg("custom domain preparations, now trying with details from DNS")
if tryBranch(log,
targetRepo, targetBranch, pathElements, canonicalLink) {
canonicalDomain, valid := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, trimmedHostStr, string(mainDomainSuffix), canonicalDomainCache)
if !valid {
- log.Warn().Msg("Custom domains, domain from DNS isn't valid/canonical")
- html.ReturnErrorPage(ctx, fasthttp.StatusMisdirectedRequest)
+ html.ReturnErrorPage(ctx, "domain not specified in .domains file", http.StatusMisdirectedRequest)
return
} else if canonicalDomain != trimmedHostStr {
// only redirect if the target is also a codeberg page!
targetOwner, _, _ = dns.GetTargetFromDNS(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix), dnsLookupCache)
if targetOwner != "" {
- ctx.Redirect("https://"+canonicalDomain+string(ctx.RequestURI()), fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect("https://"+canonicalDomain+string(ctx.Path()), http.StatusTemporaryRedirect)
return
}
- log.Warn().Msg("Custom domains, targetOwner from DNS is empty")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx, "target is no codeberg page", http.StatusFailedDependency)
return
}
- log.Info().Msg("tryBranch, now trying upstream 7")
+ log.Debug().Msg("tryBranch, now trying upstream 7")
tryUpstream(ctx, giteaClient, mainDomainSuffix, trimmedHost,
targetOptions, targetOwner, targetRepo, targetBranch, targetPath,
- canonicalDomainCache, branchTimestampCache, fileResponseCache)
+ canonicalDomainCache)
return
}
- log.Warn().Msg("Couldn't handle request, none of the options succeed")
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ html.ReturnErrorPage(ctx, "could not find target for custom domain", http.StatusFailedDependency)
return
}
}
diff --git a/server/handler_test.go b/server/handler_test.go
index f9a721a..c0aca14 100644
--- a/server/handler_test.go
+++ b/server/handler_test.go
@@ -1,44 +1,42 @@
package server
import (
- "fmt"
+ "net/http/httptest"
"testing"
"time"
- "github.com/valyala/fasthttp"
-
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
+ "github.com/rs/zerolog/log"
)
func TestHandlerPerformance(t *testing.T) {
giteaRoot := "https://codeberg.org"
- giteaClient, _ := gitea.NewClient(giteaRoot, "", false, false)
+ giteaClient, _ := gitea.NewClient(giteaRoot, "", cache.NewKeyValueCache(), false, false)
testHandler := Handler(
- []byte("codeberg.page"), []byte("raw.codeberg.org"),
+ "codeberg.page", "raw.codeberg.org",
giteaClient,
giteaRoot, "https://docs.codeberg.org/pages/raw-content/",
- [][]byte{[]byte("/.well-known/acme-challenge/")},
- [][]byte{[]byte("raw.codeberg.org"), []byte("fonts.codeberg.org"), []byte("design.codeberg.org")},
- cache.NewKeyValueCache(),
- cache.NewKeyValueCache(),
+ []string{"/.well-known/acme-challenge/"},
+ []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
cache.NewKeyValueCache(),
cache.NewKeyValueCache(),
)
testCase := func(uri string, status int) {
- ctx := &fasthttp.RequestCtx{
- Request: *fasthttp.AcquireRequest(),
- Response: *fasthttp.AcquireResponse(),
- }
- ctx.Request.SetRequestURI(uri)
- fmt.Printf("Start: %v\n", time.Now())
+ req := httptest.NewRequest("GET", uri, nil)
+ w := httptest.NewRecorder()
+
+ log.Printf("Start: %v\n", time.Now())
start := time.Now()
- testHandler(ctx)
+ testHandler(w, req)
end := time.Now()
- fmt.Printf("Done: %v\n", time.Now())
- if ctx.Response.StatusCode() != status {
- t.Errorf("request failed with status code %d", ctx.Response.StatusCode())
+ log.Printf("Done: %v\n", time.Now())
+
+ resp := w.Result()
+
+ if resp.StatusCode != status {
+ t.Errorf("request failed with status code %d", resp.StatusCode)
} else {
t.Logf("request took %d milliseconds", end.Sub(start).Milliseconds())
}
diff --git a/server/helpers.go b/server/helpers.go
index 6d55ddf..7c898cd 100644
--- a/server/helpers.go
+++ b/server/helpers.go
@@ -1,13 +1,13 @@
package server
import (
- "bytes"
+ "strings"
)
-// GetHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
+// getHSTSHeader returns a HSTS header with includeSubdomains & preload for MainDomainSuffix and RawDomain, or an empty
// string for custom domains.
-func GetHSTSHeader(host, mainDomainSuffix, rawDomain []byte) string {
- if bytes.HasSuffix(host, mainDomainSuffix) || bytes.Equal(host, rawDomain) {
+func getHSTSHeader(host, mainDomainSuffix, rawDomain string) string {
+ if strings.HasSuffix(host, mainDomainSuffix) || strings.EqualFold(host, rawDomain) {
return "max-age=63072000; includeSubdomains; preload"
} else {
return ""
diff --git a/server/setup.go b/server/setup.go
index 176bb42..e7194ed 100644
--- a/server/setup.go
+++ b/server/setup.go
@@ -1,53 +1,27 @@
package server
import (
- "bytes"
- "fmt"
"net/http"
- "time"
-
- "github.com/rs/zerolog/log"
- "github.com/valyala/fasthttp"
+ "strings"
"codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/utils"
)
-type fasthttpLogger struct{}
+func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) http.HandlerFunc {
+ challengePath := "/.well-known/acme-challenge/"
-func (fasthttpLogger) Printf(format string, args ...interface{}) {
- log.Printf("FastHTTP: %s", fmt.Sprintf(format, args...))
-}
-
-func SetupServer(handler fasthttp.RequestHandler) *fasthttp.Server {
- // Enable compression by wrapping the handler with the compression function provided by FastHTTP
- compressedHandler := fasthttp.CompressHandlerBrotliLevel(handler, fasthttp.CompressBrotliBestSpeed, fasthttp.CompressBestSpeed)
-
- return &fasthttp.Server{
- Handler: compressedHandler,
- DisablePreParseMultipartForm: true,
- NoDefaultServerHeader: true,
- NoDefaultDate: true,
- ReadTimeout: 30 * time.Second, // needs to be this high for ACME certificates with ZeroSSL & HTTP-01 challenge
- Logger: fasthttpLogger{},
- }
-}
-
-func SetupHTTPACMEChallengeServer(challengeCache cache.SetGetKey) *fasthttp.Server {
- challengePath := []byte("/.well-known/acme-challenge/")
-
- return &fasthttp.Server{
- Handler: func(ctx *fasthttp.RequestCtx) {
- if bytes.HasPrefix(ctx.Path(), challengePath) {
- challenge, ok := challengeCache.Get(string(utils.TrimHostPort(ctx.Host())) + "/" + string(bytes.TrimPrefix(ctx.Path(), challengePath)))
- if !ok || challenge == nil {
- ctx.SetStatusCode(http.StatusNotFound)
- ctx.SetBodyString("no challenge for this token")
- }
- ctx.SetBodyString(challenge.(string))
- } else {
- ctx.Redirect("https://"+string(ctx.Host())+string(ctx.RequestURI()), http.StatusMovedPermanently)
+ return func(w http.ResponseWriter, req *http.Request) {
+ ctx := context.New(w, req)
+ if strings.HasPrefix(ctx.Path(), challengePath) {
+ challenge, ok := challengeCache.Get(utils.TrimHostPort(ctx.Host()) + "/" + string(strings.TrimPrefix(ctx.Path(), challengePath)))
+ if !ok || challenge == nil {
+ ctx.String("no challenge for this token", http.StatusNotFound)
}
- },
+ ctx.String(challenge.(string))
+ } else {
+ ctx.Redirect("https://"+string(ctx.Host())+string(ctx.Path()), http.StatusMovedPermanently)
+ }
}
}
diff --git a/server/try.go b/server/try.go
index 24831c4..135c1e0 100644
--- a/server/try.go
+++ b/server/try.go
@@ -1,38 +1,37 @@
package server
import (
- "bytes"
+ "net/http"
"strings"
- "github.com/valyala/fasthttp"
-
"codeberg.org/codeberg/pages/html"
"codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
"codeberg.org/codeberg/pages/server/upstream"
)
// tryUpstream forwards the target request to the Gitea API, and shows an error page on failure.
-func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
- mainDomainSuffix, trimmedHost []byte,
+func tryUpstream(ctx *context.Context, giteaClient *gitea.Client,
+ mainDomainSuffix, trimmedHost string,
targetOptions *upstream.Options,
targetOwner, targetRepo, targetBranch, targetPath string,
- canonicalDomainCache, branchTimestampCache, fileResponseCache cache.SetGetKey,
+ canonicalDomainCache cache.SetGetKey,
) {
// check if a canonical domain exists on a request on MainDomain
- if bytes.HasSuffix(trimmedHost, mainDomainSuffix) {
+ if strings.HasSuffix(trimmedHost, mainDomainSuffix) {
canonicalDomain, _ := upstream.CheckCanonicalDomain(giteaClient, targetOwner, targetRepo, targetBranch, "", string(mainDomainSuffix), canonicalDomainCache)
if !strings.HasSuffix(strings.SplitN(canonicalDomain, "/", 2)[0], string(mainDomainSuffix)) {
- canonicalPath := string(ctx.RequestURI())
+ canonicalPath := ctx.Req.RequestURI
if targetRepo != "pages" {
path := strings.SplitN(canonicalPath, "/", 3)
if len(path) >= 3 {
canonicalPath = "/" + path[2]
}
}
- ctx.Redirect("https://"+canonicalDomain+canonicalPath, fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect("https://"+canonicalDomain+canonicalPath, http.StatusTemporaryRedirect)
return
}
}
@@ -44,7 +43,7 @@ func tryUpstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
targetOptions.Host = string(trimmedHost)
// Try to request the file from the Gitea API
- if !targetOptions.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- html.ReturnErrorPage(ctx, ctx.Response.StatusCode())
+ if !targetOptions.Upstream(ctx, giteaClient) {
+ html.ReturnErrorPage(ctx, "", ctx.StatusCode)
}
}
diff --git a/server/upstream/const.go b/server/upstream/const.go
deleted file mode 100644
index 247e1d1..0000000
--- a/server/upstream/const.go
+++ /dev/null
@@ -1,24 +0,0 @@
-package upstream
-
-import "time"
-
-// defaultBranchCacheTimeout specifies the timeout for the default branch cache. It can be quite long.
-var defaultBranchCacheTimeout = 15 * time.Minute
-
-// branchExistenceCacheTimeout specifies the timeout for the branch timestamp & existence cache. It should be shorter
-// than fileCacheTimeout, as that gets invalidated if the branch timestamp has changed. That way, repo changes will be
-// picked up faster, while still allowing the content to be cached longer if nothing changes.
-var branchExistenceCacheTimeout = 5 * time.Minute
-
-// fileCacheTimeout specifies the timeout for the file content cache - you might want to make this quite long, depending
-// on your available memory.
-// TODO: move as option into cache interface
-var fileCacheTimeout = 5 * time.Minute
-
-// fileCacheSizeLimit limits the maximum file size that will be cached, and is set to 1 MB by default.
-var fileCacheSizeLimit = 1024 * 1024
-
-// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
-var canonicalDomainCacheTimeout = 15 * time.Minute
-
-const canonicalDomainConfig = ".domains"
diff --git a/server/upstream/domains.go b/server/upstream/domains.go
index 553c148..6ad6506 100644
--- a/server/upstream/domains.go
+++ b/server/upstream/domains.go
@@ -2,11 +2,19 @@ package upstream
import (
"strings"
+ "time"
+
+ "github.com/rs/zerolog/log"
"codeberg.org/codeberg/pages/server/cache"
"codeberg.org/codeberg/pages/server/gitea"
)
+// canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
+var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"
+
// CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix string, canonicalDomainCache cache.SetGetKey) (string, bool) {
var (
@@ -36,6 +44,8 @@ func CheckCanonicalDomain(giteaClient *gitea.Client, targetOwner, targetRepo, ta
valid = true
}
}
+ } else {
+ log.Info().Err(err).Msgf("could not read %s of %s/%s", canonicalDomainConfig, targetOwner, targetRepo)
}
domains = append(domains, targetOwner+mainDomainSuffix)
if domains[len(domains)-1] == actualDomain {
diff --git a/server/upstream/helper.go b/server/upstream/helper.go
index 28f4474..6bc23c8 100644
--- a/server/upstream/helper.go
+++ b/server/upstream/helper.go
@@ -1,84 +1,36 @@
package upstream
import (
- "mime"
- "path"
- "strconv"
- "strings"
- "time"
+ "errors"
- "codeberg.org/codeberg/pages/server/cache"
- "codeberg.org/codeberg/pages/server/gitea"
"github.com/rs/zerolog/log"
-)
-type branchTimestamp struct {
- Branch string
- Timestamp time.Time
-}
+ "codeberg.org/codeberg/pages/server/gitea"
+)
// GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
// (or nil if the branch doesn't exist)
-func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(giteaClient *gitea.Client, owner, repo, branch string) *gitea.BranchTimestamp {
log := log.With().Strs("BranchInfo", []string{owner, repo, branch}).Logger()
- if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
- if result == nil {
- log.Debug().Msg("branchTimestampCache found item, but result is empty")
- return nil
- }
- log.Debug().Msg("branchTimestampCache found item, returning result")
- return result.(*branchTimestamp)
- }
- result := &branchTimestamp{
- Branch: branch,
- }
+
if len(branch) == 0 {
// Get default branch
defaultBranch, err := giteaClient.GiteaGetRepoDefaultBranch(owner, repo)
if err != nil {
log.Err(err).Msg("Could't fetch default branch from repository")
- _ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
return nil
}
- log.Debug().Msg("Succesfully fetched default branch from Gitea")
- result.Branch = defaultBranch
+ log.Debug().Msgf("Succesfully fetched default branch %q from Gitea", defaultBranch)
+ branch = defaultBranch
}
- timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, result.Branch)
+ timestamp, err := giteaClient.GiteaGetRepoBranchTimestamp(owner, repo, branch)
if err != nil {
- log.Err(err).Msg("Could not get latest commit's timestamp from branch")
+ if !errors.Is(err, gitea.ErrorNotFound) {
+ log.Error().Err(err).Msg("Could not get latest commit's timestamp from branch")
+ }
return nil
}
- log.Debug().Msg("Succesfully fetched latest commit's timestamp from branch, adding to cache")
- result.Timestamp = timestamp
- _ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
- return result
-}
-
-func (o *Options) getMimeTypeByExtension() string {
- if o.ForbiddenMimeTypes == nil {
- o.ForbiddenMimeTypes = make(map[string]bool)
- }
- mimeType := mime.TypeByExtension(path.Ext(o.TargetPath))
- mimeTypeSplit := strings.SplitN(mimeType, ";", 2)
- if o.ForbiddenMimeTypes[mimeTypeSplit[0]] || mimeType == "" {
- if o.DefaultMimeType != "" {
- mimeType = o.DefaultMimeType
- } else {
- mimeType = "application/octet-stream"
- }
- }
- return mimeType
-}
-
-func (o *Options) generateUri() string {
- return path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
-}
-
-func (o *Options) generateUriClientArgs() (targetOwner, targetRepo, ref, resource string) {
- return o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath
-}
-
-func (o *Options) timestamp() string {
- return strconv.FormatInt(o.BranchTimestamp.Unix(), 10)
+ log.Debug().Msgf("Succesfully fetched latest commit's timestamp from branch: %#v", timestamp)
+ return timestamp
}
diff --git a/server/upstream/upstream.go b/server/upstream/upstream.go
index 61c90de..d37c35e 100644
--- a/server/upstream/upstream.go
+++ b/server/upstream/upstream.go
@@ -1,20 +1,27 @@
package upstream
import (
- "bytes"
"errors"
+ "fmt"
"io"
+ "net/http"
"strings"
"time"
"github.com/rs/zerolog/log"
- "github.com/valyala/fasthttp"
"codeberg.org/codeberg/pages/html"
- "codeberg.org/codeberg/pages/server/cache"
+ "codeberg.org/codeberg/pages/server/context"
"codeberg.org/codeberg/pages/server/gitea"
)
+const (
+ headerLastModified = "Last-Modified"
+ headerIfModifiedSince = "If-Modified-Since"
+
+ rawMime = "text/plain; charset=utf-8"
+)
+
// upstreamIndexPages lists pages that may be considered as index pages for directories.
var upstreamIndexPages = []string{
"index.html",
@@ -35,61 +42,61 @@ type Options struct {
// Used for debugging purposes.
Host string
- DefaultMimeType string
- ForbiddenMimeTypes map[string]bool
- TryIndexPages bool
- BranchTimestamp time.Time
+ TryIndexPages bool
+ BranchTimestamp time.Time
// internal
appendTrailingSlash bool
redirectIfExists string
+
+ ServeRaw bool
}
// Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
-func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client, branchTimestampCache, fileResponseCache cache.SetGetKey) (final bool) {
- log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath, o.Host}).Logger()
+func (o *Options) Upstream(ctx *context.Context, giteaClient *gitea.Client) (final bool) {
+ log := log.With().Strs("upstream", []string{o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath}).Logger()
+
+ if o.TargetOwner == "" || o.TargetRepo == "" {
+ html.ReturnErrorPage(ctx, "either repo owner or name info is missing", http.StatusBadRequest)
+ return true
+ }
// Check if the branch exists and when it was modified
if o.BranchTimestamp.IsZero() {
- branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch, branchTimestampCache)
+ branch := GetBranchTimestamp(giteaClient, o.TargetOwner, o.TargetRepo, o.TargetBranch)
- if branch == nil {
- html.ReturnErrorPage(ctx, fasthttp.StatusFailedDependency)
+ if branch == nil || branch.Branch == "" {
+ html.ReturnErrorPage(ctx,
+ fmt.Sprintf("could not get timestamp of branch %q", o.TargetBranch),
+ http.StatusFailedDependency)
return true
}
o.TargetBranch = branch.Branch
o.BranchTimestamp = branch.Timestamp
}
- if o.TargetOwner == "" || o.TargetRepo == "" || o.TargetBranch == "" {
- html.ReturnErrorPage(ctx, fasthttp.StatusBadRequest)
- return true
- }
-
// Check if the browser has a cached version
- if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Request.Header.Peek("If-Modified-Since"))); err == nil {
- if !ifModifiedSince.Before(o.BranchTimestamp) {
- ctx.Response.SetStatusCode(fasthttp.StatusNotModified)
- return true
+ if ctx.Response() != nil {
+ if ifModifiedSince, err := time.Parse(time.RFC1123, string(ctx.Response().Header.Get(headerIfModifiedSince))); err == nil {
+ if !ifModifiedSince.Before(o.BranchTimestamp) {
+ ctx.RespWriter.WriteHeader(http.StatusNotModified)
+ log.Trace().Msg("check response against last modified: valid")
+ return true
+ }
}
+ log.Trace().Msg("check response against last modified: outdated")
}
log.Debug().Msg("Preparing")
- // Make a GET request to the upstream URL
- uri := o.generateUri()
- var res *fasthttp.Response
- var cachedResponse gitea.FileResponse
- var err error
- if cachedValue, ok := fileResponseCache.Get(uri + "?timestamp=" + o.timestamp()); ok && !cachedValue.(gitea.FileResponse).IsEmpty() {
- cachedResponse = cachedValue.(gitea.FileResponse)
- } else {
- res, err = giteaClient.ServeRawContent(o.generateUriClientArgs())
+ reader, header, statusCode, err := giteaClient.ServeRawContent(o.TargetOwner, o.TargetRepo, o.TargetBranch, o.TargetPath)
+ if reader != nil {
+ defer reader.Close()
}
log.Debug().Msg("Aquisting")
- // Handle errors
- if (err != nil && errors.Is(err, gitea.ErrorNotFound)) || (res == nil && !cachedResponse.Exists) {
+ // Handle not found error
+ if err != nil && errors.Is(err, gitea.ErrorNotFound) {
if o.TryIndexPages {
// copy the o struct & try if an index page exists
optionsForIndexPages := *o
@@ -97,25 +104,20 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForIndexPages.appendTrailingSlash = true
for _, indexPage := range upstreamIndexPages {
optionsForIndexPages.TargetPath = strings.TrimSuffix(o.TargetPath, "/") + "/" + indexPage
- if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
+ if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
// compatibility fix for GitHub Pages (/example → /example.html)
optionsForIndexPages.appendTrailingSlash = false
- optionsForIndexPages.redirectIfExists = strings.TrimSuffix(string(ctx.Request.URI().Path()), "/") + ".html"
+ optionsForIndexPages.redirectIfExists = strings.TrimSuffix(ctx.Path(), "/") + ".html"
optionsForIndexPages.TargetPath = o.TargetPath + ".html"
- if optionsForIndexPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
+ if optionsForIndexPages.Upstream(ctx, giteaClient) {
return true
}
}
- ctx.Response.SetStatusCode(fasthttp.StatusNotFound)
+
+ ctx.StatusCode = http.StatusNotFound
if o.TryIndexPages {
// copy the o struct & try if a not found page exists
optionsForNotFoundPages := *o
@@ -123,94 +125,84 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaClient *gitea.Client,
optionsForNotFoundPages.appendTrailingSlash = false
for _, notFoundPage := range upstreamNotFoundPages {
optionsForNotFoundPages.TargetPath = "/" + notFoundPage
- if optionsForNotFoundPages.Upstream(ctx, giteaClient, branchTimestampCache, fileResponseCache) {
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
+ if optionsForNotFoundPages.Upstream(ctx, giteaClient) {
return true
}
}
}
- if res != nil {
- // Update cache if the request is fresh
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), gitea.FileResponse{
- Exists: false,
- }, fileCacheTimeout)
- }
return false
}
- if res != nil && (err != nil || res.StatusCode() != fasthttp.StatusOK) {
- log.Warn().Msgf("Couldn't fetch contents from %q: %v (status code %d)", uri, err, res.StatusCode())
- html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
+
+ // handle unexpected client errors
+ if err != nil || reader == nil || statusCode != http.StatusOK {
+ log.Debug().Msg("Handling error")
+ var msg string
+
+ if err != nil {
+ msg = "gitea client returned unexpected error"
+ log.Error().Err(err).Msg(msg)
+ msg = fmt.Sprintf("%s: %v", msg, err)
+ }
+ if reader == nil {
+ msg = "gitea client returned no reader"
+ log.Error().Msg(msg)
+ }
+ if statusCode != http.StatusOK {
+ msg = fmt.Sprintf("Couldn't fetch contents (status code %d)", statusCode)
+ log.Error().Msg(msg)
+ }
+
+ html.ReturnErrorPage(ctx, msg, http.StatusInternalServerError)
return true
}
// Append trailing slash if missing (for index files), and redirect to fix filenames in general
// o.appendTrailingSlash is only true when looking for index pages
- if o.appendTrailingSlash && !bytes.HasSuffix(ctx.Request.URI().Path(), []byte{'/'}) {
- ctx.Redirect(string(ctx.Request.URI().Path())+"/", fasthttp.StatusTemporaryRedirect)
+ if o.appendTrailingSlash && !strings.HasSuffix(ctx.Path(), "/") {
+ ctx.Redirect(ctx.Path()+"/", http.StatusTemporaryRedirect)
return true
}
- if bytes.HasSuffix(ctx.Request.URI().Path(), []byte("/index.html")) {
- ctx.Redirect(strings.TrimSuffix(string(ctx.Request.URI().Path()), "index.html"), fasthttp.StatusTemporaryRedirect)
+ if strings.HasSuffix(ctx.Path(), "/index.html") {
+ ctx.Redirect(strings.TrimSuffix(ctx.Path(), "index.html"), http.StatusTemporaryRedirect)
return true
}
if o.redirectIfExists != "" {
- ctx.Redirect(o.redirectIfExists, fasthttp.StatusTemporaryRedirect)
+ ctx.Redirect(o.redirectIfExists, http.StatusTemporaryRedirect)
return true
}
- log.Debug().Msg("Handling error")
-
- // Set the MIME type
- mimeType := o.getMimeTypeByExtension()
- ctx.Response.Header.SetContentType(mimeType)
-
- // Set ETag
- if cachedResponse.Exists {
- ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
- } else if res != nil {
- cachedResponse.ETag = res.Header.Peek(fasthttp.HeaderETag)
- ctx.Response.Header.SetBytesV(fasthttp.HeaderETag, cachedResponse.ETag)
+ // Set ETag & MIME
+ if eTag := header.Get(gitea.ETagHeader); eTag != "" {
+ ctx.RespWriter.Header().Set(gitea.ETagHeader, eTag)
}
-
- if ctx.Response.StatusCode() != fasthttp.StatusNotFound {
- // Everything's okay so far
- ctx.Response.SetStatusCode(fasthttp.StatusOK)
+ if cacheIndicator := header.Get(gitea.PagesCacheIndicatorHeader); cacheIndicator != "" {
+ ctx.RespWriter.Header().Set(gitea.PagesCacheIndicatorHeader, cacheIndicator)
}
- ctx.Response.Header.SetLastModified(o.BranchTimestamp)
+ if length := header.Get(gitea.ContentLengthHeader); length != "" {
+ ctx.RespWriter.Header().Set(gitea.ContentLengthHeader, length)
+ }
+ if mime := header.Get(gitea.ContentTypeHeader); mime == "" || o.ServeRaw {
+ ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, rawMime)
+ } else {
+ ctx.RespWriter.Header().Set(gitea.ContentTypeHeader, mime)
+ }
+ ctx.RespWriter.Header().Set(headerLastModified, o.BranchTimestamp.In(time.UTC).Format(time.RFC1123))
log.Debug().Msg("Prepare response")
- // Write the response body to the original request
- var cacheBodyWriter bytes.Buffer
- if res != nil {
- if res.Header.ContentLength() > fileCacheSizeLimit {
- // fasthttp else will set "Content-Length: 0"
- ctx.Response.SetBodyStream(&strings.Reader{}, -1)
+ ctx.RespWriter.WriteHeader(ctx.StatusCode)
- err = res.BodyWriteTo(ctx.Response.BodyWriter())
- } else {
- // TODO: cache is half-empty if request is cancelled - does the ctx.Err() below do the trick?
- err = res.BodyWriteTo(io.MultiWriter(ctx.Response.BodyWriter(), &cacheBodyWriter))
+ // Write the response body to the original request
+ if reader != nil {
+ _, err := io.Copy(ctx.RespWriter, reader)
+ if err != nil {
+ log.Error().Err(err).Msgf("Couldn't write body for %q", o.TargetPath)
+ html.ReturnErrorPage(ctx, "", http.StatusInternalServerError)
+ return true
}
- } else {
- _, err = ctx.Write(cachedResponse.Body)
- }
- if err != nil {
- log.Error().Err(err).Msgf("Couldn't write body for %q", uri)
- html.ReturnErrorPage(ctx, fasthttp.StatusInternalServerError)
- return true
}
log.Debug().Msg("Sending response")
- if res != nil && res.Header.ContentLength() <= fileCacheSizeLimit && ctx.Err() == nil {
- cachedResponse.Exists = true
- cachedResponse.MimeType = mimeType
- cachedResponse.Body = cacheBodyWriter.Bytes()
- _ = fileResponseCache.Set(uri+"?timestamp="+o.timestamp(), cachedResponse, fileCacheTimeout)
- }
-
return true
}
diff --git a/server/utils/utils.go b/server/utils/utils.go
index 7be330f..30f948d 100644
--- a/server/utils/utils.go
+++ b/server/utils/utils.go
@@ -1,9 +1,11 @@
package utils
-import "bytes"
+import (
+ "strings"
+)
-func TrimHostPort(host []byte) []byte {
- i := bytes.IndexByte(host, ':')
+func TrimHostPort(host string) string {
+ i := strings.IndexByte(host, ':')
if i >= 0 {
return host[:i]
}
diff --git a/server/utils/utils_test.go b/server/utils/utils_test.go
index 3dc0632..2532392 100644
--- a/server/utils/utils_test.go
+++ b/server/utils/utils_test.go
@@ -7,7 +7,7 @@ import (
)
func TestTrimHostPort(t *testing.T) {
- assert.EqualValues(t, "aa", TrimHostPort([]byte("aa")))
- assert.EqualValues(t, "", TrimHostPort([]byte(":")))
- assert.EqualValues(t, "example.com", TrimHostPort([]byte("example.com:80")))
+ assert.EqualValues(t, "aa", TrimHostPort("aa"))
+ assert.EqualValues(t, "", TrimHostPort(":"))
+ assert.EqualValues(t, "example.com", TrimHostPort("example.com:80"))
}
diff --git a/html/error.go b/html/error.go
index 325dada..826c42b 100644
--- a/html/error.go
+++ b/html/error.go
@@ -1,24 +1,45 @@
package html
import (
- "bytes"
+ "net/http"
"strconv"
+ "strings"
- "github.com/valyala/fasthttp"
+ "codeberg.org/codeberg/pages/server/context"
)
-// ReturnErrorPage sets the response status code and writes NotFoundPage to the response body, with "%status" replaced
-// with the provided status code.
-func ReturnErrorPage(ctx *fasthttp.RequestCtx, code int) {
- ctx.Response.SetStatusCode(code)
- ctx.Response.Header.SetContentType("text/html; charset=utf-8")
- message := fasthttp.StatusMessage(code)
- if code == fasthttp.StatusMisdirectedRequest {
- message += " - domain not specified in