refactor (2) (#34)

move forward with refactoring:
 - initial implementation of a smal "gitea client for fasthttp"
 - move constant into const.go

Co-authored-by: 6543 <6543@obermui.de>
Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/34
Reviewed-by: Otto Richter <otto@codeberg.org>

server/upstream/const.go

@@ -19,3 +19,5 @@ var fileCacheSizeLimit = 1024 * 1024
 
 // canonicalDomainCacheTimeout specifies the timeout for the canonical domain cache.
 var canonicalDomainCacheTimeout = 15 * time.Minute
+
+const canonicalDomainConfig = ".domains"

server/upstream/domains.go

@@ -3,15 +3,19 @@ package upstream
 import (
 	"strings"
 
-	"github.com/valyala/fasthttp"
-
 	"codeberg.org/codeberg/pages/server/cache"
 )
 
 // CheckCanonicalDomain returns the canonical domain specified in the repo (using the `.domains` file).
 func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
-	domains := []string{}
+	return checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken, canonicalDomainCache)
-	valid := false
+}
+
+func checkCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, mainDomainSuffix, giteaRoot, giteaAPIToken string, canonicalDomainCache cache.SetGetKey) (string, bool) {
+	var (
+		domains []string
+		valid   bool
+	)
 	if cachedValue, ok := canonicalDomainCache.Get(targetOwner + "/" + targetRepo + "/" + targetBranch); ok {
 		domains = cachedValue.([]string)
 		for _, domain := range domains {
@@ -21,13 +25,9 @@ func CheckCanonicalDomain(targetOwner, targetRepo, targetBranch, actualDomain, m
 			}
 		}
 	} else {
-		req := fasthttp.AcquireRequest()
+		body, err := giteaRawContent(giteaRoot, targetRepo, targetBranch, giteaRoot, giteaAPIToken, canonicalDomainConfig)
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + targetOwner + "/" + targetRepo + "/raw/" + targetBranch + "/.domains" + "?access_token=" + giteaAPIToken)
+		if err == nil {
-		res := fasthttp.AcquireResponse()
+			for _, domain := range strings.Split(string(body), "\n") {
-
-		err := client.Do(req, res)
-		if err == nil && res.StatusCode() == fasthttp.StatusOK {
-			for _, domain := range strings.Split(string(res.Body()), "\n") {
 				domain = strings.ToLower(domain)
 				domain = strings.TrimSpace(domain)
 				domain = strings.TrimPrefix(domain, "http://")

server/upstream/gitea.go

@@ -0,0 +1,67 @@
+package upstream
+
+import (
+	"fmt"
+	"net/url"
+	"path"
+	"time"
+
+	"github.com/valyala/fasthttp"
+	"github.com/valyala/fastjson"
+)
+
+const giteaAPIRepos = "/api/v1/repos/"
+
+// TODOs:
+// * own client to store token & giteaRoot
+// * handle 404 -> page will show 500 atm
+
+func giteaRawContent(targetOwner, targetRepo, ref, giteaRoot, giteaAPIToken, resource string) ([]byte, error) {
+	req := fasthttp.AcquireRequest()
+
+	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, targetOwner, targetRepo, "raw", resource+"?ref="+url.QueryEscape(ref)))
+	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	if err := getFastHTTPClient(10*time.Second).Do(req, res); err != nil {
+		return nil, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return nil, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return res.Body(), nil
+}
+
+func giteaGetRepoBranchTimestamp(giteaRoot, repoOwner, repoName, branchName, giteaAPIToken string) (time.Time, error) {
+	client := getFastHTTPClient(5 * time.Second)
+
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName, "branches", branchName))
+	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	if err := client.Do(req, res); err != nil {
+		return time.Time{}, err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return time.Time{}, fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return time.Parse(time.RFC3339, fastjson.GetString(res.Body(), "commit", "timestamp"))
+}
+
+func giteaGetRepoDefaultBranch(giteaRoot, repoOwner, repoName, giteaAPIToken string) (string, error) {
+	client := getFastHTTPClient(5 * time.Second)
+
+	req := fasthttp.AcquireRequest()
+	req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, repoOwner, repoName))
+	req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
+	res := fasthttp.AcquireResponse()
+
+	if err := client.Do(req, res); err != nil {
+		return "", err
+	}
+	if res.StatusCode() != fasthttp.StatusOK {
+		return "", fmt.Errorf("unexpected status code '%d'", res.StatusCode())
+	}
+	return fastjson.GetString(res.Body(), "default_branch"), nil
+}

server/upstream/helper.go

@@ -3,9 +3,6 @@ package upstream
 import (
 	"time"
 
-	"github.com/valyala/fasthttp"
-	"github.com/valyala/fastjson"
-
 	"codeberg.org/codeberg/pages/server/cache"
 )
 
@@ -16,34 +13,31 @@ type branchTimestamp struct {
 
 // GetBranchTimestamp finds the default branch (if branch is "") and returns the last modification time of the branch
 // (or nil if the branch doesn't exist)
-func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaApiToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
+func GetBranchTimestamp(owner, repo, branch, giteaRoot, giteaAPIToken string, branchTimestampCache cache.SetGetKey) *branchTimestamp {
 	if result, ok := branchTimestampCache.Get(owner + "/" + repo + "/" + branch); ok {
 		if result == nil {
 			return nil
 		}
 		return result.(*branchTimestamp)
 	}
-	result := &branchTimestamp{}
+	result := &branchTimestamp{
-	result.Branch = branch
+		Branch: branch,
-	if branch == "" {
+	}
+	if len(branch) == 0 {
 		// Get default branch
-		body := make([]byte, 0)
+		defaultBranch, err := giteaGetRepoDefaultBranch(giteaRoot, owner, repo, giteaAPIToken)
-		// TODO: use header for API key?
+		if err != nil {
-		status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"?access_token="+giteaApiToken, 5*time.Second)
+			_ = branchTimestampCache.Set(owner+"/"+repo+"/", nil, defaultBranchCacheTimeout)
-		if err != nil || status != 200 {
-			_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, nil, defaultBranchCacheTimeout)
 			return nil
 		}
-		result.Branch = fastjson.GetString(body, "default_branch")
+		result.Branch = defaultBranch
 	}
 
-	body := make([]byte, 0)
+	timestamp, err := giteaGetRepoBranchTimestamp(giteaRoot, owner, repo, branch, giteaAPIToken)
-	status, body, err := fasthttp.GetTimeout(body, giteaRoot+"/api/v1/repos/"+owner+"/"+repo+"/branches/"+branch+"?access_token="+giteaApiToken, 5*time.Second)
+	if err != nil {
-	if err != nil || status != 200 {
 		return nil
 	}
-
+	result.Timestamp = timestamp
-	result.Timestamp, _ = time.Parse(time.RFC3339, fastjson.GetString(body, "commit", "timestamp"))
 	_ = branchTimestampCache.Set(owner+"/"+repo+"/"+branch, result, branchExistenceCacheTimeout)
 	return result
 }

server/upstream/upstream.go

@@ -38,11 +38,13 @@ type Options struct {
 	redirectIfExists    string
 }
 
-var client = fasthttp.Client{
+func getFastHTTPClient(timeout time.Duration) *fasthttp.Client {
-	ReadTimeout:        10 * time.Second,
+	return &fasthttp.Client{
-	MaxConnDuration:    60 * time.Second,
+		ReadTimeout:        timeout,
-	MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnDuration:    60 * time.Second,
-	MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+		MaxConnWaitTimeout: 1000 * time.Millisecond,
+		MaxConnsPerHost:    128 * 16, // TODO: adjust bottlenecks for best performance with Gitea!
+	}
 }
 
 // Upstream requests a file from the Gitea API at GiteaRoot and writes it to the request context.
@@ -80,7 +82,7 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 	log.Debug().Msg("preparations")
 
 	// Make a GET request to the upstream URL
-	uri := o.TargetOwner + "/" + o.TargetRepo + "/raw/" + o.TargetBranch + "/" + o.TargetPath
+	uri := path.Join(o.TargetOwner, o.TargetRepo, "raw", o.TargetBranch, o.TargetPath)
 	var req *fasthttp.Request
 	var res *fasthttp.Response
 	var cachedResponse fileResponse
@@ -89,10 +91,11 @@ func (o *Options) Upstream(ctx *fasthttp.RequestCtx, giteaRoot, giteaAPIToken st
 		cachedResponse = cachedValue.(fileResponse)
 	} else {
 		req = fasthttp.AcquireRequest()
-		req.SetRequestURI(giteaRoot + "/api/v1/repos/" + uri + "?access_token=" + giteaAPIToken)
+		req.SetRequestURI(path.Join(giteaRoot, giteaAPIRepos, uri))
+		req.Header.Set(fasthttp.HeaderAuthorization, giteaAPIToken)
 		res = fasthttp.AcquireResponse()
 		res.SetBodyStream(&strings.Reader{}, -1)
-		err = client.Do(req, res)
+		err = getFastHTTPClient(10*time.Second).Do(req, res)
 	}
 	log.Debug().Msg("acquisition")