s83/git-pages
1
0
Fork 0
mirror of https://codeberg.org/Codeberg/pages-server.git synced 2025-06-19 23:22:40 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: use a in memory map to count requests for ips and log the n most active each hour

Browse source
This commit is contained in:
crapStone 2025-06-13 23:18:01 +02:00
parent 023ea17492
commit b22d3665a9
No known key found for this signature in database
GPG key ID: 22D4BF0CF7CC29C8
7 changed files with 74 additions and 20 deletions
Download patch file Download diff file Expand all files Collapse all files

10
cli/flags.go
View file

@ -115,11 +115,11 @@ var (
EnvVars: []string{"USE_PROXY_PROTOCOL"},
Value: false,
},
&cli.BoolFlag{
Name: "log-every-request",
Usage: "logs every request with reqID, host and path",
EnvVars: []string{"LOG_EVERY_REQUEST"},
Value: false,
&cli.UintFlag{
Name: "log-most-active-ips",
Usage: "logs a the n most active IPs every hour",
EnvVars: []string{"LOG_MOST_ACTIVE_IPS"},
Value: 10,
},
// Default branches to fetch assets from

3
config/config.go
View file

@ -14,7 +14,8 @@ type ServerConfig struct {
HttpPort uint16 `default:"80"`
HttpServerEnabled bool `default:"true"`
UseProxyProtocol bool `default:"false"`
LogEveryRequest bool `default:"false"`
LogMostActiveIps bool `default:"false"`
MostActiveIpCount uint `default:"10"`
MainDomain string
RawDomain string
PagesBranches []string

5
config/setup.go
View file

@ -72,8 +72,9 @@ func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
if ctx.IsSet("use-proxy-protocol") {
config.UseProxyProtocol = ctx.Bool("use-proxy-protocol")
}
if ctx.IsSet("log-every-request") {
config.LogEveryRequest = ctx.Bool("log-every-request")
if ctx.IsSet("log-most-active-ips") {
config.LogMostActiveIps = true
config.MostActiveIpCount = ctx.Uint("log-most-active-ips")
}
if ctx.IsSet("pages-domain") {

24
config/setup_test.go
View file

@ -141,7 +141,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
HttpPort: 80,
HttpServerEnabled: false,
UseProxyProtocol: false,
LogEveryRequest: false,
LogMostActiveIps: false,
MostActiveIpCount: 10,
MainDomain: "original",
RawDomain: "original",
PagesBranches: []string{"original"},
@ -183,7 +184,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
HttpPort: 443,
HttpServerEnabled: true,
UseProxyProtocol: true,
LogEveryRequest: true,
LogMostActiveIps: true,
MostActiveIpCount: 42,
MainDomain: "changed",
RawDomain: "changed",
PagesBranches: []string{"changed"},
@ -232,7 +234,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
"--http-port", "443",
"--enable-http-server",
"--use-proxy-protocol",
"--log-every-request",
"--log-most-active-ips", "42",
// Forge
"--forge-root", "changed",
"--forge-api-token", "changed",
@ -284,7 +286,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
HttpPort: 80,
HttpServerEnabled: false,
UseProxyProtocol: false,
LogEveryRequest: false,
LogMostActiveIps: false,
MostActiveIpCount: 10,
MainDomain: "original",
RawDomain: "original",
AllowedCorsDomains: []string{"original"},
@ -299,7 +302,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
HttpPort: 443,
HttpServerEnabled: true,
UseProxyProtocol: true,
LogEveryRequest: true,
LogMostActiveIps: true,
MostActiveIpCount: 21,
MainDomain: "changed",
RawDomain: "changed",
AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}),
@ -320,7 +324,7 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
"--http-port", "443",
"--enable-http-server",
"--use-proxy-protocol",
"--log-every-request",
"--log-most-active-ips", "21",
},
)
}
@ -342,7 +346,10 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE
{args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }},
{args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }},
{args: []string{"--use-proxy-protocol"}, callback: func(sc *ServerConfig) { sc.UseProxyProtocol = true }},
{args: []string{"--log-every-request"}, callback: func(sc *ServerConfig) { sc.LogEveryRequest = true }},
{args: []string{"--log-most-active-ips", "42"}, callback: func(sc *ServerConfig) {
sc.LogMostActiveIps = true
sc.MostActiveIpCount = 42
}},
}
for _, pair := range testValuePairs {
@ -360,7 +367,8 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE
AllowedCorsDomains: []string{"original"},
BlacklistedPaths: []string{"original"},
UseProxyProtocol: false,
LogEveryRequest: false,
LogMostActiveIps: false,
MostActiveIpCount: 10,
}
expectedConfig := cfg

12
server/handler/handler.go
View file

@ -3,6 +3,7 @@ package handler
import (
"net/http"
"strings"
"sync"
"github.com/rs/zerolog/log"
@ -24,14 +25,21 @@ func Handler(
cfg config.ServerConfig,
giteaClient *gitea.Client,
canonicalDomainCache, redirectsCache cache.ICache,
mostActiveIpMap *sync.Map,
) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
ctx := context.New(w, req)
log := log.With().Str("ReqId", ctx.ReqId).Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
if cfg.LogEveryRequest {
log.Log().Str("IP", req.RemoteAddr).Msg("new request")
if cfg.LogMostActiveIps {
success := false
for !success {
value, _ := mostActiveIpMap.LoadOrStore(req.RemoteAddr, uint(0))
count := value.(uint)
success = mostActiveIpMap.CompareAndSwap(req.RemoteAddr, count, count+1)
}
}
log.Debug().Msg("\n----------------------------------------------------------")
ctx.RespWriter.Header().Set("Server", "pages-server")

3
server/handler/handler_test.go
View file

@ -3,6 +3,7 @@ package handler
import (
"net/http"
"net/http/httptest"
"sync"
"testing"
"time"
@ -29,7 +30,7 @@ func TestHandlerPerformance(t *testing.T) {
AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
PagesBranches: []string{"pages"},
}
testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache())
testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), &sync.Map{})
testCase := func(uri string, status int) {
t.Run(uri, func(t *testing.T) {

37
server/startup.go
View file

@ -7,7 +7,9 @@ import (
"net"
"net/http"
"os"
"sort"
"strings"
"sync"
"time"
"github.com/pires/go-proxyproto"
@ -135,8 +137,41 @@ func Serve(ctx *cli.Context) error {
StartProfilingServer(ctx.String("profiling-address"))
}
mostActiveIpMap := &sync.Map{}
if cfg.Server.LogMostActiveIps {
go func() {
ticker := time.NewTicker(1 * time.Hour)
for range ticker.C {
type kv struct {
Key string
Value uint
}
var kvArray []kv
mostActiveIpMap.Range(func(k, v any) bool {
kvArray = append(kvArray, kv{k.(string), v.(uint)})
return true
})
mostActiveIpMap.Clear()
sort.Slice(kvArray, func(i, j int) bool {
return kvArray[i].Value > kvArray[j].Value
})
builder := strings.Builder{}
var item kv
for i := uint(0); i < cfg.Server.MostActiveIpCount; i++ {
item = kvArray[i]
builder.WriteString(fmt.Sprintf("\n%s, %d", item.Key, item.Value))
}
log.Log().Msg(fmt.Sprintf("%d most active IPs:%s", cfg.Server.MostActiveIpCount, builder.String()))
}
}()
}
// Create ssl handler based on settings
sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache)
sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache, mostActiveIpMap)
// Start the ssl listener
log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())