From b22d3665a9d6b3fc099d1bf1d96364bd10689aa2 Mon Sep 17 00:00:00 2001 From: crapStone Date: Fri, 13 Jun 2025 23:18:01 +0200 Subject: [PATCH] feat: use a in memory map to count requests for ips and log the n most active each hour --- cli/flags.go | 10 ++++----- config/config.go | 3 ++- config/setup.go | 5 +++-- config/setup_test.go | 24 ++++++++++++++-------- server/handler/handler.go | 12 +++++++++-- server/handler/handler_test.go | 3 ++- server/startup.go | 37 +++++++++++++++++++++++++++++++++- 7 files changed, 74 insertions(+), 20 deletions(-) diff --git a/cli/flags.go b/cli/flags.go index 7f41d47..0260d24 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -115,11 +115,11 @@ var ( EnvVars: []string{"USE_PROXY_PROTOCOL"}, Value: false, }, - &cli.BoolFlag{ - Name: "log-every-request", - Usage: "logs every request with reqID, host and path", - EnvVars: []string{"LOG_EVERY_REQUEST"}, - Value: false, + &cli.UintFlag{ + Name: "log-most-active-ips", + Usage: "logs a the n most active IPs every hour", + EnvVars: []string{"LOG_MOST_ACTIVE_IPS"}, + Value: 10, }, // Default branches to fetch assets from diff --git a/config/config.go b/config/config.go index 0a226b7..0901bb7 100644 --- a/config/config.go +++ b/config/config.go @@ -14,7 +14,8 @@ type ServerConfig struct { HttpPort uint16 `default:"80"` HttpServerEnabled bool `default:"true"` UseProxyProtocol bool `default:"false"` - LogEveryRequest bool `default:"false"` + LogMostActiveIps bool `default:"false"` + MostActiveIpCount uint `default:"10"` MainDomain string RawDomain string PagesBranches []string diff --git a/config/setup.go b/config/setup.go index b111eab..a0307a1 100644 --- a/config/setup.go +++ b/config/setup.go @@ -72,8 +72,9 @@ func mergeServerConfig(ctx *cli.Context, config *ServerConfig) { if ctx.IsSet("use-proxy-protocol") { config.UseProxyProtocol = ctx.Bool("use-proxy-protocol") } - if ctx.IsSet("log-every-request") { - config.LogEveryRequest = ctx.Bool("log-every-request") + if ctx.IsSet("log-most-active-ips") { + config.LogMostActiveIps = true + config.MostActiveIpCount = ctx.Uint("log-most-active-ips") } if ctx.IsSet("pages-domain") { diff --git a/config/setup_test.go b/config/setup_test.go index 91e82bb..9e6b886 100644 --- a/config/setup_test.go +++ b/config/setup_test.go @@ -141,7 +141,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T HttpPort: 80, HttpServerEnabled: false, UseProxyProtocol: false, - LogEveryRequest: false, + LogMostActiveIps: false, + MostActiveIpCount: 10, MainDomain: "original", RawDomain: "original", PagesBranches: []string{"original"}, @@ -183,7 +184,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T HttpPort: 443, HttpServerEnabled: true, UseProxyProtocol: true, - LogEveryRequest: true, + LogMostActiveIps: true, + MostActiveIpCount: 42, MainDomain: "changed", RawDomain: "changed", PagesBranches: []string{"changed"}, @@ -232,7 +234,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T "--http-port", "443", "--enable-http-server", "--use-proxy-protocol", - "--log-every-request", + "--log-most-active-ips", "42", // Forge "--forge-root", "changed", "--forge-api-token", "changed", @@ -284,7 +286,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes HttpPort: 80, HttpServerEnabled: false, UseProxyProtocol: false, - LogEveryRequest: false, + LogMostActiveIps: false, + MostActiveIpCount: 10, MainDomain: "original", RawDomain: "original", AllowedCorsDomains: []string{"original"}, @@ -299,7 +302,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes HttpPort: 443, HttpServerEnabled: true, UseProxyProtocol: true, - LogEveryRequest: true, + LogMostActiveIps: true, + MostActiveIpCount: 21, MainDomain: "changed", RawDomain: "changed", AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}), @@ -320,7 +324,7 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes "--http-port", "443", "--enable-http-server", "--use-proxy-protocol", - "--log-every-request", + "--log-most-active-ips", "21", }, ) } @@ -342,7 +346,10 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE {args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }}, {args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }}, {args: []string{"--use-proxy-protocol"}, callback: func(sc *ServerConfig) { sc.UseProxyProtocol = true }}, - {args: []string{"--log-every-request"}, callback: func(sc *ServerConfig) { sc.LogEveryRequest = true }}, + {args: []string{"--log-most-active-ips", "42"}, callback: func(sc *ServerConfig) { + sc.LogMostActiveIps = true + sc.MostActiveIpCount = 42 + }}, } for _, pair := range testValuePairs { @@ -360,7 +367,8 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE AllowedCorsDomains: []string{"original"}, BlacklistedPaths: []string{"original"}, UseProxyProtocol: false, - LogEveryRequest: false, + LogMostActiveIps: false, + MostActiveIpCount: 10, } expectedConfig := cfg diff --git a/server/handler/handler.go b/server/handler/handler.go index 254f40b..7f53a8d 100644 --- a/server/handler/handler.go +++ b/server/handler/handler.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "strings" + "sync" "github.com/rs/zerolog/log" @@ -24,14 +25,21 @@ func Handler( cfg config.ServerConfig, giteaClient *gitea.Client, canonicalDomainCache, redirectsCache cache.ICache, + mostActiveIpMap *sync.Map, ) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { ctx := context.New(w, req) log := log.With().Str("ReqId", ctx.ReqId).Strs("Handler", []string{req.Host, req.RequestURI}).Logger() - if cfg.LogEveryRequest { - log.Log().Str("IP", req.RemoteAddr).Msg("new request") + if cfg.LogMostActiveIps { + success := false + for !success { + value, _ := mostActiveIpMap.LoadOrStore(req.RemoteAddr, uint(0)) + count := value.(uint) + success = mostActiveIpMap.CompareAndSwap(req.RemoteAddr, count, count+1) + } } + log.Debug().Msg("\n----------------------------------------------------------") ctx.RespWriter.Header().Set("Server", "pages-server") diff --git a/server/handler/handler_test.go b/server/handler/handler_test.go index 765b3b1..998ef0d 100644 --- a/server/handler/handler_test.go +++ b/server/handler/handler_test.go @@ -3,6 +3,7 @@ package handler import ( "net/http" "net/http/httptest" + "sync" "testing" "time" @@ -29,7 +30,7 @@ func TestHandlerPerformance(t *testing.T) { AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"}, PagesBranches: []string{"pages"}, } - testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache()) + testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), &sync.Map{}) testCase := func(uri string, status int) { t.Run(uri, func(t *testing.T) { diff --git a/server/startup.go b/server/startup.go index 4ae26c1..cdd87c2 100644 --- a/server/startup.go +++ b/server/startup.go @@ -7,7 +7,9 @@ import ( "net" "net/http" "os" + "sort" "strings" + "sync" "time" "github.com/pires/go-proxyproto" @@ -135,8 +137,41 @@ func Serve(ctx *cli.Context) error { StartProfilingServer(ctx.String("profiling-address")) } + mostActiveIpMap := &sync.Map{} + if cfg.Server.LogMostActiveIps { + go func() { + ticker := time.NewTicker(1 * time.Hour) + for range ticker.C { + type kv struct { + Key string + Value uint + } + + var kvArray []kv + mostActiveIpMap.Range(func(k, v any) bool { + kvArray = append(kvArray, kv{k.(string), v.(uint)}) + return true + }) + mostActiveIpMap.Clear() + + sort.Slice(kvArray, func(i, j int) bool { + return kvArray[i].Value > kvArray[j].Value + }) + + builder := strings.Builder{} + var item kv + for i := uint(0); i < cfg.Server.MostActiveIpCount; i++ { + item = kvArray[i] + builder.WriteString(fmt.Sprintf("\n%s, %d", item.Key, item.Value)) + } + + log.Log().Msg(fmt.Sprintf("%d most active IPs:%s", cfg.Server.MostActiveIpCount, builder.String())) + } + }() + } + // Create ssl handler based on settings - sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache) + sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache, mostActiveIpMap) // Start the ssl listener log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())