feat: use a in memory map to count requests for ips and log the n most active each hour

cli/flags.go

@@ -115,11 +115,11 @@ var (
 			EnvVars: []string{"USE_PROXY_PROTOCOL"},
 			Value:   false,
 		},
-		&cli.BoolFlag{
+		&cli.UintFlag{
-			Name:    "log-every-request",
+			Name:    "log-most-active-ips",
-			Usage:   "logs every request with reqID, host and path",
+			Usage:   "logs a the n most active IPs every hour",
-			EnvVars: []string{"LOG_EVERY_REQUEST"},
+			EnvVars: []string{"LOG_MOST_ACTIVE_IPS"},
-			Value:   false,
+			Value:   10,
 		},
 
 		// Default branches to fetch assets from

config/config.go

@@ -14,7 +14,8 @@ type ServerConfig struct {
 	HttpPort           uint16 `default:"80"`
 	HttpServerEnabled  bool   `default:"true"`
 	UseProxyProtocol   bool   `default:"false"`
-	LogEveryRequest    bool   `default:"false"`
+	LogMostActiveIps   bool   `default:"false"`
+	MostActiveIpCount  uint   `default:"10"`
 	MainDomain         string
 	RawDomain          string
 	PagesBranches      []string

config/setup.go

@@ -72,8 +72,9 @@ func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
 	if ctx.IsSet("use-proxy-protocol") {
 		config.UseProxyProtocol = ctx.Bool("use-proxy-protocol")
 	}
-	if ctx.IsSet("log-every-request") {
+	if ctx.IsSet("log-most-active-ips") {
-		config.LogEveryRequest = ctx.Bool("log-every-request")
+		config.LogMostActiveIps = true
+		config.MostActiveIpCount = ctx.Uint("log-most-active-ips")
 	}
 
 	if ctx.IsSet("pages-domain") {

config/setup_test.go

@@ -141,7 +141,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
 					HttpPort:           80,
 					HttpServerEnabled:  false,
 					UseProxyProtocol:   false,
-					LogEveryRequest:    false,
+					LogMostActiveIps:   false,
+					MostActiveIpCount:  10,
 					MainDomain:         "original",
 					RawDomain:          "original",
 					PagesBranches:      []string{"original"},
@@ -183,7 +184,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
 					HttpPort:           443,
 					HttpServerEnabled:  true,
 					UseProxyProtocol:   true,
-					LogEveryRequest:    true,
+					LogMostActiveIps:   true,
+					MostActiveIpCount:  42,
 					MainDomain:         "changed",
 					RawDomain:          "changed",
 					PagesBranches:      []string{"changed"},
@@ -232,7 +234,7 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
 			"--http-port", "443",
 			"--enable-http-server",
 			"--use-proxy-protocol",
-			"--log-every-request",
+			"--log-most-active-ips", "42",
 			// Forge
 			"--forge-root", "changed",
 			"--forge-api-token", "changed",
@@ -284,7 +286,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
 					HttpPort:           80,
 					HttpServerEnabled:  false,
 					UseProxyProtocol:   false,
-					LogEveryRequest:    false,
+					LogMostActiveIps:   false,
+					MostActiveIpCount:  10,
 					MainDomain:         "original",
 					RawDomain:          "original",
 					AllowedCorsDomains: []string{"original"},
@@ -299,7 +302,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
 					HttpPort:           443,
 					HttpServerEnabled:  true,
 					UseProxyProtocol:   true,
-					LogEveryRequest:    true,
+					LogMostActiveIps:   true,
+					MostActiveIpCount:  21,
 					MainDomain:         "changed",
 					RawDomain:          "changed",
 					AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}),
@@ -320,7 +324,7 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
 				"--http-port", "443",
 				"--enable-http-server",
 				"--use-proxy-protocol",
-				"--log-every-request",
+				"--log-most-active-ips", "21",
 			},
 		)
 	}
@@ -342,7 +346,10 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE
 		{args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }},
 		{args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }},
 		{args: []string{"--use-proxy-protocol"}, callback: func(sc *ServerConfig) { sc.UseProxyProtocol = true }},
-		{args: []string{"--log-every-request"}, callback: func(sc *ServerConfig) { sc.LogEveryRequest = true }},
+		{args: []string{"--log-most-active-ips", "42"}, callback: func(sc *ServerConfig) {
+			sc.LogMostActiveIps = true
+			sc.MostActiveIpCount = 42
+		}},
 	}
 
 	for _, pair := range testValuePairs {
@@ -360,7 +367,8 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE
 					AllowedCorsDomains: []string{"original"},
 					BlacklistedPaths:   []string{"original"},
 					UseProxyProtocol:   false,
-					LogEveryRequest:    false,
+					LogMostActiveIps:   false,
+					MostActiveIpCount:  10,
 				}
 
 				expectedConfig := cfg

server/handler/handler.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"net/http"
 	"strings"
+	"sync"
 
 	"github.com/rs/zerolog/log"
 
@@ -24,14 +25,21 @@ func Handler(
 	cfg config.ServerConfig,
 	giteaClient *gitea.Client,
 	canonicalDomainCache, redirectsCache cache.ICache,
+	mostActiveIpMap *sync.Map,
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		ctx := context.New(w, req)
 		log := log.With().Str("ReqId", ctx.ReqId).Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
 
-		if cfg.LogEveryRequest {
+		if cfg.LogMostActiveIps {
-			log.Log().Str("IP", req.RemoteAddr).Msg("new request")
+			success := false
+			for !success {
+				value, _ := mostActiveIpMap.LoadOrStore(req.RemoteAddr, uint(0))
+				count := value.(uint)
+				success = mostActiveIpMap.CompareAndSwap(req.RemoteAddr, count, count+1)
+			}
 		}
+
 		log.Debug().Msg("\n----------------------------------------------------------")
 
 		ctx.RespWriter.Header().Set("Server", "pages-server")

server/handler/handler_test.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"net/http"
 	"net/http/httptest"
+	"sync"
 	"testing"
 	"time"
 
@@ -29,7 +30,7 @@ func TestHandlerPerformance(t *testing.T) {
 		AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
 		PagesBranches:      []string{"pages"},
 	}
-	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache())
+	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), &sync.Map{})
 
 	testCase := func(uri string, status int) {
 		t.Run(uri, func(t *testing.T) {

server/startup.go

@@ -7,7 +7,9 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pires/go-proxyproto"
@@ -135,8 +137,41 @@ func Serve(ctx *cli.Context) error {
 		StartProfilingServer(ctx.String("profiling-address"))
 	}
 
+	mostActiveIpMap := &sync.Map{}
+	if cfg.Server.LogMostActiveIps {
+		go func() {
+			ticker := time.NewTicker(1 * time.Hour)
+			for range ticker.C {
+				type kv struct {
+					Key   string
+					Value uint
+				}
+
+				var kvArray []kv
+				mostActiveIpMap.Range(func(k, v any) bool {
+					kvArray = append(kvArray, kv{k.(string), v.(uint)})
+					return true
+				})
+				mostActiveIpMap.Clear()
+
+				sort.Slice(kvArray, func(i, j int) bool {
+					return kvArray[i].Value > kvArray[j].Value
+				})
+
+				builder := strings.Builder{}
+				var item kv
+				for i := uint(0); i < cfg.Server.MostActiveIpCount; i++ {
+					item = kvArray[i]
+					builder.WriteString(fmt.Sprintf("\n%s, %d", item.Key, item.Value))
+				}
+
+				log.Log().Msg(fmt.Sprintf("%d most active IPs:%s", cfg.Server.MostActiveIpCount, builder.String()))
+			}
+		}()
+	}
+
 	// Create ssl handler based on settings
-	sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache)
+	sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache, mostActiveIpMap)
 
 	// Start the ssl listener
 	log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())