feat: add option to log the n most active IPs each hour (#496)

Reviewed-on: https://codeberg.org/Codeberg/pages-server/pulls/496
Co-authored-by: crapStone <me@crapstone.dev>
Co-committed-by: crapStone <me@crapstone.dev>

cli/flags.go

@@ -115,6 +115,12 @@ var (
 			EnvVars: []string{"USE_PROXY_PROTOCOL"},
 			Value:   false,
 		},
+		&cli.UintFlag{
+			Name:    "log-most-active-ips",
+			Usage:   "logs a the n most active IPs every hour",
+			EnvVars: []string{"LOG_MOST_ACTIVE_IPS"},
+			Value:   10,
+		},
 
 		// Default branches to fetch assets from
 		&cli.StringSliceFlag{

config/config.go

@@ -14,6 +14,8 @@ type ServerConfig struct {
 	HttpPort           uint16 `default:"80"`
 	HttpServerEnabled  bool   `default:"true"`
 	UseProxyProtocol   bool   `default:"false"`
+	LogMostActiveIps   bool   `default:"false"`
+	MostActiveIpCount  uint   `default:"10"`
 	MainDomain         string
 	RawDomain          string
 	PagesBranches      []string

config/setup.go

@@ -72,6 +72,10 @@ func mergeServerConfig(ctx *cli.Context, config *ServerConfig) {
 	if ctx.IsSet("use-proxy-protocol") {
 		config.UseProxyProtocol = ctx.Bool("use-proxy-protocol")
 	}
+	if ctx.IsSet("log-most-active-ips") {
+		config.LogMostActiveIps = true
+		config.MostActiveIpCount = ctx.Uint("log-most-active-ips")
+	}
 
 	if ctx.IsSet("pages-domain") {
 		config.MainDomain = ctx.String("pages-domain")

config/setup_test.go

@@ -140,6 +140,9 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
 					Port:               8080,
 					HttpPort:           80,
 					HttpServerEnabled:  false,
+					UseProxyProtocol:   false,
+					LogMostActiveIps:   false,
+					MostActiveIpCount:  10,
 					MainDomain:         "original",
 					RawDomain:          "original",
 					PagesBranches:      []string{"original"},
@@ -180,6 +183,9 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
 					Port:               8443,
 					HttpPort:           443,
 					HttpServerEnabled:  true,
+					UseProxyProtocol:   true,
+					LogMostActiveIps:   true,
+					MostActiveIpCount:  42,
 					MainDomain:         "changed",
 					RawDomain:          "changed",
 					PagesBranches:      []string{"changed"},
@@ -227,6 +233,8 @@ func TestMergeConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *testing.T
 			"--port", "8443",
 			"--http-port", "443",
 			"--enable-http-server",
+			"--use-proxy-protocol",
+			"--log-most-active-ips", "42",
 			// Forge
 			"--forge-root", "changed",
 			"--forge-api-token", "changed",
@@ -277,6 +285,9 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
 					Port:               8080,
 					HttpPort:           80,
 					HttpServerEnabled:  false,
+					UseProxyProtocol:   false,
+					LogMostActiveIps:   false,
+					MostActiveIpCount:  10,
 					MainDomain:         "original",
 					RawDomain:          "original",
 					AllowedCorsDomains: []string{"original"},
@@ -290,6 +301,9 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
 					Port:               8443,
 					HttpPort:           443,
 					HttpServerEnabled:  true,
+					UseProxyProtocol:   true,
+					LogMostActiveIps:   true,
+					MostActiveIpCount:  21,
 					MainDomain:         "changed",
 					RawDomain:          "changed",
 					AllowedCorsDomains: fixArrayFromCtx(ctx, "allowed-cors-domains", []string{"changed"}),
@@ -309,6 +323,8 @@ func TestMergeServerConfigShouldReplaceAllExistingValuesGivenAllArgsExist(t *tes
 				"--port", "8443",
 				"--http-port", "443",
 				"--enable-http-server",
+				"--use-proxy-protocol",
+				"--log-most-active-ips", "21",
 			},
 		)
 	}
@@ -329,6 +345,11 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE
 		{args: []string{"--pages-branch", "changed"}, callback: func(sc *ServerConfig) { sc.PagesBranches = []string{"changed"} }},
 		{args: []string{"--allowed-cors-domains", "changed"}, callback: func(sc *ServerConfig) { sc.AllowedCorsDomains = []string{"changed"} }},
 		{args: []string{"--blacklisted-paths", "changed"}, callback: func(sc *ServerConfig) { sc.BlacklistedPaths = []string{"changed"} }},
+		{args: []string{"--use-proxy-protocol"}, callback: func(sc *ServerConfig) { sc.UseProxyProtocol = true }},
+		{args: []string{"--log-most-active-ips", "42"}, callback: func(sc *ServerConfig) {
+			sc.LogMostActiveIps = true
+			sc.MostActiveIpCount = 42
+		}},
 	}
 
 	for _, pair := range testValuePairs {
@@ -345,6 +366,9 @@ func TestMergeServerConfigShouldReplaceOnlyOneValueExistingValueGivenOnlyOneArgE
 					PagesBranches:      []string{"original"},
 					AllowedCorsDomains: []string{"original"},
 					BlacklistedPaths:   []string{"original"},
+					UseProxyProtocol:   false,
+					LogMostActiveIps:   false,
+					MostActiveIpCount:  10,
 				}
 
 				expectedConfig := cfg

flake.lock

@@ -20,11 +20,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1749285348,
+        "lastModified": 1749794982,
-        "narHash": "sha256-frdhQvPbmDYaScPFiCnfdh3B/Vh81Uuoo0w5TkWmmjU=",
+        "narHash": "sha256-Kh9K4taXbVuaLC0IL+9HcfvxsSUx8dPB5s5weJcc9pc=",
         "owner": "NixOS",
         "repo": "nixpkgs",
-        "rev": "3e3afe5174c561dee0df6f2c2b2236990146329f",
+        "rev": "ee930f9755f58096ac6e8ca94a1887e0534e2d81",
         "type": "github"
       },
       "original": {

server/handler/handler.go

@@ -1,8 +1,10 @@
 package handler
 
 import (
+	"net"
 	"net/http"
 	"strings"
+	"sync"
 
 	"github.com/rs/zerolog/log"
 
@@ -24,10 +26,28 @@ func Handler(
 	cfg config.ServerConfig,
 	giteaClient *gitea.Client,
 	canonicalDomainCache, redirectsCache cache.ICache,
+	mostActiveIpMap *sync.Map,
 ) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		ctx := context.New(w, req)
 		log := log.With().Str("ReqId", ctx.ReqId).Strs("Handler", []string{req.Host, req.RequestURI}).Logger()
+
+		if cfg.LogMostActiveIps {
+			ip, _, err := net.SplitHostPort(req.RemoteAddr)
+			if err != nil {
+				log.Warn().Err(err).Msg("Failed to get IP address. Using complete remoteAddr string")
+
+				ip = req.RemoteAddr
+			}
+
+			success := false
+			for !success {
+				value, _ := mostActiveIpMap.LoadOrStore(ip, uint(0))
+				count := value.(uint)
+				success = mostActiveIpMap.CompareAndSwap(ip, count, count+1)
+			}
+		}
+
 		log.Debug().Msg("\n----------------------------------------------------------")
 
 		ctx.RespWriter.Header().Set("Server", "pages-server")

server/handler/handler_test.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"net/http"
 	"net/http/httptest"
+	"sync"
 	"testing"
 	"time"
 
@@ -29,7 +30,7 @@ func TestHandlerPerformance(t *testing.T) {
 		AllowedCorsDomains: []string{"raw.codeberg.org", "fonts.codeberg.org", "design.codeberg.org"},
 		PagesBranches:      []string{"pages"},
 	}
-	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache())
+	testHandler := Handler(serverCfg, giteaClient, cache.NewInMemoryCache(), cache.NewInMemoryCache(), &sync.Map{})
 
 	testCase := func(uri string, status int) {
 		t.Run(uri, func(t *testing.T) {

server/startup.go

@@ -7,7 +7,9 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"sort"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pires/go-proxyproto"
@@ -135,8 +137,41 @@ func Serve(ctx *cli.Context) error {
 		StartProfilingServer(ctx.String("profiling-address"))
 	}
 
+	mostActiveIpMap := &sync.Map{}
+	if cfg.Server.LogMostActiveIps {
+		go func() {
+			ticker := time.NewTicker(1 * time.Hour)
+			for range ticker.C {
+				type kv struct {
+					Key   string
+					Value uint
+				}
+
+				var kvArray []kv
+				mostActiveIpMap.Range(func(k, v any) bool {
+					kvArray = append(kvArray, kv{k.(string), v.(uint)})
+					return true
+				})
+				mostActiveIpMap.Clear()
+
+				sort.Slice(kvArray, func(i, j int) bool {
+					return kvArray[i].Value > kvArray[j].Value
+				})
+
+				builder := strings.Builder{}
+				var item kv
+				for i := uint(0); i < cfg.Server.MostActiveIpCount; i++ {
+					item = kvArray[i]
+					builder.WriteString(fmt.Sprintf("\n%s, %d", item.Key, item.Value))
+				}
+
+				log.Log().Msg(fmt.Sprintf("%d most active IPs:%s", cfg.Server.MostActiveIpCount, builder.String()))
+			}
+		}()
+	}
+
 	// Create ssl handler based on settings
-	sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache)
+	sslHandler := handler.Handler(cfg.Server, giteaClient, canonicalDomainCache, redirectsCache, mostActiveIpMap)
 
 	// Start the ssl listener
 	log.Info().Msgf("Start SSL server using TCP listener on %s", listener.Addr())