From 5016f40edb0cda08b8e58dee665221191f2ed9f3 Mon Sep 17 00:00:00 2001 From: Solderpunk Date: Thu, 16 Mar 2023 20:27:45 +0100 Subject: [PATCH 1/6] Initial implementation of leaky bucket rate limiting. --- handler.go | 11 ++++++++++- launch.go | 3 ++- ratelim.go | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 2 deletions(-) create mode 100644 ratelim.go diff --git a/handler.go b/handler.go index b1018c7..26ba82a 100644 --- a/handler.go +++ b/handler.go @@ -36,7 +36,7 @@ func isSubdir(subdir, superdir string) (bool, error) { return false, nil } -func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, accessLogEntries chan LogEntry, wg *sync.WaitGroup) { +func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, accessLogEntries chan LogEntry, rl *RateLimiter, wg *sync.WaitGroup) { defer conn.Close() defer wg.Done() var tlsConn (*tls.Conn) = conn.(*tls.Conn) @@ -49,6 +49,15 @@ func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, defer func() { accessLogEntries <- logEntry }() } + // Enforce rate limiting + noPort := logEntry.RemoteAddr.String() + noPort = noPort[0:strings.LastIndex(noPort, ":")] + if !rl.Allowed(noPort) { + conn.Write([]byte("44 10 second cool down, please!\r\n")) + logEntry.Status = 44 + return + } + // Read request URL, err := readRequest(conn, &logEntry) if err != nil { diff --git a/launch.go b/launch.go index 5c14d52..8c302b1 100644 --- a/launch.go +++ b/launch.go @@ -159,11 +159,12 @@ func launch(sysConfig SysConfig, userConfig UserConfig, privInfo userInfo) int { // Infinite serve loop (SIGTERM breaks out) running := true var wg sync.WaitGroup + rl := newRateLimiter(100, 5) for running { conn, err := listener.Accept() if err == nil { wg.Add(1) - go handleGeminiRequest(conn, sysConfig, userConfig, accessLogEntries, &wg) + go handleGeminiRequest(conn, sysConfig, userConfig, accessLogEntries, &rl, &wg) } else { select { case <-shutdown: diff --git a/ratelim.go b/ratelim.go new file mode 100644 index 0000000..6aa08dc --- /dev/null +++ b/ratelim.go @@ -0,0 +1,54 @@ +package main + +import ( + "fmt" + "sync" + "time" +) + +type RateLimiter struct { + mu sync.Mutex + bucket map[string]int + capacity int + rate int +} + +func newRateLimiter(capacity int, rate int) RateLimiter { + var rl = new(RateLimiter) + rl.bucket = make(map[string]int) + rl.capacity = capacity + rl.rate = rate + // Leak periodically + go func () { + for(true) { + fmt.Println(rl.bucket) + rl.mu.Lock() + for addr, drips := range rl.bucket { + if drips <= rate { + delete(rl.bucket, addr) + } else { + rl.bucket[addr] = drips - rl.rate + } + } + rl.mu.Unlock() + time.Sleep(time.Second) + } + }() + return *rl +} + +func (rl *RateLimiter) Allowed(addr string) bool { + rl.mu.Lock() + defer rl.mu.Unlock() + drips, present := rl.bucket[addr] + if !present { + rl.bucket[addr] = 1 + return true + } + if drips == rl.capacity { + return false + } + rl.bucket[addr] = drips + 1 + return true +} + From a6170a355d2b3f96d0fea7bf426f8920c9f0d3dd Mon Sep 17 00:00:00 2001 From: Solderpunk Date: Fri, 17 Mar 2023 19:52:39 +0100 Subject: [PATCH 2/6] Make rate limiting configurable. --- config.go | 6 ++++++ handler.go | 14 ++++++++------ launch.go | 2 +- ratelim.go | 10 ++++------ 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/config.go b/config.go index 4f28699..8e2e749 100644 --- a/config.go +++ b/config.go @@ -22,6 +22,9 @@ type SysConfig struct { SCGIPaths map[string]string ReadMollyFiles bool AllowTLS12 bool + RateLimitEnable bool + RateLimitAverage int + RateLimitBurst int } type UserConfig struct { @@ -56,6 +59,9 @@ func getConfig(filename string) (SysConfig, UserConfig, error) { sysConfig.SCGIPaths = make(map[string]string) sysConfig.ReadMollyFiles = false sysConfig.AllowTLS12 = true + sysConfig.RateLimitEnable = false + sysConfig.RateLimitAverage = 1 + sysConfig.RateLimitBurst = 10 userConfig.GeminiExt = "gmi" userConfig.DefaultLang = "" diff --git a/handler.go b/handler.go index 26ba82a..4f4515d 100644 --- a/handler.go +++ b/handler.go @@ -50,12 +50,14 @@ func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, } // Enforce rate limiting - noPort := logEntry.RemoteAddr.String() - noPort = noPort[0:strings.LastIndex(noPort, ":")] - if !rl.Allowed(noPort) { - conn.Write([]byte("44 10 second cool down, please!\r\n")) - logEntry.Status = 44 - return + if sysConfig.RateLimitEnable { + noPort := logEntry.RemoteAddr.String() + noPort = noPort[0:strings.LastIndex(noPort, ":")] + if !rl.Allowed(noPort) { + conn.Write([]byte("44 10 second cool down, please!\r\n")) + logEntry.Status = 44 + return + } } // Read request diff --git a/launch.go b/launch.go index 8c302b1..1dcaba3 100644 --- a/launch.go +++ b/launch.go @@ -159,7 +159,7 @@ func launch(sysConfig SysConfig, userConfig UserConfig, privInfo userInfo) int { // Infinite serve loop (SIGTERM breaks out) running := true var wg sync.WaitGroup - rl := newRateLimiter(100, 5) + rl := newRateLimiter(sysConfig.RateLimitAverage, sysConfig.RateLimitBurst) for running { conn, err := listener.Accept() if err == nil { diff --git a/ratelim.go b/ratelim.go index 6aa08dc..ff399f3 100644 --- a/ratelim.go +++ b/ratelim.go @@ -1,7 +1,6 @@ package main import ( - "fmt" "sync" "time" ) @@ -9,19 +8,18 @@ import ( type RateLimiter struct { mu sync.Mutex bucket map[string]int - capacity int rate int + burst int } -func newRateLimiter(capacity int, rate int) RateLimiter { +func newRateLimiter(rate int, burst int) RateLimiter { var rl = new(RateLimiter) rl.bucket = make(map[string]int) - rl.capacity = capacity rl.rate = rate + rl.burst = burst // Leak periodically go func () { for(true) { - fmt.Println(rl.bucket) rl.mu.Lock() for addr, drips := range rl.bucket { if drips <= rate { @@ -45,7 +43,7 @@ func (rl *RateLimiter) Allowed(addr string) bool { rl.bucket[addr] = 1 return true } - if drips == rl.capacity { + if drips == rl.burst { return false } rl.bucket[addr] = drips + 1 From 3c5835f033144f2d65ca7a53c699837746543026 Mon Sep 17 00:00:00 2001 From: Solderpunk Date: Sat, 18 Mar 2023 15:45:35 +0100 Subject: [PATCH 3/6] Continue to increment drips once bucket is overflowing. --- handler.go | 5 +++-- ratelim.go | 12 +++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/handler.go b/handler.go index 4f4515d..afd58a6 100644 --- a/handler.go +++ b/handler.go @@ -53,8 +53,9 @@ func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, if sysConfig.RateLimitEnable { noPort := logEntry.RemoteAddr.String() noPort = noPort[0:strings.LastIndex(noPort, ":")] - if !rl.Allowed(noPort) { - conn.Write([]byte("44 10 second cool down, please!\r\n")) + drips, allowed := rl.Allowed(noPort) + if !allowed { + conn.Write([]byte("44 " + strconv.Itoa(drips) + " second cool down, please!\r\n")) logEntry.Status = 44 return } diff --git a/ratelim.go b/ratelim.go index ff399f3..9c97ec9 100644 --- a/ratelim.go +++ b/ratelim.go @@ -35,18 +35,16 @@ func newRateLimiter(rate int, burst int) RateLimiter { return *rl } -func (rl *RateLimiter) Allowed(addr string) bool { +func (rl *RateLimiter) Allowed(addr string) (int, bool) { rl.mu.Lock() defer rl.mu.Unlock() drips, present := rl.bucket[addr] if !present { rl.bucket[addr] = 1 - return true + return 1, true } - if drips == rl.burst { - return false - } - rl.bucket[addr] = drips + 1 - return true + drips += 1 + rl.bucket[addr] = drips + return drips, drips < rl.burst } From efde852c54f22e09da962acefff8ad0477b2f628 Mon Sep 17 00:00:00 2001 From: Solderpunk Date: Sat, 18 Mar 2023 16:40:23 +0100 Subject: [PATCH 4/6] Refactor rate limiting to have soft and hard limits, block clients exceeding hard limits for one hour. --- config.go | 6 ++++-- handler.go | 10 +++++++--- launch.go | 6 ++++-- ratelim.go | 38 ++++++++++++++++++++++++++++++++------ 4 files changed, 47 insertions(+), 13 deletions(-) diff --git a/config.go b/config.go index 8e2e749..6d0c91f 100644 --- a/config.go +++ b/config.go @@ -24,7 +24,8 @@ type SysConfig struct { AllowTLS12 bool RateLimitEnable bool RateLimitAverage int - RateLimitBurst int + RateLimitSoft int + RateLimitHard int } type UserConfig struct { @@ -61,7 +62,8 @@ func getConfig(filename string) (SysConfig, UserConfig, error) { sysConfig.AllowTLS12 = true sysConfig.RateLimitEnable = false sysConfig.RateLimitAverage = 1 - sysConfig.RateLimitBurst = 10 + sysConfig.RateLimitSoft = 10 + sysConfig.RateLimitHard = 50 userConfig.GeminiExt = "gmi" userConfig.DefaultLang = "" diff --git a/handler.go b/handler.go index afd58a6..093306d 100644 --- a/handler.go +++ b/handler.go @@ -53,9 +53,13 @@ func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, if sysConfig.RateLimitEnable { noPort := logEntry.RemoteAddr.String() noPort = noPort[0:strings.LastIndex(noPort, ":")] - drips, allowed := rl.Allowed(noPort) - if !allowed { - conn.Write([]byte("44 " + strconv.Itoa(drips) + " second cool down, please!\r\n")) + limited := rl.hardLimited(noPort) + if limited { + conn.Close() + } + delay, limited := rl.softLimited(noPort) + if limited { + conn.Write([]byte("44 " + strconv.Itoa(delay) + " second cool down, please!\r\n")) logEntry.Status = 44 return } diff --git a/launch.go b/launch.go index 1dcaba3..3738f4b 100644 --- a/launch.go +++ b/launch.go @@ -140,7 +140,9 @@ func launch(sysConfig SysConfig, userConfig UserConfig, privInfo userInfo) int { go func() { for { entry := <-accessLogEntries - writeLogEntry(accessLogFile, entry) + if entry.Status != 0 { + writeLogEntry(accessLogFile, entry) + } } }() } @@ -159,7 +161,7 @@ func launch(sysConfig SysConfig, userConfig UserConfig, privInfo userInfo) int { // Infinite serve loop (SIGTERM breaks out) running := true var wg sync.WaitGroup - rl := newRateLimiter(sysConfig.RateLimitAverage, sysConfig.RateLimitBurst) + rl := newRateLimiter(sysConfig.RateLimitAverage, sysConfig.RateLimitSoft, sysConfig.RateLimitHard) for running { conn, err := listener.Accept() if err == nil { diff --git a/ratelim.go b/ratelim.go index 9c97ec9..8ca2f1f 100644 --- a/ratelim.go +++ b/ratelim.go @@ -1,6 +1,7 @@ package main import ( + "log" "sync" "time" ) @@ -8,19 +9,25 @@ import ( type RateLimiter struct { mu sync.Mutex bucket map[string]int + bans map[string]time.Time rate int - burst int + softLimit int + hardLimit int } -func newRateLimiter(rate int, burst int) RateLimiter { +func newRateLimiter(rate int, softLimit int, hardLimit int) RateLimiter { var rl = new(RateLimiter) rl.bucket = make(map[string]int) + rl.bans = make(map[string]time.Time) rl.rate = rate - rl.burst = burst + rl.softLimit = softLimit + rl.hardLimit = hardLimit + // Leak periodically go func () { for(true) { rl.mu.Lock() + // Leak the buckets for addr, drips := range rl.bucket { if drips <= rate { delete(rl.bucket, addr) @@ -28,6 +35,15 @@ func newRateLimiter(rate int, burst int) RateLimiter { rl.bucket[addr] = drips - rl.rate } } + // Expire bans + now := time.Now() + for addr, expiry := range rl.bans { + if now.After(expiry) { + delete(rl.bans, addr) + } + } + + // Wait rl.mu.Unlock() time.Sleep(time.Second) } @@ -35,16 +51,26 @@ func newRateLimiter(rate int, burst int) RateLimiter { return *rl } -func (rl *RateLimiter) Allowed(addr string) (int, bool) { +func (rl *RateLimiter) softLimited(addr string) (int, bool) { rl.mu.Lock() defer rl.mu.Unlock() drips, present := rl.bucket[addr] if !present { rl.bucket[addr] = 1 - return 1, true + return 1, false } drips += 1 rl.bucket[addr] = drips - return drips, drips < rl.burst + if drips > rl.hardLimit { + now := time.Now() + expiry := now.Add(time.Hour) + rl.bans[addr] = expiry + log.Println("Banning " + addr + "for 1 hour due to ignoring rate limiting.") + } + return drips, drips > rl.softLimit } +func (rl *RateLimiter) hardLimited(addr string) bool { + _, present := rl.bans[addr] + return present +} From 4b9a7e8ad56e26e2cece547e14e0fd8ef21e9e79 Mon Sep 17 00:00:00 2001 From: Solderpunk Date: Sun, 19 Mar 2023 10:30:08 +0100 Subject: [PATCH 5/6] Correctly implement bans for clients exceeding hard limit. --- handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/handler.go b/handler.go index 093306d..c8eb83e 100644 --- a/handler.go +++ b/handler.go @@ -56,6 +56,7 @@ func handleGeminiRequest(conn net.Conn, sysConfig SysConfig, config UserConfig, limited := rl.hardLimited(noPort) if limited { conn.Close() + return } delay, limited := rl.softLimited(noPort) if limited { From 8e618a6304d42c11ab0878ccfb90884598962a03 Mon Sep 17 00:00:00 2001 From: Solderpunk Date: Sun, 19 Mar 2023 10:31:06 +0100 Subject: [PATCH 6/6] Double hard limit ban durations each time. --- ratelim.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/ratelim.go b/ratelim.go index 8ca2f1f..92a8ccd 100644 --- a/ratelim.go +++ b/ratelim.go @@ -3,6 +3,7 @@ package main import ( "log" "sync" + "strconv" "time" ) @@ -10,6 +11,7 @@ type RateLimiter struct { mu sync.Mutex bucket map[string]int bans map[string]time.Time + banCounts map[string]int rate int softLimit int hardLimit int @@ -19,6 +21,7 @@ func newRateLimiter(rate int, softLimit int, hardLimit int) RateLimiter { var rl = new(RateLimiter) rl.bucket = make(map[string]int) rl.bans = make(map[string]time.Time) + rl.banCounts = make(map[string]int) rl.rate = rate rl.softLimit = softLimit rl.hardLimit = hardLimit @@ -62,10 +65,18 @@ func (rl *RateLimiter) softLimited(addr string) (int, bool) { drips += 1 rl.bucket[addr] = drips if drips > rl.hardLimit { + banCount, present := rl.banCounts[addr] + if present { + banCount += 1 + } else { + banCount = 1 + } + rl.banCounts[addr] = banCount + banDuration := 1 << (banCount - 1) now := time.Now() - expiry := now.Add(time.Hour) + expiry := now.Add(time.Duration(banDuration)*time.Hour) rl.bans[addr] = expiry - log.Println("Banning " + addr + "for 1 hour due to ignoring rate limiting.") + log.Println("Banning " + addr + " for " + strconv.Itoa(banDuration) + " hours due to ignoring rate limiting.") } return drips, drips > rl.softLimit }