Skip to content

Commit c57304a

Browse files
bohdewxiaoguangGiteaBot
authored
Add middleware for request prioritization (#33951)
This adds a middleware for overload protection that is intended to help protect against malicious scrapers. It does this via [`codel`](https://github.com/bohde/codel), which will perform the following: 1. Limit the number of in-flight requests to some user-defined max 2. When in-flight requests have reached their begin queuing requests. Logged-in requests having priority above logged-out requests 3. Once a request has been queued for too long, it has a probabilistic chance to be rejected based on how overloaded the entire system is. When a server experiences more traffic than it can handle, this keeps latency low for logged-in users and rejects just enough requests from logged-out users to not overload the service. --------- Co-authored-by: wxiaoguang <[email protected]> Co-authored-by: Giteabot <[email protected]>
1 parent 3a9fcac commit c57304a

File tree

10 files changed

+301
-2
lines changed

10 files changed

+301
-2
lines changed

assets/go-licenses.json

+5
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

custom/conf/app.example.ini

+22-1
Original file line numberDiff line numberDiff line change
@@ -940,7 +940,29 @@ LEVEL = Info
940940
;;
941941
;; Disable the code explore page.
942942
;DISABLE_CODE_PAGE = false
943+
944+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
945+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
946+
;[qos]
947+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
948+
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
943949
;;
950+
;; Enable request quality of service and overload protection.
951+
; ENABLED = false
952+
;;
953+
;; The maximum number of concurrent requests that the server will
954+
;; process before enqueueing new requests. Default is "CpuNum * 4".
955+
; MAX_INFLIGHT =
956+
;;
957+
;; The maximum number of requests that can be enqueued before new
958+
;; requests will be dropped.
959+
; MAX_WAITING = 100
960+
;;
961+
;; Target maximum wait time a request may be enqueued for. Requests
962+
;; that are enqueued for less than this amount of time will not be
963+
;; dropped. When wait times exceed this amount, a portion of requests
964+
;; will be dropped until wait times have decreased below this amount.
965+
; TARGET_WAIT_TIME = 250ms
944966

945967
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
946968
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -1423,7 +1445,6 @@ LEVEL = Info
14231445
;; or use comma separated list: inline-dollar, inline-parentheses, block-dollar, block-square-brackets
14241446
;; Defaults to "inline-dollar,block-dollar" to follow GitHub's behavior.
14251447
;MATH_CODE_BLOCK_DETECTION =
1426-
;;
14271448

14281449
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
14291450
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ require (
3232
github.com/aws/aws-sdk-go-v2/service/codecommit v1.28.1
3333
github.com/blakesmith/ar v0.0.0-20190502131153-809d4375e1fb
3434
github.com/blevesearch/bleve/v2 v2.4.2
35+
github.com/bohde/codel v0.2.0
3536
github.com/buildkite/terminal-to-html/v3 v3.16.8
3637
github.com/caddyserver/certmagic v0.22.0
3738
github.com/charmbracelet/git-lfs-transfer v0.2.0

go.sum

+6
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,9 @@ github.com/blevesearch/zapx/v16 v16.1.5 h1:b0sMcarqNFxuXvjoXsF8WtwVahnxyhEvBSRJi
179179
github.com/blevesearch/zapx/v16 v16.1.5/go.mod h1:J4mSF39w1QELc11EWRSBFkPeZuO7r/NPKkHzDCoiaI8=
180180
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
181181
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
182+
github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q=
183+
github.com/bohde/codel v0.2.0 h1:fzF7ibgKmCfQbOzQCblmQcwzDRmV7WO7VMLm/hDvD3E=
184+
github.com/bohde/codel v0.2.0/go.mod h1:Idb1IRvTdwkRjIjguLIo+FXhIBhcpGl94o7xra6ggWk=
182185
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
183186
github.com/boombuler/barcode v1.0.2 h1:79yrbttoZrLGkL/oOI8hBrUKucwOL0oOjUgEguGMcJ4=
184187
github.com/boombuler/barcode v1.0.2/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
@@ -881,6 +884,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
881884
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
882885
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
883886
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
887+
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
884888
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
885889
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
886890
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
@@ -1025,6 +1029,8 @@ modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg=
10251029
modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
10261030
mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI=
10271031
mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk=
1032+
pgregory.net/rapid v0.4.2 h1:lsi9jhvZTYvzVpeG93WWgimPRmiJQfGFRNTEZh1dtY0=
1033+
pgregory.net/rapid v0.4.2/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
10281034
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 h1:mUcz5b3FJbP5Cvdq7Khzn6J9OCUQJaBwgBkCR+MOwSs=
10291035
strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1:FJGmPh3vz9jSos1L/F91iAgnC/aejc0wIIrF2ZwJxdY=
10301036
xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo=

modules/setting/service.go

+17
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package setting
55

66
import (
77
"regexp"
8+
"runtime"
89
"strings"
910
"time"
1011

@@ -98,6 +99,13 @@ var Service = struct {
9899
DisableOrganizationsPage bool `ini:"DISABLE_ORGANIZATIONS_PAGE"`
99100
DisableCodePage bool `ini:"DISABLE_CODE_PAGE"`
100101
} `ini:"service.explore"`
102+
103+
QoS struct {
104+
Enabled bool
105+
MaxInFlightRequests int
106+
MaxWaitingRequests int
107+
TargetWaitTime time.Duration
108+
}
101109
}{
102110
AllowedUserVisibilityModesSlice: []bool{true, true, true},
103111
}
@@ -255,6 +263,7 @@ func loadServiceFrom(rootCfg ConfigProvider) {
255263
mustMapSetting(rootCfg, "service.explore", &Service.Explore)
256264

257265
loadOpenIDSetting(rootCfg)
266+
loadQosSetting(rootCfg)
258267
}
259268

260269
func loadOpenIDSetting(rootCfg ConfigProvider) {
@@ -276,3 +285,11 @@ func loadOpenIDSetting(rootCfg ConfigProvider) {
276285
}
277286
}
278287
}
288+
289+
func loadQosSetting(rootCfg ConfigProvider) {
290+
sec := rootCfg.Section("qos")
291+
Service.QoS.Enabled = sec.Key("ENABLED").MustBool(false)
292+
Service.QoS.MaxInFlightRequests = sec.Key("MAX_INFLIGHT").MustInt(4 * runtime.NumCPU())
293+
Service.QoS.MaxWaitingRequests = sec.Key("MAX_WAITING").MustInt(100)
294+
Service.QoS.TargetWaitTime = sec.Key("TARGET_WAIT_TIME").MustDuration(250 * time.Millisecond)
295+
}

options/locale/locale_en-US.ini

+1
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ files = Files
117117

118118
error = Error
119119
error404 = The page you are trying to reach either <strong>does not exist</strong> or <strong>you are not authorized</strong> to view it.
120+
error503 = The server was unable to complete your request. Please try again later.
120121
go_back = Go Back
121122
invalid_data = Invalid data: %v
122123

routers/common/qos.go

+145
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
// Copyright 2025 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package common
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"net/http"
10+
"strings"
11+
12+
user_model "code.gitea.io/gitea/models/user"
13+
"code.gitea.io/gitea/modules/log"
14+
"code.gitea.io/gitea/modules/setting"
15+
"code.gitea.io/gitea/modules/templates"
16+
"code.gitea.io/gitea/modules/web/middleware"
17+
giteacontext "code.gitea.io/gitea/services/context"
18+
19+
"github.com/bohde/codel"
20+
"github.com/go-chi/chi/v5"
21+
)
22+
23+
const tplStatus503 templates.TplName = "status/503"
24+
25+
type Priority int
26+
27+
func (p Priority) String() string {
28+
switch p {
29+
case HighPriority:
30+
return "high"
31+
case DefaultPriority:
32+
return "default"
33+
case LowPriority:
34+
return "low"
35+
default:
36+
return fmt.Sprintf("%d", p)
37+
}
38+
}
39+
40+
const (
41+
LowPriority = Priority(-10)
42+
DefaultPriority = Priority(0)
43+
HighPriority = Priority(10)
44+
)
45+
46+
// QoS implements quality of service for requests, based upon whether
47+
// or not the user is logged in. All traffic may get dropped, and
48+
// anonymous users are deprioritized.
49+
func QoS() func(next http.Handler) http.Handler {
50+
if !setting.Service.QoS.Enabled {
51+
return nil
52+
}
53+
54+
maxOutstanding := setting.Service.QoS.MaxInFlightRequests
55+
if maxOutstanding <= 0 {
56+
maxOutstanding = 10
57+
}
58+
59+
c := codel.NewPriority(codel.Options{
60+
// The maximum number of waiting requests.
61+
MaxPending: setting.Service.QoS.MaxWaitingRequests,
62+
// The maximum number of in-flight requests.
63+
MaxOutstanding: maxOutstanding,
64+
// The target latency that a blocked request should wait
65+
// for. After this, it might be dropped.
66+
TargetLatency: setting.Service.QoS.TargetWaitTime,
67+
})
68+
69+
return func(next http.Handler) http.Handler {
70+
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
71+
ctx := req.Context()
72+
73+
priority := requestPriority(ctx)
74+
75+
// Check if the request can begin processing.
76+
err := c.Acquire(ctx, int(priority))
77+
if err != nil {
78+
log.Error("QoS error, dropping request of priority %s: %v", priority, err)
79+
renderServiceUnavailable(w, req)
80+
return
81+
}
82+
83+
// Release long-polling immediately, so they don't always
84+
// take up an in-flight request
85+
if strings.Contains(req.URL.Path, "/user/events") {
86+
c.Release()
87+
} else {
88+
defer c.Release()
89+
}
90+
91+
next.ServeHTTP(w, req)
92+
})
93+
}
94+
}
95+
96+
// requestPriority assigns a priority value for a request based upon
97+
// whether the user is logged in and how expensive the endpoint is
98+
func requestPriority(ctx context.Context) Priority {
99+
// If the user is logged in, assign high priority.
100+
data := middleware.GetContextData(ctx)
101+
if _, ok := data[middleware.ContextDataKeySignedUser].(*user_model.User); ok {
102+
return HighPriority
103+
}
104+
105+
rctx := chi.RouteContext(ctx)
106+
if rctx == nil {
107+
return DefaultPriority
108+
}
109+
110+
// If we're operating in the context of a repo, assign low priority
111+
routePattern := rctx.RoutePattern()
112+
if strings.HasPrefix(routePattern, "/{username}/{reponame}/") {
113+
return LowPriority
114+
}
115+
116+
return DefaultPriority
117+
}
118+
119+
// renderServiceUnavailable will render an HTTP 503 Service
120+
// Unavailable page, providing HTML if the client accepts it.
121+
func renderServiceUnavailable(w http.ResponseWriter, req *http.Request) {
122+
acceptsHTML := false
123+
for _, part := range req.Header["Accept"] {
124+
if strings.Contains(part, "text/html") {
125+
acceptsHTML = true
126+
break
127+
}
128+
}
129+
130+
// If the client doesn't accept HTML, then render a plain text response
131+
if !acceptsHTML {
132+
http.Error(w, "503 Service Unavailable", http.StatusServiceUnavailable)
133+
return
134+
}
135+
136+
tmplCtx := giteacontext.TemplateContext{}
137+
tmplCtx["Locale"] = middleware.Locale(w, req)
138+
ctxData := middleware.GetContextData(req.Context())
139+
err := templates.HTMLRenderer().HTML(w, http.StatusServiceUnavailable, tplStatus503, ctxData, tmplCtx)
140+
if err != nil {
141+
log.Error("Error occurs again when rendering service unavailable page: %v", err)
142+
w.WriteHeader(http.StatusInternalServerError)
143+
_, _ = w.Write([]byte("Internal server error, please collect error logs and report to Gitea issue tracker"))
144+
}
145+
}

routers/common/qos_test.go

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
// Copyright 2025 The Gitea Authors. All rights reserved.
2+
// SPDX-License-Identifier: MIT
3+
4+
package common
5+
6+
import (
7+
"net/http"
8+
"testing"
9+
10+
user_model "code.gitea.io/gitea/models/user"
11+
"code.gitea.io/gitea/modules/web/middleware"
12+
"code.gitea.io/gitea/services/contexttest"
13+
14+
"github.com/go-chi/chi/v5"
15+
"github.com/stretchr/testify/assert"
16+
)
17+
18+
func TestRequestPriority(t *testing.T) {
19+
type test struct {
20+
Name string
21+
User *user_model.User
22+
RoutePattern string
23+
Expected Priority
24+
}
25+
26+
cases := []test{
27+
{
28+
Name: "Logged In",
29+
User: &user_model.User{},
30+
Expected: HighPriority,
31+
},
32+
{
33+
Name: "Sign In",
34+
RoutePattern: "/user/login",
35+
Expected: DefaultPriority,
36+
},
37+
{
38+
Name: "Repo Home",
39+
RoutePattern: "/{username}/{reponame}",
40+
Expected: DefaultPriority,
41+
},
42+
{
43+
Name: "User Repo",
44+
RoutePattern: "/{username}/{reponame}/src/branch/main",
45+
Expected: LowPriority,
46+
},
47+
}
48+
49+
for _, tc := range cases {
50+
t.Run(tc.Name, func(t *testing.T) {
51+
ctx, _ := contexttest.MockContext(t, "")
52+
53+
if tc.User != nil {
54+
data := middleware.GetContextData(ctx)
55+
data[middleware.ContextDataKeySignedUser] = tc.User
56+
}
57+
58+
rctx := chi.RouteContext(ctx)
59+
rctx.RoutePatterns = []string{tc.RoutePattern}
60+
61+
assert.Exactly(t, tc.Expected, requestPriority(ctx))
62+
})
63+
}
64+
}
65+
66+
func TestRenderServiceUnavailable(t *testing.T) {
67+
t.Run("HTML", func(t *testing.T) {
68+
ctx, resp := contexttest.MockContext(t, "")
69+
ctx.Req.Header.Set("Accept", "text/html")
70+
71+
renderServiceUnavailable(resp, ctx.Req)
72+
assert.Equal(t, http.StatusServiceUnavailable, resp.Code)
73+
assert.Contains(t, resp.Header().Get("Content-Type"), "text/html")
74+
75+
body := resp.Body.String()
76+
assert.Contains(t, body, `lang="en-US"`)
77+
assert.Contains(t, body, "503 Service Unavailable")
78+
})
79+
80+
t.Run("plain", func(t *testing.T) {
81+
ctx, resp := contexttest.MockContext(t, "")
82+
ctx.Req.Header.Set("Accept", "text/plain")
83+
84+
renderServiceUnavailable(resp, ctx.Req)
85+
assert.Equal(t, http.StatusServiceUnavailable, resp.Code)
86+
assert.Contains(t, resp.Header().Get("Content-Type"), "text/plain")
87+
88+
body := resp.Body.String()
89+
assert.Contains(t, body, "503 Service Unavailable")
90+
})
91+
}

routers/web/web.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -285,7 +285,7 @@ func Routes() *web.Router {
285285

286286
webRoutes := web.NewRouter()
287287
webRoutes.Use(mid...)
288-
webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive())
288+
webRoutes.Group("", func() { registerWebRoutes(webRoutes) }, common.BlockExpensive(), common.QoS())
289289
routes.Mount("", webRoutes)
290290
return routes
291291
}

templates/status/503.tmpl

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{{template "base/head" .}}
2+
<div role="main" aria-label="503 Service Unavailable" class="page-content">
3+
<div class="ui container">
4+
<div class="status-page-error">
5+
<div class="status-page-error-title">503 Service Unavailable</div>
6+
<div class="tw-text-center">
7+
<div class="tw-my-4">{{ctx.Locale.Tr "error503"}}</div>
8+
</div>
9+
</div>
10+
</div>
11+
</div>
12+
{{template "base/footer" .}}

0 commit comments

Comments
 (0)