1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
|
https://github.com/h2o/h2o/pull/3293
From 770208bbe3955c47e005a1e8cb08266e4a8dfc9a Mon Sep 17 00:00:00 2001
From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Tue, 10 Oct 2023 15:47:57 +0200
Subject: [PATCH] [http2] delay processing requests upon observing suspicious
behavior
Backport of 94fbc54b6c9309912fe3d53e7b63408bbe9a1b0d to v2.2.x
---
include/h2o.h | 8 +++++++
include/h2o/http2_internal.h | 8 +++++++
lib/core/config.c | 1 +
lib/core/configurator.c | 9 ++++++++
lib/core/context.c | 2 ++
lib/http2/connection.c | 41 ++++++++++++++++++++++++++++++++----
6 files changed, 65 insertions(+), 4 deletions(-)
diff --git a/include/h2o.h b/include/h2o.h
index 57877bd12c..409cd5c21c 100644
--- a/include/h2o.h
+++ b/include/h2o.h
@@ -378,6 +378,10 @@ struct st_h2o_globalconf_t {
* list of callbacks
*/
h2o_protocol_callbacks_t callbacks;
+ /**
+ * milliseconds to delay processing requests when suspicious behavior is detected
+ */
+ uint64_t dos_delay;
} http2;
struct {
@@ -590,6 +594,10 @@ struct st_h2o_context_t {
* timeout entry used for graceful shutdown
*/
h2o_timeout_entry_t _graceful_shutdown_timeout;
+ /*
+ * dos timeout
+ */
+ h2o_timeout_t dos_delay_timeout;
struct {
/**
* counter for http2 errors internally emitted by h2o
diff --git a/include/h2o/http2_internal.h b/include/h2o/http2_internal.h
index 5cfc4d8204..b9cf400929 100644
--- a/include/h2o/http2_internal.h
+++ b/include/h2o/http2_internal.h
@@ -179,6 +179,7 @@ struct st_h2o_http2_stream_t {
h2o_linklist_t link;
h2o_http2_scheduler_openref_t scheduler;
} _refs;
+ unsigned reset_by_peer : 1;
h2o_send_state_t send_state; /* state of the ostream, only used in push mode */
/* placed at last since it is large and has it's own ctor */
h2o_req_t req;
@@ -232,6 +233,13 @@ struct st_h2o_http2_conn_t {
} _write;
h2o_cache_t *push_memo;
h2o_http2_casper_t *casper;
+ /**
+ * DoS mitigation; the idea here is to delay processing requests when observing suspicious behavior
+ */
+ struct {
+ h2o_timeout_entry_t process_delay;
+ size_t reset_budget; /* RST_STREAM frames are considered suspicious when this value goes down to zero */
+ } dos_mitigation;
};
int h2o_http2_update_peer_settings(h2o_http2_settings_t *settings, const uint8_t *src, size_t len, const char **err_desc);
diff --git a/lib/core/config.c b/lib/core/config.c
index ce1d320183..08e43a6d30 100644
--- a/lib/core/config.c
+++ b/lib/core/config.c
@@ -189,6 +189,7 @@ void h2o_config_init(h2o_globalconf_t *config)
config->http2.latency_optimization.min_rtt = 50; // milliseconds
config->http2.latency_optimization.max_additional_delay = 10;
config->http2.latency_optimization.max_cwnd = 65535;
+ config->http2.dos_delay = 100; /* 100ms processing delay when observing suspicious behavior */
config->http2.callbacks = H2O_HTTP2_CALLBACKS;
config->mimemap = h2o_mimemap_create();
diff --git a/lib/core/configurator.c b/lib/core/configurator.c
index 891770cc2d..4731ba2707 100644
--- a/lib/core/configurator.c
+++ b/lib/core/configurator.c
@@ -531,6 +531,12 @@ static int on_config_http2_casper(h2o_configurator_command_t *cmd, h2o_configura
return 0;
}
+
+static int on_config_http2_dos_delay(h2o_configurator_command_t *cmd, h2o_configurator_context_t *ctx, yoml_t *node)
+{
+ return config_timeout(cmd, node, &ctx->globalconf->http2.dos_delay);
+}
+
static int assert_is_mimetype(h2o_configurator_command_t *cmd, yoml_t *node)
{
if (node->type != YOML_TYPE_SCALAR) {
@@ -910,6 +916,9 @@ void h2o_configurator__init_core(h2o_globalconf_t *conf)
on_config_http2_push_preload);
h2o_configurator_define_command(&c->super, "http2-casper", H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_HOST,
on_config_http2_casper);
+ h2o_configurator_define_command(&c->super, "http2-dos-delay",
+ H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_EXPECT_SCALAR,
+ on_config_http2_dos_delay);
h2o_configurator_define_command(&c->super, "file.mime.settypes",
(H2O_CONFIGURATOR_FLAG_ALL_LEVELS & ~H2O_CONFIGURATOR_FLAG_EXTENSION) |
H2O_CONFIGURATOR_FLAG_EXPECT_MAPPING,
diff --git a/lib/core/context.c b/lib/core/context.c
index 8d11013810..ac4b0aaf08 100644
--- a/lib/core/context.c
+++ b/lib/core/context.c
@@ -101,6 +101,7 @@ void h2o_context_init(h2o_context_t *ctx, h2o_loop_t *loop, h2o_globalconf_t *co
h2o_linklist_init_anchor(&ctx->http1._conns);
h2o_timeout_init(ctx->loop, &ctx->http2.idle_timeout, config->http2.idle_timeout);
h2o_timeout_init(ctx->loop, &ctx->http2.graceful_shutdown_timeout, config->http2.graceful_shutdown_timeout);
+ h2o_timeout_init(ctx->loop, &ctx->http2.dos_delay_timeout, config->http2.dos_delay);
h2o_linklist_init_anchor(&ctx->http2._conns);
ctx->proxy.client_ctx.loop = loop;
h2o_timeout_init(ctx->loop, &ctx->proxy.io_timeout, config->proxy.io_timeout);
@@ -146,6 +147,7 @@ void h2o_context_dispose(h2o_context_t *ctx)
h2o_timeout_dispose(ctx->loop, &ctx->http1.req_timeout);
h2o_timeout_dispose(ctx->loop, &ctx->http2.idle_timeout);
h2o_timeout_dispose(ctx->loop, &ctx->http2.graceful_shutdown_timeout);
+ h2o_timeout_dispose(ctx->loop, &ctx->http2.dos_delay_timeout);
h2o_timeout_dispose(ctx->loop, &ctx->proxy.io_timeout);
/* what should we do here? assert(!h2o_linklist_is_empty(&ctx->http2._conns); */
diff --git a/lib/http2/connection.c b/lib/http2/connection.c
index e2da293043..4910e33098 100644
--- a/lib/http2/connection.c
+++ b/lib/http2/connection.c
@@ -161,7 +161,6 @@ static void update_idle_timeout(h2o_http2_conn_t *conn)
h2o_timeout_unlink(&conn->_timeout_entry);
if (conn->num_streams.pull.half_closed + conn->num_streams.push.half_closed == 0) {
- assert(h2o_linklist_is_empty(&conn->_pending_reqs));
conn->_timeout_entry.cb = on_idle_timeout;
h2o_timeout_link(conn->super.ctx->loop, &conn->super.ctx->http2.idle_timeout, &conn->_timeout_entry);
}
@@ -175,6 +174,9 @@ static int can_run_requests(h2o_http2_conn_t *conn)
static void run_pending_requests(h2o_http2_conn_t *conn)
{
+ if (h2o_timeout_is_linked(&conn->dos_mitigation.process_delay))
+ return;
+
while (!h2o_linklist_is_empty(&conn->_pending_reqs) && can_run_requests(conn)) {
/* fetch and detach a pending stream */
h2o_http2_stream_t *stream = H2O_STRUCT_FROM_MEMBER(h2o_http2_stream_t, _refs.link, conn->_pending_reqs.next);
@@ -226,6 +228,16 @@ void h2o_http2_conn_unregister_stream(h2o_http2_conn_t *conn, h2o_http2_stream_t
assert(h2o_http2_scheduler_is_open(&stream->_refs.scheduler));
h2o_http2_scheduler_close(&stream->_refs.scheduler);
+ /* Decrement reset_budget if the stream was reset by peer, otherwise increment. By doing so, we penalize connections that
+ * generate resets for >50% of requests. */
+ if (stream->reset_by_peer) {
+ if (conn->dos_mitigation.reset_budget > 0)
+ --conn->dos_mitigation.reset_budget;
+ } else {
+ if (conn->dos_mitigation.reset_budget < conn->super.ctx->globalconf->http2.max_concurrent_requests_per_connection)
+ ++conn->dos_mitigation.reset_budget;
+ }
+
switch (stream->state) {
case H2O_HTTP2_STREAM_STATE_IDLE:
case H2O_HTTP2_STREAM_STATE_RECV_HEADERS:
@@ -272,6 +284,8 @@ void close_connection_now(h2o_http2_conn_t *conn)
h2o_hpack_dispose_header_table(&conn->_output_header_table);
assert(h2o_linklist_is_empty(&conn->_pending_reqs));
h2o_timeout_unlink(&conn->_timeout_entry);
+ if (h2o_timeout_is_linked(&conn->dos_mitigation.process_delay))
+ h2o_timeout_unlink(&conn->dos_mitigation.process_delay);
h2o_buffer_dispose(&conn->_write.buf);
if (conn->_write.buf_in_flight != NULL)
h2o_buffer_dispose(&conn->_write.buf_in_flight);
@@ -797,11 +811,19 @@ static int handle_rst_stream_frame(h2o_http2_conn_t *conn, h2o_http2_frame_t *fr
return H2O_HTTP2_ERROR_PROTOCOL;
}
- stream = h2o_http2_conn_get_stream(conn, frame->stream_id);
- if (stream != NULL) {
+ if ((stream = h2o_http2_conn_get_stream(conn, frame->stream_id)) == NULL)
+ return 0;
+
/* reset the stream */
+ stream->reset_by_peer = 1;
h2o_http2_stream_reset(conn, stream);
- }
+
+ /* setup process delay if we've just ran out of reset budget */
+ if (conn->dos_mitigation.reset_budget == 0 && conn->super.ctx->globalconf->http2.dos_delay != 0 &&
+ !h2o_timeout_is_linked(&conn->dos_mitigation.process_delay))
+ h2o_timeout_link(conn->super.ctx->loop, &conn->super.ctx->http2.dos_delay_timeout,
+ &conn->dos_mitigation.process_delay);
+
/* TODO log */
return 0;
@@ -1204,6 +1226,14 @@ static h2o_iovec_t log_priority_actual_weight(h2o_req_t *req)
return h2o_iovec_init(s, len);
}
+static void on_dos_process_delay(h2o_timeout_entry_t *timer)
+{
+ h2o_http2_conn_t *conn = H2O_STRUCT_FROM_MEMBER(h2o_http2_conn_t, dos_mitigation.process_delay, timer);
+
+ assert(!h2o_timeout_is_linked(&conn->dos_mitigation.process_delay));
+ run_pending_requests(conn);
+}
+
static h2o_http2_conn_t *create_conn(h2o_context_t *ctx, h2o_hostconf_t **hosts, h2o_socket_t *sock, struct timeval connected_at)
{
static const h2o_conn_callbacks_t callbacks = {
@@ -1240,6 +1270,9 @@ static h2o_http2_conn_t *create_conn(h2o_context_t *ctx, h2o_hostconf_t **hosts,
conn->_write.timeout_entry.cb = emit_writereq;
h2o_http2_window_init(&conn->_write.window, &conn->peer_settings);
+ conn->dos_mitigation.process_delay.cb = on_dos_process_delay;
+ conn->dos_mitigation.reset_budget = conn->super.ctx->globalconf->http2.max_concurrent_requests_per_connection;
+
return conn;
}
|
