Throttle requests with limit_req rate based on header from response to auth subrequest

classic Classic list List threaded Threaded
6 messages Options
Reply | Threaded
Open this post in threaded view
|

Throttle requests with limit_req rate based on header from response to auth subrequest

wld75
I'm hoping to use the limit_req directive with different rates based on a
header that is returned from the auth subrequest.  I got some ideas from
https://www.ruby-forum.com/topic/4418040 but am running into a few problems.
 Here is my configuration:

> user                     nginx;
> worker_processes         auto;
> error_log                /var/log/nginx/error.log warn;
> pid                      /var/run/nginx.pid;
> events {
>   worker_connections     10000;
> }
>
> worker_rlimit_nofile 10000;
>
> http {
>   log_subrequest on;
>
>   log_format  main  escape=json  '{ "timestamp": "$time_local", "client":
"$remote_addr",'
>                               ' "method": "$request_method", "uri":
"$uri",'
>                               ' "request_length": $request_length,'
>                               ' "status": $status, "bytes_sent":
$bytes_sent,'
>                               ' "upstream_status": "$upstream_status",'
>                               ' "request_id": "$request_id",'
>                               ' "request_uri": "$request_uri",'
>                               ' "tier": "$tier",'
>                               ' "upstream_http_tier":
"$upstream_http_tier",'
>                               ' "2X_key": "$2X_key",'
>                               ' "3X_key": "$3X_key",'
>                               ' "2X_key_from_upstream":
"$2X_key_from_upstream",'
>                               ' "3X_key_from_upstream":
"$3X_key_from_upstream",'

>                               ' "origin": "$http_origin"}' ;
>
>   access_log  /var/log/nginx/access.log  main;
>   sendfile               on;
>   tcp_nopush             on;
>   tcp_nodelay            on;
>   keepalive_timeout      65;
>   types_hash_max_size    2048;
>   include                /etc/nginx/mime.types;
>   default_type           application/octet-stream;
>   proxy_buffering        on;
>   proxy_buffers          8 64k;
>   proxy_cache_path       /dev/shm/nginx/auth use_temp_path=off levels=1:2
keys_zone=auth_cache:1024m inactive=30m max_size=1g;
>   proxy_cache_path       /dev/shm/nginx/manifests use_temp_path=off
levels=1:2 keys_zone=manifest_cache:100m inactive=30s max_size=10g;

>   proxy_cache_methods    GET HEAD;
>   proxy_cache_lock       on;
>   proxy_cache_use_stale  updating;
>   proxy_bind             0.0.0.0;
>   proxy_ignore_headers   Expires;
>   proxy_pass_header Server;
>
>
>   map $request_uri $endpoint_id {
>     default "unknown";
>     ~^/out/v\d+/(?P<endpoint.+?)/.+$ $endpoint;
>   }
>
>   # Mappings based on the tier header from the /auth request
>   map $tier $2X_key {~02x $endpoint_id; default "";}
>   map $tier $3X_key {~03x $endpoint_id; default "";}
>   map $upstream_http_tier $2X_key_from_upstream {~02x $endpoint_id;
default "";}
>   map $upstream_http_tier $3X_key_from_upstream {~03x $endpoint_id;
default "";}
>
>   # Throttle zones based on the results of the above mapping
>   limit_req_zone $2X_key zone=2x_zone:20m rate=10r/s;
>   limit_req_zone $3X_key zone=3x_zone:20m rate=100r/s;
>   limit_req_zone $2X_key_from_upstream zone=2x_zone_from_upstream:20m
rate=10r/s;
>   limit_req_zone $3X_key_from_upstream zone=3x_zone_from_upstream:20m
rate=100r/s;

>
>
>   server {
>     listen               80 default_server;
>     listen               [::]:80 default_server;
>     server_name          default_backend;
>     server_tokens        off;
>     access_log           /var/log/nginx/access.log main;
>
>     root /var/www/html;
>
>     error_page 401                    /error_pages/401.html;
>     error_page 403                    /error_pages/403.html;
>     error_page 404                    /error_pages/404.html;
>     error_page 429                    /error_pages/429.html;
>     error_page 500 501 502 503 504    /error_pages/5xx.html;
>
>     set $upstream_server <a href="http://my_server:80;">http://my_server:80;
>
>     proxy_http_version 1.1;
>     proxy_set_header   Connection "";
>     proxy_connect_timeout       10;
>     proxy_send_timeout          30;
>     proxy_read_timeout          30;
>
>     proxy_cache_valid  404 412 1s;
>
>     location ~* \.(m3u8|mpd|isml?/manifest)$ {
>       auth_request       /auth;
>
>       # Capture the tier header from auth request
>       auth_request_set $tier $upstream_http_tier;
>
>       # Throttling based on mappings from tier
>       limit_req zone=2x_zone burst=10 nodelay;
>       limit_req zone=3x_zone burst=10 nodelay;
>       limit_req zone=2x_zone_from_upstream burst=10 nodelay;
>       limit_req zone=3x_zone_from_upstream burst=10 nodelay;
>       limit_req_status 429;
>
>       proxy_pass         $upstream_server;
>       proxy_cache        manifest_cache;
>       set $cache_key "${endpoint_id}";
>       proxy_cache_key    $cache_key;
>       proxy_cache_valid  200 301 302  2s;
>
>       access_log         /var/log/nginx/access.log  main;
>     }
>
>     location /auth {
>       internal;
>
>       set $auth_type 1;
>       proxy_pass_request_body off;
>       proxy_pass         $upstream_server/auth?endpoint=$endpoint_id;
>
>       proxy_cache        auth_cache;
>       set $auth_cache_key "${endpoint_id}";
>       proxy_cache_key    $auth_cache_key;
>       proxy_cache_valid  200 301 302  5m;
>       proxy_cache_valid  400 401 403 404  5m;
>
>       access_log         /var/log/nginx/access.log  main;
>     }
>   }
> }
>

I'm expecting the following line to capture the "tier" header that comes
back from the  auth subrequest (sometimes it will be "02x", and sometimes
"03x"):

>  auth_request_set $tier $upstream_http_tier;

Then, that value will be passed into several mappings that should either
return "" or a value, depending on which throttle zone should be applied.
(There are variants using both $tier and $upstream_http_tier from attempts
at troubleshooting.)
 
>  map $tier $2X_key {~02x $endpoint_id; default "";}
>  map $tier $3X_key {~03x $endpoint_id; default "";}
>  map $upstream_http_tier $2X_key_from_upstream {~02x $endpoint_id; default
"";}
>  map $upstream_http_tier $3X_key_from_upstream {~03x $endpoint_id; default
"";}

Finally, I expect only the limit_req directive with a non-empty key to be
applied:

>  limit_req_zone $2X_key zone=2x_zone:20m rate=10r/s;
>  limit_req_zone $3X_key zone=3x_zone:20m rate=100r/s;
>  limit_req_zone $2X_key_from_upstream zone=2x_zone_from_upstream:20m
rate=10r/s;
>  limit_req_zone $3X_key_from_upstream zone=3x_zone_from_upstream:20m
rate=100r/s;
 

However, it's look like at least two things are not behaving as I expect.
1) The only throttling that I see is coming from 2x_zone, even when the
header comes back as "03x".

2018/08/29 22:37:31 [error] 12626#0: *1596735 limiting requests, excess:
10.010 by zone "2x_zone", client: 10.0.136.178, server: default_backend,
request: "GET /out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8
HTTP/1.1", host: "myserver"

2) Mapping based on the variable captured from auth_request_set don't appear
to be set as I expect on the main request.

Here's what I see in the access log for an auth request and a main request:


{
    "timestamp": "29/Aug/2018:22:37:31 +0000",
    "client": "10.0.136.178",
    "method": "GET",
    "uri": "/auth",
    "status": 200,
    "bytes_sent": 0,
    "upstream_status": "",
    "server_name": "default_backend",
    "request_id": "d6545f5758c2b3944438c80f2e964678",
    "request_uri":
"/out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8",
    "tier": "",
    "upstream_http_tier": "02x",
    "2X_key": "",
    "3X_key": "",
    "2X_key_from_upstream": "04b1719535444a1d84389aeb0a1fb912",
    "3X_key_from_upstream": "",
    "origin": ""
}
{
    "timestamp": "29/Aug/2018:22:37:31 +0000",
    "client": "10.0.136.178",
    "method": "GET",
    "uri": "/out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8",
    "status": 200,
    "bytes_sent": 652,
    "upstream_status": "",
    "server_name": "default_backend",
    "request_id": "d6545f5758c2b3944438c80f2e964678",
    "request_uri":
"/out/v1/04b1719535444a1d84389aeb0a1fb912/throttleCanary.m3u8",
    "tier": "02x",
    "upstream_http_tier": "",
    "2X_key": "",
    "3X_key": "",
    "2X_key_from_upstream": "04b1719535444a1d84389aeb0a1fb912",
    "3X_key_from_upstream": "",
    "origin": ""
}

Notice that in the main request, "tier" is "02x" as expected, but the
mapping based on $tier (2X_key) is empty while the mapping based on
$http_upstream_tier (2X_key_from_upstream) has a value.  Moreover, since
only 2X_key_from_upstream has a value, I would expect the throttle based on
that key (2x_zone_from_upstream) to take effect, not 2x_zone whose key is
empty.

I would really appreciate any help explaining my misunderstanding or advice
with how better to implement what I'm trying to do.

Thanks,
Jared

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,281034,281034#msg-281034

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: Throttle requests with limit_req rate based on header from response to auth subrequest

Francis Daly
On Wed, Aug 29, 2018 at 07:14:01PM -0400, jarstewa wrote:

Hi there,

I do not know the answer, and I have not tested the code you provided.

But, one suggestion which might be quick for you to test:

what happens if you change all of your variable names so that they do
not start with a digit?

As in: rename $2X_key to be (for example) $a2X_key.

It is possible that "$2X_key" will be expanded as "X_key" when $2 has
no value.

> >   # Throttle zones based on the results of the above mapping
> >   limit_req_zone $2X_key zone=2x_zone:20m rate=10r/s;
> >   limit_req_zone $3X_key zone=3x_zone:20m rate=100r/s;
> >   limit_req_zone $2X_key_from_upstream zone=2x_zone_from_upstream:20m
> rate=10r/s;
> >   limit_req_zone $3X_key_from_upstream zone=3x_zone_from_upstream:20m
> rate=100r/s;

If the first limit_req_zone argument is true in each case, the lowest rates is
on the first one, so that is the one that will always take effect.

        f
--
Francis Daly        [hidden email]
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: Throttle requests with limit_req rate based on header from response to auth subrequest

wld75
Francis Daly Wrote:
-------------------------------------------------------

> On Wed, Aug 29, 2018 at 07:14:01PM -0400, jarstewa wrote:
>
> Hi there,
>
> I do not know the answer, and I have not tested the code you provided.
>
> But, one suggestion which might be quick for you to test:
>
> what happens if you change all of your variable names so that they do
> not start with a digit?
>
> As in: rename $2X_key to be (for example) $a2X_key.
>
> It is possible that "$2X_key" will be expanded as "X_key" when $2 has
> no value.
>

Thanks for the suggestion.  I modified my example to have names likes
$key_two rather than $2X_key and I do see the same behavior.

> If the first limit_req_zone argument is true in each case, the lowest
> rates is
> on the first one, so that is the one that will always take effect.

Hm, it seems to me that the first limit_req_zone argument is "", based on
the value of the variable that I see in the logs, so I don't know why that
limit is taking effect.

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,281034,281049#msg-281049

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: Throttle requests with limit_req rate based on header from response to auth subrequest

wld75
In reply to this post by wld75
Digging into this some more today, I've continued to find what seems to be
odd behavior.  If I remove all of the limit_req directives, then the mapped
variables based on the upstream are always present:

{

    "upstream_http_tier": "",
    "tier": "02x",
    "http_tier": "",
    "key_two": "",
    "key_three": "",
    "key_two_from_upstream": "match",
    "key_three_from_upstream": "nonempty",
}

Whereas if I add the limit_req directives that reference "key_*" back in,
then those keys are no longer populated correctly:

{

    "upstream_http_tier": "",
    "tier": "02x",
    "http_tier": "",
    "key_two": "",
    "key_three": "",
    "key_two_from_upstream": "",
    "key_three_from_upstream": "",
}

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,281034,281056#msg-281056

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: Throttle requests with limit_req rate based on header from response to auth subrequest

wld75
Hmm, I notice this from the map documentation:

> Since variables are evaluated only when they are used, the mere
declaration even of a large number of “map” variables does not add any extra
costs to request processing.

Here is what I suspect:

1) The limit_req directive is being processed before the auth subrequest
2) Therefore, when the limit_req is present, the mapped variables are empty
since the header from the auth request is not present.  But when limit_req
is removed, the mapped variables are evaluated by the access_log, which
happens after the auth subrequest has been made.

Can anyone kindly confirm or reject this theory?

Thanks again,
Jared

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,281034,281057#msg-281057

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: Throttle requests with limit_req rate based on header from response to auth subrequest

Francis Daly
On Thu, Aug 30, 2018 at 08:13:33PM -0400, jarstewa wrote:

Hi there,

For what it's worth:

> 1) The limit_req directive is being processed before the auth subrequest
> 2) Therefore, when the limit_req is present, the mapped variables are empty
> since the header from the auth request is not present.  But when limit_req
> is removed, the mapped variables are evaluated by the access_log, which
> happens after the auth subrequest has been made.
>
> Can anyone kindly confirm or reject this theory?

I'm pretty sure that that is what is happening.

In the mail thread you linked to originally, the limit_req part was
being calculated based on something in the request.

You are trying to calculate it based on something not in the request.

I don't have a suggested solution.

Cheers,

        f
--
Francis Daly        [hidden email]
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx