nginx error : cache file has too long header

classic Classic list List threaded Threaded
14 messages Options
xrd
Reply | Threaded
Open this post in threaded view
|

nginx error : cache file has too long header

xrd
Hi ,

We are getting too many Cache file has too long header errors in nginx
error.log file. Some servers are getting such error as many as 273 times a
day.
Below is sample nginx file in our environment :
worker_processes  auto;
events {
 worker_connections  4096;
 use epoll;
 multi_accept on;
}
worker_rlimit_nofile   100001;
http {
 include       mime.types;
 default_type  video/mp4;
 proxy_buffering           on;
 proxy_buffer_size         4096k;
 proxy_buffers             5 4096k;
 sendfile                  on;
 keepalive_timeout         30;
 tcp_nodelay               on;
 tcp_nopush                on;
 reset_timedout_connection on;
 gzip                      off;
 server_tokens          off;
log_format access '$remote_addr $http_x_forwarded_for $host [$time_local] '
'$upstream_cache_status ' '"$request" $status $body_bytes_sent '
'"$http_referer" "$http_user_agent" $request_time' ' Patna
sptnacdnds01.cdnsrv.jio.com sptnacdnds01 DS';
proxy_cache_path    /cache/11452 keys_zone=a11452:8m levels=1:2 max_size=50g
inactive=10m;
proxy_cache_path    /cache/11506 keys_zone=a11506:8m levels=1:2 max_size=50g
inactive=10m;
proxy_cache_path    /cache/12151 keys_zone=a12151:200m levels=1:2
max_size=100g inactive=10m;
proxy_cache_path    /cache/12053 keys_zone=a12053:200m levels=1:2
max_size=50g inactive=10m;
proxy_cache_path    /cache/11502 keys_zone=a11502:50m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/11503 keys_zone=a11503:50m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/11504 keys_zone=a11504:50m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/11505 keys_zone=a11505:50m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/11507 keys_zone=a11507:50m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/12202 keys_zone=a12202:200m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/12201 keys_zone=a12201:200m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/12003 keys_zone=a12003:200m levels=1:2
max_size=700g inactive=10d;
proxy_cache_path    /cache/12008 keys_zone=a12008:300m levels=1:2
max_size=20g inactive=10d;
proxy_cache_path    /cache/12007 keys_zone=a12007:200m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/12152 keys_zone=a12152:200m levels=1:2
max_size=700g inactive=10d;
proxy_cache_path    /cache/12005 keys_zone=a12005:200m levels=1:2
max_size=100g inactive=10d;
proxy_cache_path    /cache/12153 keys_zone=a12153:200m levels=1:2
max_size=100g inactive=10d;
proxy_cache_path    /cache/12006 keys_zone=a12006:200m levels=1:2
max_size=100g inactive=10d;
proxy_cache_path    /cache/11501 keys_zone=a11501:50m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/12054 keys_zone=a12054:200m levels=1:2
max_size=500g inactive=3d;
proxy_cache_path    /cache/12251 keys_zone=a12251:200m levels=1:2
max_size=200g inactive=15m;
proxy_cache_path    /cache/12252 keys_zone=a12252:200m levels=1:2
max_size=200g inactive=10d;
proxy_cache_path    /cache/12301 keys_zone=a12301:200m levels=1:2
max_size=200g inactive=10d;


Please suggest some solution.

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271382#msg-271382

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

Roman Arutyunyan
Hi,

On Wed, Dec 07, 2016 at 02:02:32AM -0500, omkar_jadhav_20 wrote:
> Hi ,
>
> We are getting too many Cache file has too long header errors in nginx
> error.log file. Some servers are getting such error as many as 273 times a
> day.

You should increase proxy_buffer_size.

> Below is sample nginx file in our environment :
> worker_processes  auto;
> events {
>  worker_connections  4096;
>  use epoll;
>  multi_accept on;
> }
> worker_rlimit_nofile   100001;
> http {
>  include       mime.types;
>  default_type  video/mp4;
>  proxy_buffering           on;
>  proxy_buffer_size         4096k;
>  proxy_buffers             5 4096k;
>  sendfile                  on;
>  keepalive_timeout         30;
>  tcp_nodelay               on;
>  tcp_nopush                on;
>  reset_timedout_connection on;
>  gzip                      off;
>  server_tokens          off;
> log_format access '$remote_addr $http_x_forwarded_for $host [$time_local] '
> '$upstream_cache_status ' '"$request" $status $body_bytes_sent '
> '"$http_referer" "$http_user_agent" $request_time' ' Patna
> sptnacdnds01.cdnsrv.jio.com sptnacdnds01 DS';
> proxy_cache_path    /cache/11452 keys_zone=a11452:8m levels=1:2 max_size=50g
> inactive=10m;
> proxy_cache_path    /cache/11506 keys_zone=a11506:8m levels=1:2 max_size=50g
> inactive=10m;
> proxy_cache_path    /cache/12151 keys_zone=a12151:200m levels=1:2
> max_size=100g inactive=10m;
> proxy_cache_path    /cache/12053 keys_zone=a12053:200m levels=1:2
> max_size=50g inactive=10m;
> proxy_cache_path    /cache/11502 keys_zone=a11502:50m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/11503 keys_zone=a11503:50m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/11504 keys_zone=a11504:50m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/11505 keys_zone=a11505:50m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/11507 keys_zone=a11507:50m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/12202 keys_zone=a12202:200m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/12201 keys_zone=a12201:200m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/12003 keys_zone=a12003:200m levels=1:2
> max_size=700g inactive=10d;
> proxy_cache_path    /cache/12008 keys_zone=a12008:300m levels=1:2
> max_size=20g inactive=10d;
> proxy_cache_path    /cache/12007 keys_zone=a12007:200m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/12152 keys_zone=a12152:200m levels=1:2
> max_size=700g inactive=10d;
> proxy_cache_path    /cache/12005 keys_zone=a12005:200m levels=1:2
> max_size=100g inactive=10d;
> proxy_cache_path    /cache/12153 keys_zone=a12153:200m levels=1:2
> max_size=100g inactive=10d;
> proxy_cache_path    /cache/12006 keys_zone=a12006:200m levels=1:2
> max_size=100g inactive=10d;
> proxy_cache_path    /cache/11501 keys_zone=a11501:50m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/12054 keys_zone=a12054:200m levels=1:2
> max_size=500g inactive=3d;
> proxy_cache_path    /cache/12251 keys_zone=a12251:200m levels=1:2
> max_size=200g inactive=15m;
> proxy_cache_path    /cache/12252 keys_zone=a12252:200m levels=1:2
> max_size=200g inactive=10d;
> proxy_cache_path    /cache/12301 keys_zone=a12301:200m levels=1:2
> max_size=200g inactive=10d;
>
>
> Please suggest some solution.
>
> Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271382#msg-271382
>
> _______________________________________________
> nginx mailing list
> [hidden email]
> http://mailman.nginx.org/mailman/listinfo/nginx

--
Roman Arutyunyan
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
Please note that we already have proxy_buffer_size 4096k, please suggest
what should be ideal size for proxy_buffer_size in this scenario.

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271389#msg-271389

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

Roman Arutyunyan
On Wed, Dec 07, 2016 at 05:24:23AM -0500, omkar_jadhav_20 wrote:
> Please note that we already have proxy_buffer_size 4096k, please suggest
> what should be ideal size for proxy_buffer_size in this scenario.

Yes, I saw that.  But it looks like you have too long proxy_cache_key.
Or maybe, those files producing errors are just broken for some reason.

Can you read those files?  Each cache file has the following parts in it:

- Serialized struct ngx_http_file_cache_header_t (unreadable characters in
  the beginnig of the file).  This one is small.
- Cache key in plaintext.  You can easily see, if it's really long.
- Response header
- Response body

--
Roman Arutyunyan
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
I can open and read such cache files and below things are there inside these
files :

KEY: (this is one liner key)
HTTP/1.1 200 OK
x-mobi-fs-ver:
X-Frame-Options:
Cache-Control:
Last-Modified:
ETag:
Content-Type:
Content-Length:
Accept-Ranges:

etc. and at last a very long binary content.

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271392#msg-271392

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

Roman Arutyunyan
On Wed, Dec 07, 2016 at 06:00:34AM -0500, omkar_jadhav_20 wrote:

> I can open and read such cache files and below things are there inside these
> files :
>
> KEY: (this is one liner key)
> HTTP/1.1 200 OK
> x-mobi-fs-ver:
> X-Frame-Options:
> Cache-Control:
> Last-Modified:
> ETag:
> Content-Type:
> Content-Length:
> Accept-Ranges:
>
> etc. and at last a very long binary content.

Is this one of the files, which produce 'too long header' errors?

Is the file offset of the point, where the body starts, greater than 4096?
That part of the cache file (file header + key + HTTP header) should fit in
proxy_buffer_size.

--
Roman Arutyunyan
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
Yes this is one of those cache file for which we received said error.
I can see total size of this file is just 5300 bytes however we have set
proxy_buffer size set to 4096k
ls -lrt /cache/12054/1/fd/54ab395a128225b98118b08cf9d89fd1
x 5300 Dec  7 16:21 /cache/12054/1/fd/54ab395a128225b98118b08cf9d89fd1

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271394#msg-271394

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

Roman Arutyunyan
On Wed, Dec 07, 2016 at 06:34:29AM -0500, omkar_jadhav_20 wrote:
> Yes this is one of those cache file for which we received said error.
> I can see total size of this file is just 5300 bytes however we have set
> proxy_buffer size set to 4096k
> ls -lrt /cache/12054/1/fd/54ab395a128225b98118b08cf9d89fd1
> x 5300 Dec  7 16:21 /cache/12054/1/fd/54ab395a128225b98118b08cf9d89fd1

Could you post hexdump -C of the file header (what's before the key) ?

--
Roman Arutyunyan
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
Please find the below :
00000000  fe eb 47 58 00 00 00 00  f3 6f 47 58 00 00 00 00
|..GX.....oGX....|
00000010  a6 e9 47 58 00 00 00 00  e9 9a 8c e8 00 00 a6 00
|..GX............|
00000020  1f 02 00 00 00 00 00 00  0a 4b 45 59 3a 20 31 30  |.........KEY:
10|

Please let me know if anything else is required.

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271397#msg-271397

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

Roman Arutyunyan
On Wed, Dec 07, 2016 at 07:21:01AM -0500, omkar_jadhav_20 wrote:
> Please find the below :
> 00000000  fe eb 47 58 00 00 00 00  f3 6f 47 58 00 00 00 00
> |..GX.....oGX....|
> 00000010  a6 e9 47 58 00 00 00 00  e9 9a 8c e8 00 00 a6 00
> |..GX............|
> 00000020  1f 02 00 00 00 00 00 00  0a 4b 45 59 3a 20 31 30  |.........KEY:
> 10|
>
> Please let me know if anything else is required.

What nginx version do you have?

--
Roman Arutyunyan
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

Roman Arutyunyan
On Wed, Dec 07, 2016 at 03:31:58PM +0300, Roman Arutyunyan wrote:

> On Wed, Dec 07, 2016 at 07:21:01AM -0500, omkar_jadhav_20 wrote:
> > Please find the below :
> > 00000000  fe eb 47 58 00 00 00 00  f3 6f 47 58 00 00 00 00
> > |..GX.....oGX....|
> > 00000010  a6 e9 47 58 00 00 00 00  e9 9a 8c e8 00 00 a6 00
> > |..GX............|
> > 00000020  1f 02 00 00 00 00 00 00  0a 4b 45 59 3a 20 31 30  |.........KEY:
> > 10|
> >
> > Please let me know if anything else is required.
>
> What nginx version do you have?

I see, the cache file is from an old nginx version.
Anyway, the file seems to be ok.

--
Roman Arutyunyan
_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
In reply to this post by Roman Arutyunyan
nginx-1.4.0 is the version but there are many servers on which we have
installed nginx with version nginx-1.10.2 , giving same error:
below is hexdump output before KEY from those servers on which nginx with
version 1.10.2 is running :

00000000  03 00 00 00 00 00 00 00  a8 09 48 58 00 00 00 00
|..........HX....|
00000010  99 ee 47 58 00 00 00 00  50 07 48 58 00 00 00 00
|..GX....P.HX....|
00000020  6d 50 49 e8 00 00 d6 00  65 02 22 22 63 37 32 36
|mPI.....e.""c726|
00000030  64 30 35 38 39 35 37 61  34 62 32 36 62 31 35 62
|d058957a4b26b15b|
00000040  35 32 31 64 31 66 65 64  31 65 36 33 22 00 00 00
|521d1fed1e63"...|
00000050  00 00 00 00 00 00 00 00  00 00 00 00 00 00 00 00
|................|
*
00000090  0a 4b 45 59 3a 20 68 74  74 70 3a 2f 2f 4c 49 56  |.KEY:
http://LIV|

-------------------------------------
actual size of the cache file -
ls -lh /cache/12053/c/71/be411798f851da373230d41a50d5971c
 458K Dec  7 18:27 /cache/12053/c/71/be411798f851da373230d41a50d5971c

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271402#msg-271402

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
can someone please assist here ...

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,271439#msg-271439

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx
xrd
Reply | Threaded
Open this post in threaded view
|

Re: nginx error : cache file has too long header

xrd
In reply to this post by Roman Arutyunyan
Is time of last request also stored with the cached file on disk? If not,
how does nginx track inactive timeout if it gets restarted?

Posted at Nginx Forum: https://forum.nginx.org/read.php?2,271382,285638#msg-285638

_______________________________________________
nginx mailing list
[hidden email]
http://mailman.nginx.org/mailman/listinfo/nginx