[squid-users] TCP: out of memory -- consider tuning tcp_mem

Mon Sep 18 17:01:42 UTC 2017

Hi again,

I'm suddenly getting these errors in the log:

2017/09/18 18:13:48 kid1| Error negotiating SSL on FD 11010: error:1409F07F:SSL routines:ssl3_write_pending:bad write retry (1/-1/0)
2017/09/18 18:13:57 kid1| Error negotiating SSL on FD 11124: error:1409F07F:SSL routines:ssl3_write_pending:bad write retry (1/-1/0)
2017/09/18 18:13:57 kid1| Error negotiating SSL on FD 11124: error:1409F07F:SSL routines:ssl3_write_pending:bad write retry (1/-1/0)
2017/09/18 18:14:00 kid1| Error negotiating SSL connection on FD 11064: error:1408A0C1:SSL routines:ssl3_get_client_hello:no shared cipher (1/-1)
2017/09/18 18:14:00 kid1| Error negotiating SSL connection on FD 11064: error:1408A0C1:SSL routines:ssl3_get_client_hello:no shared cipher (1/-1)
2017/09/18 18:14:03 kid1| Error negotiating SSL connection on FD 10857: error:1408A0C1:SSL routines:ssl3_get_client_hello:no shared cipher (1/-1)
2017/09/18 18:14:04 kid1| Error negotiating SSL connection on FD 10857: error:1408A0C1:SSL routines:ssl3_get_client_hello:no shared cipher (1/-1)

This must be a kernel issue because I'm getting lots of these in /var/log/messages:

kernel: TCP: out of memory -- consider tuning tcp_mem

Here are my values:

# sysctl net.ipv4.tcp_mem
net.ipv4.tcp_mem = 384027       512036  768054
# sysctl net.ipv4.tcp_rmem
net.ipv4.tcp_rmem = 4096        87380   6291456
# sysctl net.ipv4.tcp_wmem
net.ipv4.tcp_wmem = 4096        16384   4194304
# sysctl net.core.rmem_max
net.core.rmem_max = 212992
# sysctl net.core.wmem_max
net.core.wmem_max = 212992

# uname -a
Linux inf-fw2 4.9.34-gentoo #1 SMP Mon Jul 10 11:05:23 CEST 2017 x86_64 AMD FX(tm)-8320 Eight-Core Processor AuthenticAMD GNU/Linux

# top
top - 17:51:33 up 19 days, 10:18,  2 users,  load average: 1.38, 1.49, 1.42
Tasks: 344 total,   1 running, 343 sleeping,   0 stopped,   0 zombie
%Cpu0  :  2.2 us,  0.5 sy,  0.0 ni, 93.0 id,  0.0 wa,  0.0 hi,  4.3 si,  0.0 st
%Cpu1  :  0.5 us,  0.0 sy,  0.0 ni, 97.9 id,  0.0 wa,  0.0 hi,  1.6 si,  0.0 st
%Cpu2  :  1.1 us,  0.0 sy,  0.5 ni, 95.2 id,  0.0 wa,  0.0 hi,  3.2 si,  0.0 st
%Cpu3  :  1.1 us,  0.5 sy,  0.0 ni, 96.3 id,  0.0 wa,  0.0 hi,  2.1 si,  0.0 st
%Cpu4  :  2.1 us,  0.0 sy,  0.0 ni, 96.3 id,  0.0 wa,  0.0 hi,  1.6 si,  0.0 st
%Cpu5  :  0.5 us,  0.0 sy,  0.0 ni, 98.9 id,  0.0 wa,  0.0 hi,  0.5 si,  0.0 st
%Cpu6  :  0.5 us,  1.1 sy,  0.0 ni, 96.8 id,  0.0 wa,  0.0 hi,  1.6 si,  0.0 st
%Cpu7  :  1.6 us,  0.0 sy,  0.0 ni, 90.9 id,  0.0 wa,  0.0 hi,  7.5 si,  0.0 st
KiB Mem : 32865056 total,   820664 free, 20358972 used, 11685420 buff/cache
KiB Swap: 37036988 total, 34924984 free,  2112004 used. 12014564 avail Mem

# cat /proc/net/sockstat
sockets: used 13121
TCP: inuse 10010 orphan 11 tw 246 alloc 12597 mem 772909
UDP: inuse 92 mem 59
UDPLITE: inuse 0
RAW: inuse 7
FRAG: inuse 0 memory 0

# cat /proc/net/sockstat6
TCP6: inuse 282
UDP6: inuse 40
UDPLITE6: inuse 0
RAW6: inuse 5
FRAG6: inuse 0 memory 0

#  sysctl -a |grep tcp
fs.nfs.nfs_callback_tcpport = 0
fs.nfs.nlm_tcpport = 0
net.ipv4.tcp_abort_on_overflow = 0
net.ipv4.tcp_adv_win_scale = 1
net.ipv4.tcp_allowed_congestion_control = cubic reno
net.ipv4.tcp_app_win = 31
net.ipv4.tcp_autocorking = 1
net.ipv4.tcp_available_congestion_control = cubic reno
net.ipv4.tcp_base_mss = 1024
net.ipv4.tcp_challenge_ack_limit = 1000
sysctl: net.ipv4.tcp_congestion_control = cubic
reading key "net.ipv6.conf.all.stable_secret"net.ipv4.tcp_dsack = 1

net.ipv4.tcp_early_retrans = 3
net.ipv4.tcp_ecn = 2
net.ipv4.tcp_ecn_fallback = 1
net.ipv4.tcp_fack = 1
net.ipv4.tcp_fastopen = 1
net.ipv4.tcp_fastopen_key = 6707aeac-2dd079df-0dee3da3-befd1107
net.ipv4.tcp_fin_timeout = 60
net.ipv4.tcp_frto = 2
net.ipv4.tcp_fwmark_accept = 0
net.ipv4.tcp_invalid_ratelimit = 500
net.ipv4.tcp_keepalive_intvl = 75
net.ipv4.tcp_keepalive_probes = 9
net.ipv4.tcp_keepalive_time = 7200
net.ipv4.tcp_limit_output_bytes = 262144
net.ipv4.tcp_low_latency = 0
net.ipv4.tcp_max_orphans = 131072
net.ipv4.tcp_max_reordering = 300
net.ipv4.tcp_max_syn_backlog = 1024
net.ipv4.tcp_max_tw_buckets = 131072
net.ipv4.tcp_mem = 384027       512036  768054
net.ipv4.tcp_min_rtt_wlen = 300
net.ipv4.tcp_min_tso_segs = 2
net.ipv4.tcp_moderate_rcvbuf = 1
net.ipv4.tcp_mtu_probing = 0
net.ipv4.tcp_no_metrics_save = 0
net.ipv4.tcp_notsent_lowat = -1
net.ipv4.tcp_orphan_retries = 0
net.ipv4.tcp_pacing_ca_ratio = 120
net.ipv4.tcp_pacing_ss_ratio = 200
net.ipv4.tcp_probe_interval = 600
net.ipv4.tcp_probe_threshold = 8
net.ipv4.tcp_recovery = 1
net.ipv4.tcp_reordering = 3
net.ipv4.tcp_retrans_collapse = 1
net.ipv4.tcp_retries1 = 3
net.ipv4.tcp_retries2 = 15
net.ipv4.tcp_rfc1337 = 0
net.ipv4.tcp_rmem = 4096        87380   6291456
net.ipv4.tcp_sack = 1
net.ipv4.tcp_slow_start_after_idle = 1
net.ipv4.tcp_stdurg = 0
net.ipv4.tcp_syn_retries = 6
net.ipv4.tcp_synack_retries = 5
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_thin_dupack = 0
net.ipv4.tcp_thin_linear_timeouts = 0
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_tso_win_divisor = 3
net.ipv4.tcp_tw_recycle = 0
net.ipv4.tcp_tw_reuse = 0
net.ipv4.tcp_window_scaling = 1
net.ipv4.tcp_wmem = 4096        16384   4194304
net.ipv4.tcp_workaround_signed_windows = 0
sysctl: reading key "net.ipv6.conf.default.stable_secret"
sysctl: reading key "net.ipv6.conf.enp10s0.stable_secret"
sysctl: reading key "net.ipv6.conf.enp5s0.stable_secret"
sysctl: reading key "net.ipv6.conf.enp6s0.stable_secret"
sysctl: reading key "net.ipv6.conf.enp7s0f0.stable_secret"
sysctl: reading key "net.ipv6.conf.enp7s0f1.stable_secret"
sysctl: reading key "net.ipv6.conf.enp7s0f2.stable_secret"
sysctl: reading key "net.ipv6.conf.enp7s0f3.stable_secret"
sysctl: reading key "net.ipv6.conf.enp8s5.stable_secret"
sysctl: reading key "net.ipv6.conf.lo.stable_secret"
net.netfilter.nf_conntrack_tcp_be_liberal = 0
net.netfilter.nf_conntrack_tcp_loose = 1
net.netfilter.nf_conntrack_tcp_max_retrans = 3
net.netfilter.nf_conntrack_tcp_timeout_close = 10
net.netfilter.nf_conntrack_tcp_timeout_close_wait = 60
net.netfilter.nf_conntrack_tcp_timeout_established = 432000
net.netfilter.nf_conntrack_tcp_timeout_fin_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_last_ack = 30
net.netfilter.nf_conntrack_tcp_timeout_max_retrans = 300
net.netfilter.nf_conntrack_tcp_timeout_syn_recv = 60
net.netfilter.nf_conntrack_tcp_timeout_syn_sent = 120
net.netfilter.nf_conntrack_tcp_timeout_time_wait = 120
net.netfilter.nf_conntrack_tcp_timeout_unacknowledged = 300

Obviously, I'm having connection issues now.

Some suggest to increase tcp_mem, others say not to, but increase the other values such as:

sysctl -w net.core.rmem_max=8738000
sysctl -w net.core.wmem_max=6553600
sysctl -w net.ipv4.tcp_rmem=8192 873800 8738000
sysctl -w net.ipv4.tcp_wmem=4096 655360 6553600

Others suggest to also increase net.ipv4.tcp_max_orphans.

Can anyone please advise?
Why aren't the kernel defaults enough?
In any case, how should I calculate my optimum values given my RAM?

Also, if the kernel defaults are sensible then how can I find out if there's a memory leak?

If I stop/start squid 3.5.26 then the issue is solved, at least for some time.

Thanks,

Vieri