[squid-users] leaking memory In Squid 3.4.6

Steve Hill steve at opendium.com
Thu Oct 9 09:38:51 UTC 2014 

On 08.10.14 15:05, Amos Jeffries wrote:

> New patch added to bug 4088. Please see if it resolves the
> external_acl_type leak.

Despite the external ACL cache leak being plugged, I'm still getting a 
serious memory leak.  This data was captured over night on a production 
server, graphing memory usage against requests:
   http://persephone.opendium.net/~steve/squid-memory.png

The graph starts at around 18:00 yesterday evening, ending at around 
09:00 this morning.  I've included the yellow "requests per minute" line 
so you can see how busy the server is - it starts off pretty quiet in 
the evening and gets quieter through the night, but then traffic picks 
up this morning.

The accounted memory increases slightly through the run, but not 
significantly enough for me to worry about for the time being.  My 
concern is the unaccounted memory rapidly increasing.  From the graph, 
it is clear that it is not leaking a fixed amount per request, but I 
can't figure out what correlates with the leak.

Here's an overview of what this Squid is doing:
	- Single process - no SMP workers
	- External ACLs
	- TPROXY
	- Kerberos and Basic auth.
	- SSL Bump
	- ICAP
	- No memory caching (cache_mem 0)
	- No disk caching (cache_dir isn't set)
	- Almost all non-HTTPS traffic is sent to a parent proxy.
	- HTTPS traffic is sent direct

Config file:

auth_param negotiate program /usr/lib64/squid/negotiate_wrapper_auth 
--ntlm /usr/bin/ntlm_auth --helper-protocol=squid-2.5-ntlmssp --kerberos 
/usr/lib64/squid/negotiate_kerberos_auth -s HTTP/proxy.example.com
auth_param negotiate children 50
auth_param negotiate keep_alive off

auth_param basic program /usr/lib64/squid/basic_pam_auth -r
auth_param basic children 50
auth_param basic realm "Iceni Web Proxy"
auth_param basic credentialsttl 2 hours


shutdown_lifetime 3 seconds
forward_max_tries 40
icap_service_failure_limit -1
host_verify_strict off
spoof_client_ip deny all

logformat iceni %tg.%03tu %6tr %>a %Ss/%03>Hs %<st %rm %ru %[un %Sh/%<a 
%mt "%{User-Agent}>h" %lp
access_log stdio:/var/log/squid-nocache/access.log iceni
cache_log /var/log/squid-nocache/cache.log
cache_store_log none
pid_filename /var/run/squid-nocache.pid
coredump_dir /var/spool/squid-nocache
state_dir /var/run/squid-nocache


######
# ACL definitions
######

external_acl_type preauth cache=0 children-max=1 concurrency=100 ttl=60 
negative_ttl=0 %SRC %>{User-Agent} %URI %METHOD /usr/sbin/squid-preauth 
/etc/iceni/authcached/authcached.psk
acl preauth		external preauth
acl preauth_tproxy	external preauth transparent
acl preauth_ok		note auth_tag preauth_ok
acl preauth_done	note auth_tag preauth_done
acl need_http_auth	note auth_tag http_auth
acl need_cp_auth	note auth_tag cp_auth
acl need_postauth_sync	note auth_tag postauth_sync
acl need_postauth_async	note auth_tag postauth_async

external_acl_type postauth_async cache=0 children-max=1 concurrency=100 
ttl=0 grace=100 %SRC %>{User-Agent} %LOGIN %EXT_USER 
/usr/sbin/squid-postauth /etc/iceni/authcached/authcached.psk
external_acl_type postauth_sync cache=0 children-max=1 concurrency=100 
ttl=0 grace=0 %SRC %>{User-Agent} %LOGIN %EXT_USER 
/usr/sbin/squid-postauth /etc/iceni/authcached/authcached.psk
#external_acl_type postauth_async cache=1 children-max=1 concurrency=100 
ttl=1 negative_ttl=1 grace=100 %SRC %>{User-Agent} %LOGIN %EXT_USER 
/usr/sbin/squid-postauth /etc/iceni/authcached/authcached.psk
#external_acl_type postauth_sync cache=1 children-max=1 concurrency=100 
ttl=1 negative_ttl=1 grace=0 %SRC %>{User-Agent} %LOGIN %EXT_USER 
/usr/sbin/squid-postauth /etc/iceni/authcached/authcached.psk
acl postauth_async	external postauth_async
acl postauth_sync	external postauth_sync

# Show the captive portal login page (use with "http_access deny")
acl show_login_page	src all
deny_info 
302:https://%h/webproxy/captive_portal/captive_portal_login?c=%o 
show_login_page

# A bodge to ensure accesses to this machine aren't authenticated or 
filtered.
# /etc/squid/local_ips is automatically updated by the init script when 
Squid
# starts or reloads, so Squid should be reloaded whenever the machine's IPs
# change (yuck!).
acl local_ips		dst "/etc/squid/local_ips"

acl SSL_ports		port 443

acl Safe_ports		port 80		# http
acl Safe_ports		port 21		# ftp
acl Safe_ports		port 443	# https
acl Safe_ports		port 70		# gopher
acl Safe_ports		port 210	# wais
acl Safe_ports		port 1025-65535	# unregistered ports
acl Safe_ports		port 280	# http-mgmt
acl Safe_ports		port 488	# gss-http
acl Safe_ports		port 591	# filemaker
acl Safe_ports		port 777	# multiling http

# CONNECT matches the encrypted tunnel, https matches the decrypted requests
# inside it when it is bumped.
acl CONNECT		method CONNECT
acl https		proto https

acl proxy_auth		proxy_auth REQUIRED
acl tproxy		myportname tproxy
acl tproxy_ssl		myportname tproxy_ssl

# The "you have been blocked" page comes from the web server on 
localhost and
# needs to be excluded from filtering and being forwarded to the 
upstream proxy.
acl dstdomain_localhost	dstdomain localhost


######
# Start of http_access access control.
######

http_access allow manager localhost
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access deny to_localhost

# Unauthenticated access to the local server
http_access allow local_ips

http_access allow !tproxy !tproxy_ssl !https preauth
http_access allow !preauth_done preauth_tproxy
http_access allow need_http_auth need_postauth_sync proxy_auth postauth_sync
http_access allow need_http_auth need_postauth_async proxy_auth 
postauth_async
http_access allow need_http_auth proxy_auth

http_access deny preauth_ok show_login_page

http_access deny all


######
# Other services
######

icp_access deny all
htcp_access deny all


######
# SSL bumping - 
http://www.squid-cache.org/mail-archive/squid-dev/201206/0089.html
# When the web filter wants a CONNECT request to be bumped it sets the
# icap_says_bump header on it, which we trap for here.  Transparently
# proxied SSL connections are always bumped.
######

acl icap_says_bump req_header X-SSL-Bump -i Yes
ssl_bump server-first icap_says_bump
ssl_bump server-first tproxy_ssl
sslproxy_cert_error allow all


######
# Listening ports
######

http_port 3128 ssl-bump generate-host-certificates=on 
cert=/etc/pki/tls/certs/squid-sslbump.crt 
key=/etc/pki/tls/private/squid-sslbump.key
http_port 8080 ssl-bump generate-host-certificates=on 
cert=/etc/pki/tls/certs/squid-sslbump.crt 
key=/etc/pki/tls/private/squid-sslbump.key
http_port 3130 tproxy name=tproxy
https_port 3131 ssl-bump generate-host-certificates=on 
cert=/etc/pki/tls/certs/squid-sslbump.crt 
key=/etc/pki/tls/private/squid-sslbump.key tproxy name=tproxy_ssl


######
# Set a Netfilter mark on transparently proxied connections so they can have
# special routing
######

tcp_outgoing_mark 0x2 tproxy
tcp_outgoing_mark 0x2 tproxy_ssl


######
# Since we do no caching in this instance of Squid, we use a second 
instance as
# an upstream caching proxy.  For efficiency reasons we try to send 
uncachable
# traffic directly to the web server rather than via the upstream proxy.
######

cache_peer [::1] parent 3129 0 proxy-only no-query no-digest no-tproxy 
name=caching
cache_peer_access caching deny CONNECT
cache_peer_access caching deny https
cache_peer_access caching deny tproxy_ssl
cache_peer_access caching deny to_localhost
cache_peer_access caching deny dstdomain_localhost
cache_peer_access caching allow all

cache_mem 0
cache deny all
never_direct deny CONNECT
never_direct deny https
never_direct deny tproxy_ssl
never_direct deny to_localhost
never_direct deny dstdomain_localhost
never_direct allow all


######
# Interface with the web filter
######
icap_enable on
icap_service_revival_delay 30
icap_preview_enable on
icap_preview_size 50000
icap_send_client_ip on
icap_send_client_username on

icap_service iceni_reqmod_precache reqmod_precache 0 
icap://localhost6:1344/reqmod_precache
icap_service iceni_respmod_postcache respmod_precache 0 
icap://localhost6:1344/respmod_postcache

adaptation_service_set iceni_reqmod_precache iceni_reqmod_precache
adaptation_service_set iceni_respmod_postcache iceni_respmod_postcache

adaptation_access iceni_reqmod_precache deny local_ips
adaptation_access iceni_reqmod_precache deny to_localhost
adaptation_access iceni_reqmod_precache deny dstdomain_localhost
adaptation_access iceni_reqmod_precache allow all

adaptation_access iceni_respmod_postcache deny local_ips
adaptation_access iceni_respmod_postcache deny to_localhost
adaptation_access iceni_respmod_postcache deny dstdomain_localhost
adaptation_access iceni_respmod_postcache allow all



-- 
  - Steve Hill
    Technical Director
    Opendium Limited     http://www.opendium.com

Direct contacts:
    Instant messager: xmpp:steve at opendium.com
    Email:            steve at opendium.com
    Phone:            sip:steve at opendium.com

Sales / enquiries contacts:
    Email:            sales at opendium.com
    Phone:            +44-1792-824568 / sip:sales at opendium.com

Support contacts:
    Email:            support at opendium.com
    Phone:            +44-1792-825748 / sip:support at opendium.com

More information about the squid-users mailing list