[squid-users] Squid 3.5.11 always can't cache Wikipedia content
Yuri Voinov
yvoinov at gmail.com
Thu Nov 26 16:01:15 UTC 2015
-----BEGIN PGP SIGNED MESSAGE-----
Hash: SHA256
One simple Wikipedia URL.
This is second-third-etc. query access log:
http://i.imgur.com/AOpeSQx.png
This is redbot URL info:
http://i.imgur.com/dmwjH26.png
This is squid.conf (3.5.11 latest build):
# -------------------------------------
# Access Control Lists
# -------------------------------------
acl localnet src 192.168.0.0/16 # RFC1918 possible internal network
acl SSL_ports port 443
acl SSL_ports port 8443 # Telecom exclusion
acl SSL_ports port 2041 # ICQ/MRA
acl SSL_ports port 2042 # ICQ/MRA
acl SSL_ports port 5160 # ICQ/MRA
acl SSL_ports port 5228 # ICQ/MRA
acl Safe_ports port 80 # http
acl Safe_ports port 21 # ftp
acl Safe_ports port 443 # https
acl Safe_ports port 70 # gopher
acl Safe_ports port 210 # wais
acl Safe_ports port 1025-65535 # unregistered ports
acl Safe_ports port 280 # http-mgmt
acl Safe_ports port 488 # gss-http
acl Safe_ports port 591 # filemaker
acl Safe_ports port 777 # multiling http
# Common methods
acl CONNECT method CONNECT
acl PURGE method PURGE
acl GET method GET
# Windows update acls
acl windowsupdate dstdomain sls.update.microsoft.com.akadns.net
acl windowsupdate dstdomain windowsupdate.microsoft.com
acl windowsupdate dstdomain .update.microsoft.com
acl windowsupdate dstdomain download.windowsupdate.com
acl windowsupdate dstdomain redir.metaservices.microsoft.com
acl windowsupdate dstdomain images.metaservices.microsoft.com
acl windowsupdate dstdomain c.microsoft.com
acl windowsupdate dstdomain www.download.windowsupdate.com
acl windowsupdate dstdomain wustat.windows.com
acl windowsupdate dstdomain crl.microsoft.com
acl windowsupdate dstdomain sls.microsoft.com
acl windowsupdate dstdomain productactivation.one.microsoft.com
acl windowsupdate dstdomain ntservicepack.microsoft.com
# Windows update methods
acl wuCONNECT dstdomain www.update.microsoft.com
acl wuCONNECT dstdomain sls.microsoft.com
# Youtube & CDN store rewrite ACLs
acl store_rewrite_list urlpath_regex
\.(jp(e?g|e|2)|gif|png|bmp|ico|svg|web(p|m)|flv|f4f|mp(3|4)|ttf|eot|woff2?|(c|x|j)ss|js(t?|px?))\?
\/ads\?
acl store_rewrite_list_web url_regex "/usr/local/squid/etc/url.rewrite_web"
acl store_rewrite_list_web_cdn url_regex
"/usr/local/squid/etc/url.rewrite_cdn"
# Adobe/Java and other updates
acl adobe_java_updates url_regex "/usr/local/squid/etc/url.updates"
# No-cache
acl dont_cache_url url_regex "/usr/local/squid/etc/url.nocache"
# Tor acl
acl tor_url dstdom_regex -i "/usr/local/squid/etc/url.tor"
# SSL bump acl
acl net_bump src "/usr/local/squid/etc/net.bump"
# TLD acl
acl block_tld dstdomain "/usr/local/squid/etc/dstdom.tld"
# -------------------------------------
# Access parameters
# -------------------------------------
# Deny requests to unsafe ports
http_access deny !Safe_ports
# Deny CONNECT to other than SSL ports
http_access deny CONNECT !SSL_ports
# Only allow cachemgr access from localhost
http_access allow localhost manager
http_access deny manager
http_access deny to_localhost
# Allow purge from localhost
http_access allow PURGE localhost
http_access deny PURGE
# Block top level domains
http_access deny block_tld
deny_info TCP_RESET block_tld
# Rule allowing access from local networks
http_access allow localnet
http_access allow localhost
# No cache directives
cache deny dont_cache_url
# ICP/HTCP access
#icp_access allow localnet
#icp_access deny all
#htcp_access allow localnet
#htcp_access deny all
# For YT block useragent header
acl user-agent-browser browser -i
(android|blackberry|iphone|ipad|ipod|opera mini|iemobile)
acl youtube_dom dstdomain .youtube.com
request_header_access User-Agent deny youtube_dom !user-agent-browser
request_header_replace User-Agent Mozilla/5.0 (compatible;
Googlebot/2.1; +http://www.google.com/bot.html)
# Normalize Accept-Encoding to support compression via eCAP
request_header_access Accept-Encoding deny all
request_header_replace Accept-Encoding gzip;q=1.0, identity;q=0.5, *;q=0
# Disable alternate protocols
request_header_access Alternate-Protocol deny all
reply_header_access Alternate-Protocol deny all
# Disable HSTS
reply_header_access Strict-Transport-Security deny all
reply_header_replace Strict-Transport-Security max-age=0; includeSubDomains
# Remove User-Agent from Vary
reply_header_access Vary deny all
reply_header_replace Vary Accept-Encoding
# 302 loop
acl text_mime rep_mime_type text/html text/plain
acl http302 http_status 302
store_miss deny text_mime http302
send_hit deny text_mime http302
# Windows updates rules
http_access allow CONNECT wuCONNECT localnet
http_access allow CONNECT wuCONNECT localhost
http_access allow windowsupdate localnet
http_access allow windowsupdate localhost
# SSL bump rules
sslproxy_cert_error allow all
acl DiscoverSNIHost at_step SslBump1
ssl_bump peek DiscoverSNIHost
acl NoSSLIntercept ssl::server_name_regex -i
"/usr/local/squid/etc/url.nobump"
acl NoSSLIntercept ssl::server_name_regex -i "/usr/local/squid/etc/url.tor"
ssl_bump splice NoSSLIntercept
ssl_bump bump net_bump
# Privoxy+Tor access rules
never_direct allow tor_url
# And finally deny all other access to this proxy
http_access deny all
# -------------------------------------
# HTTP parameters
# -------------------------------------
# Local Privoxy is cache parent
cache_peer 127.0.0.1 parent 8118 0 no-query no-digest default
cache_peer_access 127.0.0.1 allow tor_url
cache_peer_access 127.0.0.1 deny all
# Don't cache 404 long time
negative_ttl 5 minutes
positive_dns_ttl 15 hours
negative_dns_ttl 1 minutes
# -------------------------------------
# Cache parameters
# -------------------------------------
http_port 3126 ssl-bump generate-host-certificates=on
dynamic_cert_mem_cache_size=4MB cert=/usr/local/squid/etc/rootCA.crt
key=/usr/local/squid/etc/rootCA.key options=NO_SSLv3
dhparams=/usr/local/squid/etc/dhparam.pem
http_port 3127
http_port 3128 intercept
https_port 3129 intercept ssl-bump generate-host-certificates=on
dynamic_cert_mem_cache_size=4MB cert=/usr/local/squid/etc/rootCA.crt
key=/usr/local/squid/etc/rootCA.key options=NO_SSLv3
dhparams=/usr/local/squid/etc/dhparam.pem
sslproxy_capath /etc/opt/csw/ssl/certs
#sslproxy_cafile /usr/local/squid/etc/ca-bundle.crt
sslproxy_options NO_SSLv3,SINGLE_DH_USE
sslproxy_cipher
EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:HIGH:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS
sslcrtd_program /usr/local/squid/libexec/ssl_crtd -s /var/lib/ssl_db -M 4MB
cachemgr_passwd disable shutdown reconfigure rotate
# Cache user
cache_effective_user squid
cache_effective_group squid
# Turn off collect per-client statistics
client_db off
# Forces reload-into-ims
reload_into_ims on
# Hide internal networks details outside
via off
forwarded_for delete
# Do not show Squid version
httpd_suppress_version_string on
# WCCPv2 parameters
wccp2_router 192.168.200.2
wccp2_forwarding_method l2
wccp2_return_method l2
wccp2_rebuild_wait off
wccp2_service standard 0
wccp2_service dynamic 70
wccp2_service_info 70 protocol=tcp
flags=dst_ip_hash,src_ip_alt_hash,src_port_alt_hash priority=240 ports=443
# Prioritization of local hits
qos_flows tos local-hit=0x68
# Default is on
digest_generation off
# -------------------------------------
# Adaptation parameters
# -------------------------------------
icap_enable on
icap_service_failure_limit -1
icap_send_client_ip on
icap_send_client_username on
icap_client_username_header X-Authenticated-User
icap_preview_enable on
icap_preview_size 1024
icap_service service_avi_req reqmod_precache
icap://localhost:1344/squidclamav bypass=off
adaptation_access service_avi_req allow all
icap_service service_avi_resp respmod_precache
icap://localhost:1344/squidclamav bypass=on
adaptation_access service_avi_resp allow all
ecap_enable on
acl HTTP_STATUS_OK http_status 200
loadable_modules /usr/local/lib/ecap_adapter_gzip.so
ecap_service gzip_service respmod_precache
ecap://www.vigos.com/ecap_gzip bypass=off
adaptation_access gzip_service allow HTTP_STATUS_OK
# -------------------------------------
# Store parameters
# -------------------------------------
maximum_object_size 8 Gb
cache_dir aufs /data/cache/d1 32767 16 256
cache_dir aufs /data/cache/d2 32767 16 256
cache_dir aufs /data/cache/d3 32767 16 256
cache_dir aufs /data/cache/d4 32767 16 256
# -------------------------------------
# Memory parameters
# -------------------------------------
cache_mem 512 Mb
#memory_pools off
# -------------------------------------
# Tuning parameters
# -------------------------------------
memory_replacement_policy heap GDSF
cache_replacement_policy heap LFUDA
# Shutdown delay before terminate connections
shutdown_lifetime 15 second
# -------------------------------------
# Process/log parameters
# -------------------------------------
access_log daemon:/data/cache/log/access.log buffer-size=256KB !tor_url
# Don't log ICP queries
#log_icp_queries off
# Turn off internal log rotation
logfile_rotate 0
cache_log /data/cache/log/cache.log
cache_store_log none
# Default is off
buffered_logs on
coredump_dir /var/core
pid_filename /tmp/squid.pid
strip_query_terms off
# -------------------------------------
# Content parameters
# -------------------------------------
range_offset_limit none all
quick_abort_min -1 KB
# Updates: Windows, Adobe, Java
refresh_pattern -i
microsoft.com.*\.(cab|exe|ms[i|u|f|p]|asf|wm[v|a]|dat|zip) 4320 80%
43200 reload-into-ims
refresh_pattern -i
windowsupdate.com.*\.(cab|exe|ms[i|u|f|p]|asf|wm[v|a]|dat|zip) 4320
80% 43200 reload-into-ims
refresh_pattern -i
my.windowsupdate.website.com.*\.(cab|exe|ms[i|u|f|p]|asf|wm[v|a]|dat|zip)
4320 80% 43200 reload-into-ims
refresh_pattern -i adobe.com.*\.(zip|exe) 4320 80% 43200
reload-into-ims
refresh_pattern -i java.com.*\.(zip|exe) 4320 80% 43200
reload-into-ims
refresh_pattern -i sun.com.*\.(zip|exe) 4320 80% 43200
reload-into-ims
refresh_pattern -i google\.com.*\.(zip|exe) 4320 80% 43200
reload-into-ims
refresh_pattern -i macromedia\.com.*\.(zip|exe) 4320 80%
43200 reload-into-ims
# Other setups and updates
refresh_pattern -i \.(zip|(g|b)z2?|exe|msi)$ 4320 80% 43200
reload-into-ims
# Cacle squidinternal
refresh_pattern -i video-srv\.youtube\.squidinternal 0 0% 0
refresh_pattern -i squidinternal 14400 100% 518400
override-expire override-lastmod refresh-ims reload-into-ims
ignore-private ignore-auth ignore-must-revalidate store-stale
ignore-no-store
# Keep swf in cache
refresh_pattern -i \.swf$ 10080 100% 43200 override-expire
reload-into-ims ignore-private
# .NET cache
refresh_pattern -i \.((a|m)s(h|p)x?)$ 10080 100% 43200
reload-into-ims ignore-private
# Other long-lived items
refresh_pattern -i
\.(jp(e?g|e|2)|gif|png|bmp|ico|svg|web(p|m)|flv|f4f|mp(3|4)|ttf|eot|woff2?|(c|x|j)ss|js(t?|px?))(\?.*)?$
14400 100% 518400 override-expire override-lastmod
reload-into-ims ignore-private ignore-no-store ignore-must-revalidate
refresh_pattern -i
\.((cs|d?|m?|p?|r?|s?|w?|x?|z?)h?t?m?(l?)|php(3?|5?)|rss|atom|vr(t|ml))(\?.*)?$
10080 100% 86400 override-expire override-lastmod
reload-into-ims ignore-private ignore-no-store ignore-must-revalidate
# Default patterns
refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
refresh_pattern . 0 20% 4320 reload-into-ims
# -------------------------------------
# Rewriter parameters
# -------------------------------------
# ufdbGuard rewriter
url_rewrite_program /usr/local/ufdbguard/bin/ufdbgclient -C
url_rewrite_children 64 startup=0 idle=1 concurrency=2
redirector_bypass off
# Storeurl rewriter
store_id_program /usr/local/squid/libexec/storeid_file_rewrite
/usr/local/squid/etc/storeid.conf
store_id_children 32 startup=0 idle=1 concurrency=4
# Store ID access
store_id_access deny !GET
store_id_access allow store_rewrite_list
store_id_access allow store_rewrite_list_web
store_id_access allow store_rewrite_list_web_cdn
store_id_access allow adobe_java_updates
store_id_access deny all
store_id_bypass off
######
Note: 3.4.14 cache the same URL(s) perfectly, with over 86% HIT.
3.5 _always_ returns MISS.
One simple question:
Why?
-----BEGIN PGP SIGNATURE-----
Version: GnuPG v2
iQEcBAEBCAAGBQJWVyzKAAoJENNXIZxhPexGlGgIAMMa8eAdv+ABBf4TWWk4p61f
H0VGGP7D/j/0QxxsR7j4CcdBc2rVZ5G7nT+KSVDn+MUd7WQ563ZZYsVYsYPb3iBs
Zhmv39Hz0WtpjTtExMXrjKa31cXIP0YaRUwky1M7gnNolvQq2EsYqUU+F9Iw45LV
BQcbrzR0UBSU9jTyG67y626y43Cx5WUzAO+5xW7SFut2gYxR8np6p8Evt+Jfbu9w
pMR0/qTCbHml+nq/dC0sCEeKMQhWZ4tygErdnNn/3RGuwi8rY3CJumbNDhW9JMhV
zt46zhcui4x2/KIGlj/v0rXRqaWVtY5z/v+I1gLhkNm4+pjclEZHIu27Cag6czs=
=4jTH
-----END PGP SIGNATURE-----
More information about the squid-users
mailing list