[squid-users] Squid 3.5.11 always can't cache Wikipedia content
Eliezer Croitoru
eliezer at ngtech.co.il
Thu Nov 26 18:02:26 UTC 2015
Hey Yuri,
Maybe there is a bug.
I will try to see if I am having the same issue.
Eliezer
On 26/11/2015 18:01, Yuri Voinov wrote:
>
> -----BEGIN PGP SIGNED MESSAGE-----
> Hash: SHA256
>
> One simple Wikipedia URL.
>
> This is second-third-etc. query access log:
>
> http://i.imgur.com/AOpeSQx.png
>
> This is redbot URL info:
>
> http://i.imgur.com/dmwjH26.png
>
> This is squid.conf (3.5.11 latest build):
>
> # -------------------------------------
> # Access Control Lists
> # -------------------------------------
> acl localnet src 192.168.0.0/16 # RFC1918 possible internal network
>
> acl SSL_ports port 443
> acl SSL_ports port 8443 # Telecom exclusion
> acl SSL_ports port 2041 # ICQ/MRA
> acl SSL_ports port 2042 # ICQ/MRA
> acl SSL_ports port 5160 # ICQ/MRA
> acl SSL_ports port 5228 # ICQ/MRA
> acl Safe_ports port 80 # http
> acl Safe_ports port 21 # ftp
> acl Safe_ports port 443 # https
> acl Safe_ports port 70 # gopher
> acl Safe_ports port 210 # wais
> acl Safe_ports port 1025-65535 # unregistered ports
> acl Safe_ports port 280 # http-mgmt
> acl Safe_ports port 488 # gss-http
> acl Safe_ports port 591 # filemaker
> acl Safe_ports port 777 # multiling http
>
> # Common methods
> acl CONNECT method CONNECT
> acl PURGE method PURGE
> acl GET method GET
>
> # Windows update acls
> acl windowsupdate dstdomain sls.update.microsoft.com.akadns.net
> acl windowsupdate dstdomain windowsupdate.microsoft.com
> acl windowsupdate dstdomain .update.microsoft.com
> acl windowsupdate dstdomain download.windowsupdate.com
> acl windowsupdate dstdomain redir.metaservices.microsoft.com
> acl windowsupdate dstdomain images.metaservices.microsoft.com
> acl windowsupdate dstdomain c.microsoft.com
> acl windowsupdate dstdomain www.download.windowsupdate.com
> acl windowsupdate dstdomain wustat.windows.com
> acl windowsupdate dstdomain crl.microsoft.com
> acl windowsupdate dstdomain sls.microsoft.com
> acl windowsupdate dstdomain productactivation.one.microsoft.com
> acl windowsupdate dstdomain ntservicepack.microsoft.com
>
> # Windows update methods
> acl wuCONNECT dstdomain www.update.microsoft.com
> acl wuCONNECT dstdomain sls.microsoft.com
>
> # Youtube & CDN store rewrite ACLs
> acl store_rewrite_list urlpath_regex
> \.(jp(e?g|e|2)|gif|png|bmp|ico|svg|web(p|m)|flv|f4f|mp(3|4)|ttf|eot|woff2?|(c|x|j)ss|js(t?|px?))\?
> \/ads\?
> acl store_rewrite_list_web url_regex "/usr/local/squid/etc/url.rewrite_web"
> acl store_rewrite_list_web_cdn url_regex
> "/usr/local/squid/etc/url.rewrite_cdn"
>
> # Adobe/Java and other updates
> acl adobe_java_updates url_regex "/usr/local/squid/etc/url.updates"
>
> # No-cache
> acl dont_cache_url url_regex "/usr/local/squid/etc/url.nocache"
>
> # Tor acl
> acl tor_url dstdom_regex -i "/usr/local/squid/etc/url.tor"
>
> # SSL bump acl
> acl net_bump src "/usr/local/squid/etc/net.bump"
>
> # TLD acl
> acl block_tld dstdomain "/usr/local/squid/etc/dstdom.tld"
>
> # -------------------------------------
> # Access parameters
> # -------------------------------------
> # Deny requests to unsafe ports
> http_access deny !Safe_ports
> # Deny CONNECT to other than SSL ports
> http_access deny CONNECT !SSL_ports
>
> # Only allow cachemgr access from localhost
> http_access allow localhost manager
> http_access deny manager
> http_access deny to_localhost
> # Allow purge from localhost
> http_access allow PURGE localhost
> http_access deny PURGE
>
> # Block top level domains
> http_access deny block_tld
> deny_info TCP_RESET block_tld
>
> # Rule allowing access from local networks
> http_access allow localnet
> http_access allow localhost
>
> # No cache directives
> cache deny dont_cache_url
>
> # ICP/HTCP access
> #icp_access allow localnet
> #icp_access deny all
> #htcp_access allow localnet
> #htcp_access deny all
>
> # For YT block useragent header
> acl user-agent-browser browser -i
> (android|blackberry|iphone|ipad|ipod|opera mini|iemobile)
> acl youtube_dom dstdomain .youtube.com
> request_header_access User-Agent deny youtube_dom !user-agent-browser
> request_header_replace User-Agent Mozilla/5.0 (compatible;
> Googlebot/2.1; +http://www.google.com/bot.html)
> # Normalize Accept-Encoding to support compression via eCAP
> request_header_access Accept-Encoding deny all
> request_header_replace Accept-Encoding gzip;q=1.0, identity;q=0.5, *;q=0
> # Disable alternate protocols
> request_header_access Alternate-Protocol deny all
> reply_header_access Alternate-Protocol deny all
> # Disable HSTS
> reply_header_access Strict-Transport-Security deny all
> reply_header_replace Strict-Transport-Security max-age=0; includeSubDomains
> # Remove User-Agent from Vary
> reply_header_access Vary deny all
> reply_header_replace Vary Accept-Encoding
>
> # 302 loop
> acl text_mime rep_mime_type text/html text/plain
> acl http302 http_status 302
> store_miss deny text_mime http302
> send_hit deny text_mime http302
>
> # Windows updates rules
> http_access allow CONNECT wuCONNECT localnet
> http_access allow CONNECT wuCONNECT localhost
> http_access allow windowsupdate localnet
> http_access allow windowsupdate localhost
>
> # SSL bump rules
> sslproxy_cert_error allow all
> acl DiscoverSNIHost at_step SslBump1
> ssl_bump peek DiscoverSNIHost
> acl NoSSLIntercept ssl::server_name_regex -i
> "/usr/local/squid/etc/url.nobump"
> acl NoSSLIntercept ssl::server_name_regex -i "/usr/local/squid/etc/url.tor"
> ssl_bump splice NoSSLIntercept
> ssl_bump bump net_bump
>
> # Privoxy+Tor access rules
> never_direct allow tor_url
>
> # And finally deny all other access to this proxy
> http_access deny all
>
> # -------------------------------------
> # HTTP parameters
> # -------------------------------------
> # Local Privoxy is cache parent
> cache_peer 127.0.0.1 parent 8118 0 no-query no-digest default
>
> cache_peer_access 127.0.0.1 allow tor_url
> cache_peer_access 127.0.0.1 deny all
>
> # Don't cache 404 long time
> negative_ttl 5 minutes
> positive_dns_ttl 15 hours
> negative_dns_ttl 1 minutes
>
> # -------------------------------------
> # Cache parameters
> # -------------------------------------
> http_port 3126 ssl-bump generate-host-certificates=on
> dynamic_cert_mem_cache_size=4MB cert=/usr/local/squid/etc/rootCA.crt
> key=/usr/local/squid/etc/rootCA.key options=NO_SSLv3
> dhparams=/usr/local/squid/etc/dhparam.pem
> http_port 3127
> http_port 3128 intercept
> https_port 3129 intercept ssl-bump generate-host-certificates=on
> dynamic_cert_mem_cache_size=4MB cert=/usr/local/squid/etc/rootCA.crt
> key=/usr/local/squid/etc/rootCA.key options=NO_SSLv3
> dhparams=/usr/local/squid/etc/dhparam.pem
> sslproxy_capath /etc/opt/csw/ssl/certs
> #sslproxy_cafile /usr/local/squid/etc/ca-bundle.crt
> sslproxy_options NO_SSLv3,SINGLE_DH_USE
> sslproxy_cipher
> EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH+aRSA+RC4:EECDH:EDH+aRSA:HIGH:!RC4:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS
> sslcrtd_program /usr/local/squid/libexec/ssl_crtd -s /var/lib/ssl_db -M 4MB
>
> cachemgr_passwd disable shutdown reconfigure rotate
>
> # Cache user
> cache_effective_user squid
> cache_effective_group squid
>
> # Turn off collect per-client statistics
> client_db off
>
> # Forces reload-into-ims
> reload_into_ims on
>
> # Hide internal networks details outside
> via off
> forwarded_for delete
>
> # Do not show Squid version
> httpd_suppress_version_string on
>
> # WCCPv2 parameters
> wccp2_router 192.168.200.2
> wccp2_forwarding_method l2
> wccp2_return_method l2
> wccp2_rebuild_wait off
> wccp2_service standard 0
> wccp2_service dynamic 70
> wccp2_service_info 70 protocol=tcp
> flags=dst_ip_hash,src_ip_alt_hash,src_port_alt_hash priority=240 ports=443
>
> # Prioritization of local hits
> qos_flows tos local-hit=0x68
>
> # Default is on
> digest_generation off
>
> # -------------------------------------
> # Adaptation parameters
> # -------------------------------------
> icap_enable on
> icap_service_failure_limit -1
> icap_send_client_ip on
> icap_send_client_username on
> icap_client_username_header X-Authenticated-User
> icap_preview_enable on
> icap_preview_size 1024
> icap_service service_avi_req reqmod_precache
> icap://localhost:1344/squidclamav bypass=off
> adaptation_access service_avi_req allow all
> icap_service service_avi_resp respmod_precache
> icap://localhost:1344/squidclamav bypass=on
> adaptation_access service_avi_resp allow all
>
> ecap_enable on
> acl HTTP_STATUS_OK http_status 200
> loadable_modules /usr/local/lib/ecap_adapter_gzip.so
> ecap_service gzip_service respmod_precache
> ecap://www.vigos.com/ecap_gzip bypass=off
> adaptation_access gzip_service allow HTTP_STATUS_OK
>
> # -------------------------------------
> # Store parameters
> # -------------------------------------
> maximum_object_size 8 Gb
>
> cache_dir aufs /data/cache/d1 32767 16 256
> cache_dir aufs /data/cache/d2 32767 16 256
> cache_dir aufs /data/cache/d3 32767 16 256
> cache_dir aufs /data/cache/d4 32767 16 256
>
> # -------------------------------------
> # Memory parameters
> # -------------------------------------
> cache_mem 512 Mb
>
> #memory_pools off
>
> # -------------------------------------
> # Tuning parameters
> # -------------------------------------
> memory_replacement_policy heap GDSF
> cache_replacement_policy heap LFUDA
>
> # Shutdown delay before terminate connections
> shutdown_lifetime 15 second
>
> # -------------------------------------
> # Process/log parameters
> # -------------------------------------
> access_log daemon:/data/cache/log/access.log buffer-size=256KB !tor_url
> # Don't log ICP queries
> #log_icp_queries off
>
> # Turn off internal log rotation
> logfile_rotate 0
>
> cache_log /data/cache/log/cache.log
> cache_store_log none
>
> # Default is off
> buffered_logs on
>
> coredump_dir /var/core
>
> pid_filename /tmp/squid.pid
>
> strip_query_terms off
>
> # -------------------------------------
> # Content parameters
> # -------------------------------------
> range_offset_limit none all
> quick_abort_min -1 KB
>
> # Updates: Windows, Adobe, Java
> refresh_pattern -i
> microsoft.com.*\.(cab|exe|ms[i|u|f|p]|asf|wm[v|a]|dat|zip) 4320 80%
> 43200 reload-into-ims
> refresh_pattern -i
> windowsupdate.com.*\.(cab|exe|ms[i|u|f|p]|asf|wm[v|a]|dat|zip) 4320
> 80% 43200 reload-into-ims
> refresh_pattern -i
> my.windowsupdate.website.com.*\.(cab|exe|ms[i|u|f|p]|asf|wm[v|a]|dat|zip)
> 4320 80% 43200 reload-into-ims
> refresh_pattern -i adobe.com.*\.(zip|exe) 4320 80% 43200
> reload-into-ims
> refresh_pattern -i java.com.*\.(zip|exe) 4320 80% 43200
> reload-into-ims
> refresh_pattern -i sun.com.*\.(zip|exe) 4320 80% 43200
> reload-into-ims
> refresh_pattern -i google\.com.*\.(zip|exe) 4320 80% 43200
> reload-into-ims
> refresh_pattern -i macromedia\.com.*\.(zip|exe) 4320 80%
> 43200 reload-into-ims
> # Other setups and updates
> refresh_pattern -i \.(zip|(g|b)z2?|exe|msi)$ 4320 80% 43200
> reload-into-ims
> # Cacle squidinternal
> refresh_pattern -i video-srv\.youtube\.squidinternal 0 0% 0
> refresh_pattern -i squidinternal 14400 100% 518400
> override-expire override-lastmod refresh-ims reload-into-ims
> ignore-private ignore-auth ignore-must-revalidate store-stale
> ignore-no-store
> # Keep swf in cache
> refresh_pattern -i \.swf$ 10080 100% 43200 override-expire
> reload-into-ims ignore-private
> # .NET cache
> refresh_pattern -i \.((a|m)s(h|p)x?)$ 10080 100% 43200
> reload-into-ims ignore-private
> # Other long-lived items
> refresh_pattern -i
> \.(jp(e?g|e|2)|gif|png|bmp|ico|svg|web(p|m)|flv|f4f|mp(3|4)|ttf|eot|woff2?|(c|x|j)ss|js(t?|px?))(\?.*)?$
> 14400 100% 518400 override-expire override-lastmod
> reload-into-ims ignore-private ignore-no-store ignore-must-revalidate
> refresh_pattern -i
> \.((cs|d?|m?|p?|r?|s?|w?|x?|z?)h?t?m?(l?)|php(3?|5?)|rss|atom|vr(t|ml))(\?.*)?$
> 10080 100% 86400 override-expire override-lastmod
> reload-into-ims ignore-private ignore-no-store ignore-must-revalidate
> # Default patterns
> refresh_pattern -i (/cgi-bin/|\?) 0 0% 0
> refresh_pattern . 0 20% 4320 reload-into-ims
>
> # -------------------------------------
> # Rewriter parameters
> # -------------------------------------
> # ufdbGuard rewriter
> url_rewrite_program /usr/local/ufdbguard/bin/ufdbgclient -C
> url_rewrite_children 64 startup=0 idle=1 concurrency=2
> redirector_bypass off
>
> # Storeurl rewriter
> store_id_program /usr/local/squid/libexec/storeid_file_rewrite
> /usr/local/squid/etc/storeid.conf
> store_id_children 32 startup=0 idle=1 concurrency=4
> # Store ID access
> store_id_access deny !GET
> store_id_access allow store_rewrite_list
> store_id_access allow store_rewrite_list_web
> store_id_access allow store_rewrite_list_web_cdn
> store_id_access allow adobe_java_updates
> store_id_access deny all
> store_id_bypass off
> ######
>
> Note: 3.4.14 cache the same URL(s) perfectly, with over 86% HIT.
> 3.5 _always_ returns MISS.
>
> One simple question:
>
> Why?
> -----BEGIN PGP SIGNATURE-----
> Version: GnuPG v2
>
> iQEcBAEBCAAGBQJWVyzKAAoJENNXIZxhPexGlGgIAMMa8eAdv+ABBf4TWWk4p61f
> H0VGGP7D/j/0QxxsR7j4CcdBc2rVZ5G7nT+KSVDn+MUd7WQ563ZZYsVYsYPb3iBs
> Zhmv39Hz0WtpjTtExMXrjKa31cXIP0YaRUwky1M7gnNolvQq2EsYqUU+F9Iw45LV
> BQcbrzR0UBSU9jTyG67y626y43Cx5WUzAO+5xW7SFut2gYxR8np6p8Evt+Jfbu9w
> pMR0/qTCbHml+nq/dC0sCEeKMQhWZ4tygErdnNn/3RGuwi8rY3CJumbNDhW9JMhV
> zt46zhcui4x2/KIGlj/v0rXRqaWVtY5z/v+I1gLhkNm4+pjclEZHIu27Cag6czs=
> =4jTH
> -----END PGP SIGNATURE-----
>
>
> _______________________________________________
> squid-users mailing list
> squid-users at lists.squid-cache.org
> http://lists.squid-cache.org/listinfo/squid-users
>
More information about the squid-users
mailing list