[squid-users] MS update woes

Sun Jan 24 22:20:45 UTC 2016

Hi

Seems like I getting a bit confused in my conf now .. with
never_direct, always_direct. and miss_access

# ##
# acl
# ##
acl sblMal dstdomain -i "/etc/squid/lists/squid-malicious.acl"
acl sblPorn dstdomain -i "/etc/squid/lists/squid-porn.acl"
acl localnet src 10.32.80.0/24
acl localnet_auth src 10.32.0.0/14
acl localnet_auth src 10.172.0.0/16
acl localnet_auth src 10.43.200.51/32
acl localnet_guest src 10.172.202.0/24
acl localnet_appproxy src 10.172.203.30/32
acl sblYBOveride dstdomain -i "/etc/squid/lists/yb-nonsquidblacklist.acl"
acl nonAuthDom dstdomain -i "/etc/squid/lists/nonAuthDom.lst"
acl nonAuthSrc src "/etc/squid/lists/nonAuthServer.lst"
acl FTP proto FTP
acl DMZSRV src 10.32.20.110
acl DMZSRV src 10.32.20.111
acl DirectExceptions url_regex -i
^http://(www.|)smh.com.au/business/markets-live/.*
acl SSL_ports port 443
acl Safe_ports port 80          # http
acl Safe_ports port 21          # ftp
acl Safe_ports port 443         # https
acl CONNECT method CONNECT
acl SQUIDSPECIAL urlpath_regex ^/squid-internal-static/
acl AuthorizedUsers proxy_auth REQUIRED
acl icp_allowed src 10.32.20.110/32
acl icp_allowed src 10.32.20.111/32
acl icp_allowed src 10.172.203.30/32
acl icp_allowed src 10.172.203.34/32
acl windowsupdate_url url_regex -i
microsoft.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?]
acl windowsupdate_url url_regex -i
windowsupdate.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?]
acl windowsupdate_url url_regex -i
windows.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?]
acl notwindowsupdate_url dstdomain ctldl.windowsupdate.com
acl nonCacheDom dstdomain -i "/etc/squid/lists/nonCacheDom.lst"
acl nonCacheURL urlpath_regex /x86_64/repodata/repomd.xml$
acl Delay_Domain dstdomain -i "/etc/squid/lists/delayDom.lst"

##http_access
## presume this is processed first

# manager access
http_access allow manager localhost
http_access allow manager icp_allowed
http_access deny manager

# icp access
http_access allow icp_allowed

# the squid special url
http_access allow SQUIDSPECIAL
# block non safe ports
http_access deny !Safe_ports
# block ssl non non ssl  ports
http_access deny CONNECT !SSL_ports

#http_access deny to_localhost

# Who can access
# network with no auth
http_access allow localnet
# local machine
http_access allow localhost
# other downstreams
http_access allow localnet_appproxy

# this is my just in case MS update goes wild again turn this on ACL
#http_access deny !DMZSRV windowsupdate_url

# the catch all for ip address range
http_access deny !localnet_auth

# special guest network rules (basically non auth)
http_access allow localnet_guest sblYBOveride
http_access deny localnet_guest sblMal
http_access deny localnet_guest sblPorn
http_access allow localnet_guest

# non guest sources that can access via non auth
http_access allow nonAuthSrc
# non auth dest domains
http_access allow nonAuthDom

# over ride some black list sites
http_access allow sblYBOveride FTP
http_access allow sblYBOveride AuthorizedUsers

# squid blacklists
http_access deny sblMal
http_access deny sblPorn

# allow FTP
http_access allow FTP
# allow Authorised
http_access allow AuthorizedUsers
# deny every one else
http_access deny all

# Alway direct
# if its FTP then go direct
always_direct allow FTP
# stop the looping. so peer cache requests are always direct
always_direct allow DMZSRV
# Some url's still have issues with looping and caching back responses
# this makes them allways do direct and never loop
always_direct allow DirectExceptions

# never Direct
# there are some MS urls that should be direct (they are usually not cached)
never_direct deny notwindowsupdate_url
# block all MS update's except from certain sources from going direct
# does this allow a cache peer to start a windows update ???
never_direct allow !DMZSRV windowsupdate_url

# ### This is my newly added
# miss_access
# http://www.squid-cache.org/Doc/config/miss_access/
# Some MS urls are need and can't be cached !
miss_access allow notwindowsupdate_url
# Deny Access to MS Update only from DMZ boxes
miss_access deny !DMZSRV windowsupdate_url

# http://wiki.squid-cache.org/SquidFaq/WindowsUpdate
# 800M for MS SQL patch file
# made bigger to handle bigger Patch files !
range_offset_limit 800 MB
maximum_object_size 800 MB
quick_abort_min -1

# special refresh pattarns that force files to be cached. I have
changed it up to 90days of caching
# also added in the [^?] to stop it trying to cache those
refresh_pattern -i
microsoft.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?] 4320
80% 129600 reload-into-ims
refresh_pattern -i
windowsupdate.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?]
4320 80% 129600 reload-into-ims
refresh_pattern -i
windows.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?] 4320
80% 129600 reload-into-ims

# Add any of your own refresh_pattern entries above these.
refresh_pattern ^ftp:           1440    20%     10080
refresh_pattern ^gopher:        1440    0%      1440
refresh_pattern -i (/cgi-bin/|\?) 0     0%      0
refresh_pattern .               0       20%     4320

# NON Cache Domain
acl nonCacheDom dstdomain -i "/etc/squid/lists/nonCacheDom.lst"
cache deny nonCacheDom

# NON Cache URL
acl nonCacheURL urlpath_regex /x86_64/repodata/repomd.xml$
cache deny nonCacheURL

So what I have hoped to have done here is
1) stop all except DMZSRV hosts from access the Microsoft Update urls,
unless its cached ...
2) allowed DMZSRV hosts to request those files and place them in the cache.

I had thought I had done that before, but i noticed this morning a
spike as machine where turned on and they started to make request

These are lines before I added the miss_access config. I had though
the never direct would have stopped these !
I had to turn on the explicit
#http_access deny !DMZSRV windowsupdate_url

# ##
1453672641.992     28 10.172.202.102 TCP_MISS/206 1819330 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672652.908   9943 10.172.202.102 TCP_MISS/206 3639200 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672661.916   8973 10.172.202.102 TCP_MISS/206 1686624 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672662.026     20 10.172.202.102 TCP_MISS/206 1160541 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672664.922   1918 10.172.202.102 TCP_MISS/206 3119331 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672697.955  32927 10.172.202.102 TCP_MISS/206 1697038 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672698.245     16 10.172.202.102 TCP_MISS/206 1140456 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672699.359    130 10.172.202.102 TCP_MISS/206 3424893 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
1453672700.269     38 10.172.202.102 TCP_MISS/206 2338346 GET
http://wsus.ds.download.windowsupdate.com/c/msdownload/update/software/secu/2015/12/ie11-windows6.1-kb3124275-x86_da23592568a57c26665a23d23d888428d831d739.psf
- HIER_NONE/- application/octet-stream
# ##

any comments welcome

Thanks

On 20 January 2016 at 14:27, Amos Jeffries <squid3 at treenet.co.nz> wrote:
> On 20/01/2016 1:56 p.m., Alex Samad wrote:
>> Oh
>>
>> I am missing something. your saying the actualy get include more past
>> the ? and that squid logging isn't recording it !
>
> Yes. There is part of the URL that is not logged by default. Sometimes
> that part is very big by many KB, and/or wrongly containing sensitive info.
> Set <http://www.squid-cache.org/Doc/config/strip_query_terms/> to
> show/hide that part.
>
>>
>> So what I really need to do is modify the original to exclude any urls
>> that have ?
>>
>> something like ?
>> "windowsupdate.com/.*\.(cab|exe|ms[i|u|f]|[ap]sf|wm[v|a]|dat|zip)[^?]"
>>
>
> What I suspect is that some part of the hidden query-string is different
> between the MISS and possibly between your prefetch request.
>
> You may be able to use the Store-ID feature to compact duplicates if the
> changing part is unimportant. But that would have to be done very
> carefully as there are some nasty side effects worse than bandwidth
> usage if it goes wrong.
>  So leave off trying for a fix until you/we are clear on what exactly
> the reason for the MISS is.
>
> Amos