create local ubuntu update repository with apt-mirror package

1. install apt-mirror package
# apt-get update
# apt-get install apt-mirror

2. modify apt-mirror configuration file (example for 12.04)
# vim /etc/apt/mirror.list

############# config ##################
#
set base_path    /var/spool/apt-mirror
#
set mirror_path  $base_path/mirror
set skel_path    $base_path/skel
set var_path     $base_path/var
# set cleanscript $var_path/clean.sh
# set defaultarch  
# set postmirror_script $var_path/postmirror.sh
# set run_postmirror 0
set nthreads     20
set _tilde 0
#
############# end config ##############

deb http://archive.ubuntu.com/ubuntu precise main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu precise-security main restricted universe multiverse
deb http://archive.ubuntu.com/ubuntu precise-updates main restricted universe multiverse

deb-amd64 http://archive.ubuntu.com/ubuntu precise main restricted universe multiverse
deb-amd64 http://archive.ubuntu.com/ubuntu precise-security main restricted universe multiverse
deb-amd64 http://archive.ubuntu.com/ubuntu precise-updates main restricted universe multiverse

deb-i386 http://archive.ubuntu.com/ubuntu precise main restricted universe multiverse
deb-i386 http://archive.ubuntu.com/ubuntu precise-security main restricted universe multiverse
deb-i386 http://archive.ubuntu.com/ubuntu precise-updates main restricted universe multiverse
deb-amd64 http://archive.ubuntu.com/ubuntu precise-proposed main restricted universe multiverse
deb-amd64 http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse
deb-i386 http://archive.ubuntu.com/ubuntu precise-proposed main restricted universe multiverse
deb-i386 http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse

deb-src http://archive.ubuntu.com/ubuntu precise main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu precise-security main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu precise-updates main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu precise-proposed main restricted universe multiverse
deb-src http://archive.ubuntu.com/ubuntu precise-backports main restricted universe multiverse

clean http://archive.ubuntu.com/

3. initial apt-mirror

# /etc/init.d/apt-mirror

4. mkdir softlink in apache root directory
# ln -s /var/spool/apt-mirror/mirror/archive.ubuntu.com/ubuntu/ /var/www/ubuntu

5. modify client or server source.list direct to this server
# vim /etc/apt/source.list
change all hostname to this url

deb http://us.archive.ubuntu.com/ubuntu/ precise main restricted
deb-src http://us.archive.ubuntu.com/ubuntu/ precise main restricted

deb http://update.csp.com/ubuntu/ precise main restricted
deb-src http://update.csp.com/ubuntu/ precise main restricted

squid server use ntlm authenction with multi group

1. modify squid.conf as below
# vim /etc/squid3/squid.conf

http_port  3128
icp_port 0
htcp_port 0
cache_mgr admin@cps.com
visible_hostname squid.csp.com
cache_dir diskd /var/spool/squid3 81920 16 256
cache_mem 1024 MB
cache_swap_low  80
cache_swap_high 95
maximum_object_size  1024 KB
maximum_object_size_in_memory 800 KB
ipcache_size 65536
ipcache_low 80
ipcache_high 95

acl QUERY urlpath_regex cgi-bin \?
no_cache deny QUERY

digest_generation off
pipeline_prefetch on
shutdown_lifetime 1 second

cache_access_log /var/log/squid3/access.log
#cache_access_log syslog:local5.info squid
cache_log /var/log/squid3/cache.log
cache_store_log none
pid_filename /var/run/squid3.pid
cache_swap_log /var/log/squid3/cache_swap.log
read_timeout 10 minutes
request_timeout 8 minutes
pconn_timeout 120 seconds
ftp_user anonymous
ftp_list_width 64
ftp_passive on
ftp_sanitycheck on
hosts_file /etc/hosts
negative_ttl 2 minutes

cache_peer 10.10.2.2 parent 9119 0 no-query name=fproxy
cache_peer_domain fproxy .yahoo.co.jp .gov.tw

# authentication
authenticate_ttl 8 hours
auth_param ntlm program /usr/bin/ntlm_auth --helper-protocol=squid-2.5-ntlmssp

auth_param ntlm children 50

auth_param basic program /usr/bin/ntlm_auth --helper-protocol=squid-2.5-basic

auth_param basic children 10

auth_param basic realm squidt1-6 Proxy

auth_param basic credentialsttl 8 hours

external_acl_type wbinfo_check %LOGIN /usr/lib/squid3/wbinfo_group.pl

acl tw_www_disabled external wbinfo_check tw_www_disabled

acl tw_www_enabled external wbinfo_check tw_www_enabled

acl tw_www_cloud external wbinfo_check tw_www_cloud

#ACLS

#general

acl java browser -i ^JAVA/

acl svn browser -i ^SVN/

acl CONNECT method CONNECT

acl password proxy_auth REQUIRED

acl manager proto cache_object

acl Safe_ports port 20 21 80 81 82 443 888 1025-65535

acl SSL_ports port 22 94 443 2083-2093 8443 10443

acl ftp proto FTP

acl http proto http

acl localhost src 127.0.0.1/32

acl gissrc src 10.77.0.0/16

acl cspdst dst 10.0.0.0/8  192.168.0.0/16

acl cspsrc src 10.0.0.0/8

acl site_block dstdomain "/etc/squid3/site_block"

acl site_allow dstdomain "/etc/squid3/site_allow"

acl ip_block dst "/etc/squid3/ip_block"

acl ip_china_proxy src "/etc/squid3/ip_china_proxy"

acl ip_gmail_allow src "/etc/squid3/ip_gmail_allow"

acl ip_cloud_allow src "/etc/squid3/ip_cloud_allow"

acl site_webmail_allow dstdomain "/etc/squid3/site_webmail_allow"

acl site_cloud_allow dstdomain "/etc/squid3/site_cloud_allow"

#allow_specific

http_access allow tw_www_disabled site_allow

http_access deny tw_www_disabled !site_allow

http_access allow ip_gmail_allow site_webmail_allow

http_access allow tw_www_cloud site_cloud_allow

#deny rule

http_access deny site_block

http_access deny ip_block

http_access deny !Safe_ports

http_access deny CONNECT !SSL_ports

http_access deny !cspsrc

snmp_access deny !cspsrc

icp_access deny !cspsrc

#allow rule

http_access allow java

http_access allow svn

http_access allow ip_china_proxy

http_access allow gissrc

http_access allow manager localhost

http_access deny manager

http_access allow ftp

http_access allow Safe_ports tw_www_enabled

install squidgurard to implement acl

1. install squidguard
# apt-get update && apt-get install squidguard

2. modify squidguard configuration
# vim /etc/squid/squidguard.conf

dbhome /var/lib/squidguard/blacklists
logdir /var/log/squid

#
# TIME RULES:
# abbrev for weekdays:
# s = sun, m = mon, t =tue, w = wed, h = thu, f = fri, a = sat

time workhours {
        weekly mtwhf 00:00 - 24:00
        date *-*-01  00:00 - 24:00
}

src admin {

        ip              10.124.20.159

        user            root tommy

        within          workhours

}

src client-src {

        ip              10.0.0.0/8

}

#

# DESTINATION CLASSES:

#

dest good {

        domainlist      good/domains

        urllist         good/urls

        log             block.log

}

dest ads {

        domainlist      ads/domains

        urllist         ads/urls

        log             block.log

}

dest adult {

        domainlist      adult/domains

        urllist         adult/urls

        log             block.log

}

dest chat {

        domainlist      chat/domains

        urllist         chat/urls

        log             block.log

}

acl {

        admin {

                pass     any

        }

        client-src within workhours {

                pass     good !ads !adult !chat all

        } else {

                pass any

        }

        default {

                pass     none

                redirect http://xxx.xxxx.xxxx/cgi-bin/blocked.cgi?clientaddr=%a&clientname=%n&clientuser=%i&clientgroup=%s&targetgroup=%t&url=%u

        }

}

3. download the blacklist from internet 

# cd /var/lib/squidguard/

# tar zxvf bigblacklist.tar.gz

4. create good folder in the db location

# cd /var/lib/squidguard/blacklists

# mkdir good

# touch good/domains

# touch good/urls

5. initial the database and change the permission for squid service account

# squidGuard -C all

# chown -Rf proxy:proxy  /var/lib/squidguard/blacklists

6. touch the block.log 

# touch /var/log/squid/block.log

# chown -Rf proxy:proxy /var/lig/squid

7. add the line into the squid.conf

# vim /etc/squid3/squid.conf

url_rewrite_program /usr/bin/squidGuard -c /etc/squid/squidGuard.conf

8. restart the squid service 

# /etc/init.d/squid restart

How to direct the squid log to splunk

1. modify squid config and change  cache_access_log  to rsyslog
# vim /etc/squid3/squid.conf

#cache_access_log /var/log/squid3/access.log
cache_access_log syslog:local5.info squid

2. modify rsyslog configurtion to direct squid.log to splunk
# vim /etc/rsyslog.d/50-default.conf

add this line to the file
local5.*                                  @splunk.xxx.xxx.xxx:514

3.restart squid  and rsyslog
# /etc/init.d/squid3 restart
# /etc/init.d/rsyslog restart

4. confirm squid server status
# /etc/init.d/squid3 status

5. confirm the log data in splunk server