2010年11月30日 星期二

how to install hadoop on ubuntu 10.04 LTS server installion 1

1.install ubuntu l0.04 LTS version

2.update,upgrade and install ssh
#apt-get update
#apt-get upgrade
#apt-get install ssh

3.install sun-6-java jdk
#apt-get install openjdk-6-java

4.create new user hadoop and group
#addgroup hadoop
#adduser --ingroup hadoop hadoop

5.generate ssh key to auto login to manager all nodes
#ssh-keygen -t rsa -P ""

6.disable ipv6 and reboot machine
#vim /etc/modprobe.d/blacklist
add new line in it
blacklist ipv6

7.download hadoop from mirror site
#wget http://XXX.XXX.XXX.XXX/hadoop-0.21.0.tar.gz

8.untar hadoop-0.21.0.tar.gz and rename it to hadoop
#tar zxvf hadoop-0.21.0.tar.gz
#mv hadoop-0.21.0

9.change owner and group to hadoop on hadoop folder
#chown -R hadoop:hadoop hadoop/

10.confirm java home folder
#ls -l `whereis javac`

11.modify hadoop home folder hadoop-env.sh
#vim hadoop/conf/hadoop-env.sh
uncomment export JAVA_HOME and modify it
export JAVA_HOME = /usr/lib/jvm/java-6-openjdk/

12.config hadoop config file
#vim hadoop/conf/core-site.xml
add these line into it
<property>
    <name>hadoop.tmp.dir</name>
    <value>/home/hadoop/hadoop/tmp/dir/hadoop-hadoop</value>
</property>
<property>
    <name>fs.default.name</name>
    <value>hdfs://10.124.2.225</value>
</property>

13.config hadoop file with hdfs-site.xml
#vim hadoop/conf/hdfs-site.xml
add these line into it
<property>
    <name>dfs.replication</name>
    <value>1</value>
<property>

14.config hadoop file with mapred-site.xml
#vim hadoop/conf/mapred-site.xml
add these line into it
<property>
    <name>mapred.job.tracker</name>
    <value>10.124.2.225:54311</value>
</property>

15.formating the namenode
#hadoop/bin/hadoop namenode -format

16.start cluster
#hadoop/bin/start-all.sh

17.check hadoop process
#hadoop/bin/jps

18.use netstat to check all service running status
#netstat -plten | grep java

19.stop cluster
#hadoop/bin/stop-all.sh

20.start cluster
#hadoop/bin/start-all.sh

21.mkdir a folder for gutenberg and touch three files with contents
#mkdir /tmp/gutenberg
#cd /tmp/gutenberg
#vim 1.txt
#vim 2.txt
#vim 3.txt

21.use hadoop fs copyFromLocal copy files to hdfs folder
#hadoop/bin/hadoop fs -copyFromLocal /tmp/gutenberg gutenber

22.check hdfs folder content
#hadoop/bin/hadoop fs -ls
#hadoop/bin/hadoop fs -ls gutenberg

23.use java wordcount to calculate the words number
#hadoop/bin/hadoop jar hadoop-mapred-examples-0.21.0.jar wordcount gutenberg gutenberg-output

24.list gutenberg-output folder content
#bin/hadoop fs -ls gutenberg-output

25.mkdir one subfolder on folder tmp
#mkdir /tmp/gutenberg-output

26.copy data from hdfs to local folder
# bin/hadoop fs -copyToLocal gutenberg-output/part-r-00000 /tmp/gutenberg-output/

27.show content for part-r-000000
#more /tmp/gutenberg-content/part-r-00000

2010年11月18日 星期四

how to confirm java_home

1.install java 1.6 jdk
#apt-get install openjdk-6-jdk

2.use whereis to find javac
#whereis javac

3.use ls -al to confirm if javac is a symbolic
#ls -al `whereis javac`

4.use export java_home
#export JAVA_HOME=/usr/lib/jvm/java-6-openjdk/jre

2010年9月8日 星期三

squid server with ad authenticate

reqirement
1.install squid and squidguard
#sudo apt-get install squid squidguard
2.install winbind and samba server to auth with active directory
3.setting keberos and pam module and join domain
4.setting squid server and squidGuard
5.authicated groups memory and grant access privileges to web site

6.open ip_forward
#vim /etc/sysctl.conf
uncommnet #net.ipv4.ip_forward=1 to net.ipv4.ip_forward=1
#echo "1" > /proc/sys/net/ipv4/ip_forward

7.modify /etc/squid/squid.conf
http_port 60000
icp_port 0
htcp_port 0
cache_mgr mingda.chiang@garmin.com
visible_hostname t3proxy.garmin.com
cache_dir diskd /var/spool/squid 81920 16 256
cache_mem 2048 MB
cache_swap_low 80
cache_swap_high 95
maximum_object_size 4096 KB
maximum_object_size_in_memory 800 KB
ipcache_size 65536
ipcache_low 80
ipcache_high 95
acl QUERY urlpath_regex cgi-bin \?
no_cache deny QUERY
digest_generation off
pipeline_prefetch on
shutdown_lifetime 1 second

cache_access_log /var/log/squid/access.log
cache_log /var/log/squid/cache.log
cache_store_log none
pid_filename /var/run/squid.pid
cache_swap_log /var/log/squid/cache_swap.log
read_timeout 10 minutes
request_timeout 8 minutes
pconn_timeout 60 seconds
ftp_user anonymous
ftp_list_width 64
ftp_passive on
ftp_sanitycheck on
hosts_file /etc/hosts
negative_ttl 2 minutes
authenticate_ttl 2 hours
auth_param ntlm program /usr/bin/ntlm_auth --require-membership-of="csp+tw_enabled" --helper-protocol=squid-2.5-ntlmss
p
auth_param ntlm children 10
auth_param ntlm max_challenge_reuses 0
auth_param ntlm max_challenge_lifetime 2 minutes
auth_param basic program /usr/bin/ntlm_auth --require-membership-of="csp+tw_enabled" --helper-protocol=squid-2.5-basic
auth_param basic children 3
auth_param basic realm Linkou Proxy
auth_param basic credentialsttl 2 hours

#ACLs
acl password proxy_auth REQUIRED
acl allowusers proxy_auth -i tommylin nicloelin
acl shoppingsites dstdomain "//etc/squid3/shoppingsites"
acl localhost src 127.0.0.1/255.255.255.255
acl all src 0.0.0.0/0.0.0.0
acl lansrc src 192.168.1.0/255.255.255.0
acl manager proto cache_object
acl Safe_ports port 21 80 82 443 888 1025-65535
acl SSL_ports port 443 2083-2093 8443
acl CONNECT method CONNECT
acl ftp proto FTP

http_access allow shoopingsites allowusers
http_access allow manager garminsrc
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow Safe_ports password
http_access allow ftp password
http_access allow lansrc
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
http_access allow Safe_ports password
http_access allow ftp password
http_access allow manager localhost
http_access deny all
snmp_access deny all
icp_access deny all

cache_effective_user proxy
cache_effective_group proxy

header_access Via deny all
header_access X-Forwarded-For deny all
header_access Cache-Control deny all

8.grant permission for squid db
#chgrp proxy /var/run/samba/winbindd_privileged/
#chmod g+rx /var/run/samba/winbindd_privileged/

9.create folder for squid swap
#cd /var/cache
#mkdir squid
#chown proxy:proxy squid/

2010年9月7日 星期二

join win2003 domain

#apt-get install krb5-user libpam-krb5 libnss-winbin libpam-winbind krb5-config winbind acl samba
install kerberos samba module

#vim /etc/netwwork/interface
modify network setting
===========================================
auto lo
iface lo inet loopback

auto eth0
iface eth0 inet static
address 10.10.10.100
netmask 255.255.255.0
gateway 10.10.10.1

#vim /etc/hosts
modify the host file to add domain controller ip
===========================================
127.0.0.1 localhost
10.10.10.100 ubuntu.csp.com.tw ubuntu
10.10.10.1 twad1.csp.com.tw twad1

#vim /etc/resolve.conf
add new dns server
search csp.com.tw
nameserver 10.10.10.1

#ntpdate 10.10.10.1
sync with ntp server

#vim /etc/krb5.conf
modify kerberos configuration
add lines as below
=================================================
[logging]
default = FILE:/var/log/krb5.conf
[libdefaults]
default_realm = CSP.COM.TW
dns_lookup_realm = false
dns_lookup_kdc = false
ticket_lifetime = 24h
forwardable = yes

CSP.COM.TW={
kdc = twad1.csp.com.tw
admin_server = twad1.csp.com.tw
default_domain = csp.com.tw
}

[domain_realm]
.csp.com.tw = CSP.COM.TW
csp.com.tw = CPS.COM.TW

#vim /etc/samba/smb.conf
modify samba configuration
===========================================
#vim /etc/nsswitch.conf
passwd: compat file winbind
group: compat file winbind
===========================================
workgroup = CSP
preferred master = no
idmap uid = 10000-20000
idmap gid = 10000-20000
winbind separator =+
winbind enum users = yes
winbind enum groups = yes
template homedir = /home/%D/%U
template primary groups= "Domain users"
template shell = /bin/bash
client use spnego = yes
ntlm auth = yes
encrypt passwords = yes
winbind use default domain = yes
winbind nested groups = yes
restrict anonymous = 2
security = ads
realm = CSP.COM.TW
password server = 10.10.10.1

[homes]
comment = home folders
valid user = %U
read only = no
browseable = no
[Share]
comment = share for active directory auth
path = /share
browseable = yes
read only = no
inherit acles = yes
inherit permissions = yes
create mask = 700
directory mask = 700
valid users = @"CSP+linux team","CSP+tommy","CSP+john"
admin users = @"CSP+admin team"

#/etc/init.d/smbd stop
#/etc/init.d/nmbd stop
#/etc/init.d/winbind stop
#/etc/init.d/winbind start
#/etc/init.d/nmbd start
#/etc/init.d/smbd start
restart samba service and winbind service

#net ads join -S TWAD1 -U username%password
join ad with ad account and password

#vim /etc/pam.d/session-session
add one line
session required pam_mkhomedir.so umask=0022 skel=/etc/skel

#mkdir /home/CSP
#chmod 755 /home/CSP
mkdir home folder for all CSP Domain user

#wbinfo -g
#wbinfo -u
use wbinfo to list all username and group about domain

#smbstatus
to list samba vesion

add proxy to winbind_privilege group
#vim /etc/group
winbindd_priv:x:112:proxy

2010年9月2日 星期四

openvpn client install

1.install openvpn gui client on windows
2.copy ca.crt client1.crt client1.key from server to C:\Program Files (x86)\OpenVPN\config(64 bit or 32bit)
3.modify openvpn1.ovpn
add content as below
================================================================
client
dev tun
proto udp
remote-cert-tls server
remote 210.200.200.200 1195
resolv-retry infinite
nobind
persist-key
persist-tun
ca C:\\Program\ Files\ (x86)\\OpenVPN\\config\\ca.crt
cert C:\\Program\ Files\ (x86)\\OpenVPN\\config\\client1.crt
key C:\\Program\ Files\ (x86)\\OpenVPN\\config\\client1.key
tls-auth C:\\Program\ Files\ (x86)\\OpenVPN\\config\\ta.key 1
tls-client
comp-lzo
verb 3
==============================================================

2010年9月1日 星期三

openvpn server installion

#apt-get update
#apt-get upgrade
update and upgrade package

#apt-get install openvpn
install openvpn package

#cd /etc/openvpn
#mkdir /etc/openvpn/easy-rsa/
#cp -r /usr/share/doc/openvpn/examples/easy-rsa/2.0/* /etc/openvpn/easy-rsa/
copy sample file to /etc/openvpn folder

#edit /etc/openvpn/easy-rsa/vars
edit some vars for generate cert and key use
export KEY_COUNTRY="TW"
export KEY_PROVINCE="TW"
export KEY_CITY="TAIPEI"
export KEY_ORG="Cloud Service"
export KEY_EMAIL="tommy@csp.com"


#cd /etc/openvpn/easy-rsa/2.0
#source ./vars
#./clean-all
#./build-ca
#./build-key-server server
#./build-key client1
#./build-dh
create server certificate and client keys

#cd .../../
#mkdir cert
#cd cert
#mv ca.crt ca.key server.crt server.key dh1024.pem /etc/openvpn/cert/
move the keys that server needs

#cd ..
#vim server.conf
modify server.conf and add below content to it
=====================================================================
port 1195
proto udp
dev tun0

ca /etc/openvpn/cert/ca.crt
cert /etc/openvpn/cert/server.crt
key /etc/openvpn/cert/server.key # This file should be kept secret
dh /etc/openvpn/cert/dh1024.pem
;tls-auth /etc/openvpn/cert/ta.key 0 # This file is secret

server 172.16.0.192 255.255.255.192
ifconfig-pool-persist ipp.txt
push "route 10.0.0.0 255.0.0.0"
push "route 192.168.0.0 255.255.0.0"
push "dhcp-option DNS x.x.x.x"
push "dhcp-option DNS x.x.x.x"
push "dhcp-option WINS x.x.x.x"
push "dhcp-option WINS x.x.x.x"
push "dhcp-option DOMAIN csp.com.tw"

keepalive 10 120
comp-lzo
persist-key
persist-tun
status openvpn-status.log
log openvpn.log
log-append openvpn.log

#vim /etc/network/interfaces
add wanip for vpn client connect use
# The loopback network interface
auto lo
iface lo inet loopback

# The primary network interface
auto eth0
iface eth0 inet static
address 172.16.0.10
netmask 255.255.255.0
up /etc/network/route up
down /etc/network/route down

auto eth1
iface eth1 inet static
address 210.200.200.200
netmask 255.255.255.128
gateway 210.200.200.1

#vim /etc/network/route
add eth0 routing table
#!/bin/sh

PATH=/sbin:/bin:/usr/bin:/usr/sbin
case $1 in
up)
route add 172.16.0.0 netmask 255.255.0.0 gw 172.16.0.1
route add 10.0.0.0 netmask 255.0.0.0 gw 172.16.0.1
;;
down)
route del 172.16.0.0 netmask 255.255.0.0. gw 172.16.0.1
route del 10.0.0.0 netmask 255.0.0.0 gw 172.16.0.1
;;
esac

#vim /etc/sysctl.conf
uncomment the line below
#net.ipv4.ip_forward=1 to net.ipv4_forward=1

#permit all traffice from vpn interface
iptables -t nat -A POSTROUTING -s 172.16.0.192/26 -o eth0 -j MASQUERADE

#/etc/init.d/openvpn start
start openvpn service

2010年7月21日 星期三

how to install c compiler

#sudo apt-get install build-essential

show gcc version
#gcc --version
gcc (Ubuntu 4.4.3-4ubuntu5) 4.4.3
Copyright (C) 2009 Free Software Foundation, Inc.
This is free software; see the source for copying conditions. There is NO
warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

2010年7月19日 星期一

how to check ubunut version

#cat /etc/issue
Ubuntu 10.04 LTS \n \l

#cat /proc/version
Linux version 2.6.32-21-generic (buildd@rothera) (gcc version 4.4.3 (Ubuntu 4.4. 3-4ubuntu5) ) #32-Ubuntu SMP Fri Apr 16 08:10:02 UTC 2010

#uname -a
Linux ubuntu 2.6.32-21-generic #32-Ubuntu SMP Fri Apr 16 08:10:02 UTC 2010 i686 GNU/Linux

#cat /etc/lsb-release
DISTRIB_ID=Ubuntu
DISTRIB_RELEASE=10.04
DISTRIB_CODENAME=lucid
DISTRIB_DESCRIPTION="Ubuntu 10.04 LTS"


#lsb_release -a
Distributor ID: Ubuntu
Description: Ubuntu 10.04 LTS
Release: 10.04
Codename: lucid

#file /usr/bin/file