Cuma buat dokumentasi biar gak lupa, script ini untuk FTP dengan menggunakan expect.
Biasa dipake buat mining data sih jadi transfer raw log ke server data mining lalu di olah sebelum dimasukin ke database.
Dibawah ini scriptnya.
#!/bin/bash
VER="V3.0"
VER_DATE="26-04-2010"
#MapUssd
REMOTE_TRANSFER_ENABLE[0]="" # 0 -disable 1 -enable(default)
REMOTE_TRANSFER_MODE[0]="" # 0 -put files to remote 1 -get files from remote
REMOTE_USERNAME[0]="" # Username for sftp (Remote Destination)
REMOTE_PASSWORD[0]="" # password (Remote Password)
REMOTE_IP[0]="" # Remote destination server ip (UMBRPTCL5)
REMOTE_IP_PORT[0]="22" # Remote destination server ip port
REMOTE_DEST_PATH[0]="" # Remote destination directory
LOGFILE_PREFIX[0]="" # File prefix
LOGFILE_TYPE[0]="" # File extention
LOGFILE_MINSIZE[0]="" # File minimum size
LOCAL_DEST_PATH[0]="" # Local destination
ZIP_AFTER_TRANSFER[0]="" # 0 -disable 1 -enable zip after file transferred (realtime mode only)
DELETE_AFTER_TRANSFER[0]="" # 0 -disable 1 -enable delete after file transferred (realtime mode only)
# [Systerm Default value setting ]
DATETIME=`date +'%Y%m%d'`
FILE_DATE=`TZ='GMT+23.59' date +'%Y%m%d'`
#FILE_DATE=`TZ='GMT+23.59' date +'%Y%m%d' --date='1 days ago'` # Satu hari yang lalu
LAST_MODIFIED_TIME="30" # value in min
FILELIST_TMPFILE="/tmp/tmpfilelist"
RES_TMPFILE="/tmp/tmpRESULTOUTPUT"
SFTP_OUTPUT_TMPFILE="" #fill with your directory for output log
###################################################################
# Do not Edit any value after this line #
###################################################################
# Check OS type and define command path
OSTYPE=`uname |tr [A-Z] [a-z]`
case $OSTYPE in
sunos|SunOS)
FIND="/usr/local/bin/find"
EXPECT="/usr/local/bin/expect"
SFTP="/usr/local/bin/sftp"
SSH="/usr/local/bin/ssh"
if [ ! -e "$FIND" ]; then
echo "Unable to locate $FIND"
echo "Please install findutils-4.2.30-sol9-sparc-local or above"
EXIT=1
fi
if [ ! -e "$EXPECT" ]; then
echo "Unable to locate $EXPECT"
echo "Please install expect-5.43.0-sol9-sparc-local or above"
EXIT=1
fi
if [ ! -e "$SFTP" ]; then
echo "Unable to locate $SFTP"
echo "Please install openssh-4.7p1-sol9-sparc-local or above"
EXIT=1
fi
if [ "$EXIT" == "1" ]; then
echo "Unable to continue due to missing packages!"
exit 1
fi
;;
linux|Linux)
FIND="/usr/bin/find"
EXPECT="/usr/bin/expect"
SFTP="/usr/bin/sftp"
SSH="/usr/bin/ssh"
if [ ! -e "$SFTP" ]; then
echo "Unable to locate $SFTP"
echo "Please install expect-5.42.1-1 or above"
EXIT=1
fi
if [ "$EXIT" == "1" ]; then
echo "Unable to continue due to missing packages!"
exit 1
fi
;;
*) echo "Unsupported OS type [$OSTYPE] "
exit 1
;;
esac
# calculate number of configured destination ip
numarr=${#REMOTE_IP[*]}
numarr=`expr $numarr - 1`
# will not allow interrupt during script run
OWN_PID=`echo $$`
trap "echo -n "wait." " 1 2 3 15
# Function to Display current script usage
usage()
{
echo "SFTP_uclog.sh $VER [ $VER_DATE ]"
echo "Usage: $(basename $0) [-a] [-d value] " >&2
echo "OPTIONS"
echo " -a Interactive mode"
echo " For e.g. $(basename $0) -a"
echo " -d Date of the file needed to transfer (YYYYMMDD format)"
echo " For e.g. $(basename $0) -d 20070831"
echo " -r RealTime mode"
echo " For e.g. $(basename $0) -r"
exit 2
}
# Function to display transfering progress but not for REALTIME mode
PROGRESSBAR(){
trap "echo '.DONE'; exit" 1 2 3 15
echo -n "Session $1 Progressing..."
while true ;
do
sleep 3
echo -n .
done
}
# Function to convert month which in number to Alphabet
MONTH_CHECK() {
MONTHS="Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec"
echo $MONTHS |awk '{print $'"${1}"' }'
}
# Function to transfer file using SFTP protocol
# This function included to GET or PUT or Batch file features
SFTP_FUNC()
{
# start sftp session and transfer log from remote to local
# Begin file transfer using expect command
# 1 = local dest path
# 2 = remote username
# 3 = remote password
# 4 = remote ip
# 5 = remote port
# 6 = remote dest path
# 7 = filename
# 8 = GET/PUT file
# 9 = Batch file list
if [ "$9" == "" ]; then
PROGRESSBAR $i &
PROGRESSBAR_PID=`echo $!`
var=`$EXPECT << EOF
set timeout -1
log_user 1
# log_file -noappend $SFTP_OUTPUT_TMPFILE
log_file $SFTP_OUTPUT_TMPFILE
spawn $SFTP -oPort=$5 $2@$4
expect {
"Are you sure you want to continue connecting"
{
send "yes\r"
exp_continue
}
"*?assword*"
{
send "$3\r"
exp_continue
}
"sftp> "
{
send "lcd $1\r"
expect "sftp> "
send "cd $6\r"
expect "sftp> "
send "$8 $7\r"
}
}
expect "sftp>"
send "bye\r"
EOF`
kill $PROGRESSBAR_PID
return 0
else
var=`$EXPECT << EOF
set timeout -1
log_user 1
# log_file -noappend $SFTP_OUTPUT_TMPFILE
log_file $SFTP_OUTPUT_TMPFILE
spawn $SFTP -o "batchmode no" -b $9 -oPort=$5 $2@$4
expect {
"Are you sure you want to continue connecting"
{
send "yes\r"
exp_continue
}
"*?assword*"
{
send "$3\r"
exp_continue
}
}
EOF`
return 0
fi
}
# Function to Get remote Server latest file list thru SFTP
GET_REMOTE_LATEST_FILELIST_FUNC()
{
A=$1
B=$2
C=$3
D=$4
E=$5
F=$6
G=$7
H=$8
I=$9
var=`$EXPECT << EOF
set timeout -1
log_user 1
log_file -noappend $RES_TMPFILE
spawn $SSH -oPort=$E $B@$D
expect {
"Are you sure you want to continue connecting"
{
send "yes\r"
exp_continue
}
"*?assword*"
{
send "$C\r"
exp_continue
}
"$ "
{
send "cd .. \r"
expect "$ "
send "nice -n 19 $FIND $F -name $G\*$I -exec basename {} '\;' \r"
expect "$ "
send "\r"
}
}
expect "$ "
send "bye\r"
EOF`
#fi
return 0
}
# Function to zip remote server file which being transferred completely
ZIP_REMOTE_FILE_AFTER_TRANSFER()
{
A=$1
B=$2
C=$3
D=$4
E=$5
F=$6
G=$7
H=$8
I=$9
var=`$EXPECT << EOF
set timeout -1
log_user 1
# log_file -a $SFTP_OUTPUT_TMPFILE
log_file $SFTP_OUTPUT_TMPFILE
spawn $SSH -oPort=$E $B@$D
expect {
"Are you sure you want to continue connecting"
{
send "yes\r"
exp_continue
}
"*?assword*"
{
send "$C\r"
exp_continue
}
"$ "
{
send "cd .. \r"
expect "$ "
send "nice -n 19 $FIND $F -name $G -exec nice -n 19 gzip -f {} '\;' \r"
expect "$ "
send "\r"
}
}
expect "$ "
send "bye\r"
EOF`
#fi
return 0
}
# Function to cater RealTime options
REALTIME_FUNC()
{
echo "Session $1 REALTIME start"
A=${LOCAL_DEST_PATH[$1]}
B=${REMOTE_USERNAME[$1]}
C=${REMOTE_PASSWORD[$1]}
D=${REMOTE_IP[$1]}
E=${REMOTE_IP_PORT[$1]}
F=${REMOTE_DEST_PATH[$1]}
G=${LOGFILE_PREFIX[$1]}
H=$FILE_DATE
I=${LOGFILE_TYPE[$1]}
MON_DIR=`echo $FILE_DATE|cut -c 5-6`
MON_DIR=`MONTH_CHECK $MON_DIR`
A=`echo $A | sed 's/MONTH_DIR/'$MON_DIR'/g'`
F=`echo $F | sed 's/MONTH_DIR/'$MON_DIR'/g'`
echo > $FILELIST_TMPFILE
if [ ${REMOTE_TRANSFER_MODE[$1]} == "0" ]; then
echo "Session $1 Transfer local files to remote"
GET_FILELIST=(`nice -n 19 $FIND $A -maxdepth 1 -name \*$I -regex .*$G.* -mmin -60 -exec basename {} \; `)
echo "Session $1 Total Files = ${#GET_FILELIST[*]}"
if [ "${#GET_FILELIST[*]}" -gt 0 ]; then
FILELISTARR=${#GET_FILELIST[*]}
FILELISTARR=`expr $FILELISTARR - 1`
echo "lcd $A" > $FILELIST_TMPFILE
for (( i = 0 ; i <= $FILELISTARR ; i++ ))
do
if [ `/sbin/fuser -uv $A${GET_FILELIST[$i]} |wc -l` == "0" ]; then
echo "put ${GET_FILELIST[$i]} $F${GET_FILELIST[$i]}.tmp" >> $FILELIST_TMPFILE
echo "rename $F${GET_FILELIST[$i]}.tmp $F${GET_FILELIST[$i]}" >> $FILELIST_TMPFILE
sleep 2
fi
done
echo bye >> $FILELIST_TMPFILE
SFTP_FUNC $A $B $C $D $E $F Null put $FILELIST_TMPFILE
if [[ "${ZIP_AFTER_TRANSFER[$1]}" == "1" ]]; then
for (( i = 0 ; i <= $FILELISTARR ; i++ ))
do
nice -n 19 $FIND $A -maxdepth 1 -name ${GET_FILELIST[$i]} -exec nice -n 19 gzip -f {} \;
done
echo "Session $1 FILE Zipping was Done!"
else
echo "Session $1 FILE Zipping Skipped!"
echo "Session $1 FILE Transfer Completed"
fi
if [[ "${DELETE_AFTER_TRANSFER[$1]}" == "1" ]]; then
for (( i = 0 ; i <= $FILELISTARR ; i++ ))
do
nice -n 19 $FIND $A -maxdepth 1 -name ${GET_FILELIST[$i]} -exec nice -n 19 rm -f {} \;
done
echo "Session $1 FILE Delete was Done!"
else
echo "Session $1 FILE Delete Skipped!"
echo "Session $1 FILE Delete Completed"
fi
return 0
else
echo "Session $1 NO FILE FOUND!"
return 1
fi
else
echo "Session $1 Transfer remote files to local"
GET_REMOTE_LATEST_FILELIST_FUNC $A $B $C $D $E $F $G $H $I
wait
dos2unix -a $RES_TMPFILE >/dev/null 2>&1
GET_FILELIST=(`grep -v $FIND $RES_TMPFILE |grep ${LOGFILE_TYPE[$1]} | grep ${LOGFILE_PREFIX[$1]} `)
echo "Session $1 Total Files = ${#GET_FILELIST[*]}"
if [ "${#GET_FILELIST[*]}" -gt 0 ]; then
FILELISTARR=${#GET_FILELIST[*]}
FILELISTARR=`expr $FILELISTARR - 1`
echo "cd $F" > $FILELIST_TMPFILE
for (( i = 0 ; i <= $FILELISTARR ; i++ ))
do
echo "get ${GET_FILELIST[$i]}" $A >> $FILELIST_TMPFILE
done
echo bye >> $FILELIST_TMPFILE
SFTP_FUNC $A $B $C $D $E $F Null get $FILELIST_TMPFILE
if [[ "${ZIP_AFTER_TRANSFER[$1]}" == "1" ]]; then
for (( i = 0 ; i <= $FILELISTARR ; i++ ))
do
ZIP_REMOTE_FILE_AFTER_TRANSFER $A $B $C $D $E $F ${GET_FILELIST[$i]}
done
echo "Session $1 FILE Zipping was Done!"
else
echo "Session $1 FILE Zipping Skipped!"
echo "Session $1 FILE Transfer Completed"
fi
if [[ "${DELETE_AFTER_TRANSFER[$1]}" == "1" ]]; then
for ((i = 0; i <= $FILELISTARR; i++))
do
DELETE_REMOTE_FILE_AFTER_TRANSFER $A $B $C $D $E $F ${GET_FILELIST[$i]}
done
echo "Session $1 FILE Deleting was DONE!"
else
echo "Session $1 FILE Deleting Skipped!"
echo "Session $1 FILE Delete Completed"
fi
return 0
else
echo "Session $1 NO FILE FOUND!"
return 1
fi
fi
}
# Funtion to cater non-realtime mode
# For eg, transfer yesterday file or user specified
SESSION_FUNC()
{
echo "Session $1 start"
A=${LOCAL_DEST_PATH[$1]}
B=${REMOTE_USERNAME[$1]}
C=${REMOTE_PASSWORD[$1]}
D=${REMOTE_IP[$1]}
E=${REMOTE_IP_PORT[$1]}
F=${REMOTE_DEST_PATH[$1]}
G=${LOGFILE_PREFIX[$1]}*$FILE_DATE*${LOGFILE_TYPE[$1]}
MON_DIR=`echo $FILE_DATE|cut -c 5-6`
MON_DIR=`MONTH_CHECK $MON_DIR`
A=`echo $A | sed 's/MONTH_DIR/'$MON_DIR'/g'`
F=`echo $F | sed 's/MONTH_DIR/'$MON_DIR'/g'`
if [ "${REMOTE_IP[$1]}" != "" ]; then
if [ ${REMOTE_TRANSFER_ENABLE[$1]} == "1" ]; then
echo "Session $1 Normal Transfer enabled"
if [ ${LOCAL_DEST_PATH[$1]} != "" ]; then
if [ ${REMOTE_TRANSFER_MODE[$1]} == "0" ]; then
echo "Session $i Transfer local files to remote"
SFTP_FUNC $A $B $C $D $E $F $G put
else
echo "Session $i Transfer remote files to local"
SFTP_FUNC $A $B $C $D $E $F $G get
fi
wait
fi
elif [ "${REMOTE_TRANSFER_ENABLE[$1]}" == "0" ]; then
echo "Remote Transfer for $1 = Disabled"
return 0
else
echo "Session $1 Invalid Option"
return 1
fi
else
echo "Session $1 Invalid Option"
return 1
fi
}
# Function to delete remote server file which being transferred completely
DELETE_REMOTE_FILE_AFTER_TRANSFER()
{
A=$1
B=$2
C=$3
D=$4
E=$5
F=$6
G=$7
H=$8
I=$9
var=`$EXPECT << EOF
set timeout -1
log_user 1
log_file -a $SFTP_OUTPUT_TMPFILE
spawn $SSH -oPort=$E $B@$D
expect {
"Are you sure you want to continue connecting"
{
send "yes\r"
exp_continue
}
"*?assword*"
{
send "$C\r"
exp_continue
}
"$ "
{
send "cd .. \r"
expect "$ "
send "nice -n 19 $FIND $F -name $G -exec nice -n 19 rm -f {} '\;' \r"
expect "$ "
send "\r"
}
}
expect "$ "
send "bye\r"
EOF`
#fi
return 0
}
###########################################
# MAIN
###########################################
# To collect User specify info if any
if [ $# == "0" ]; then
echo "Transfer All Log Files with date = $FILE_DATE"
else
echo "Options on"
while getopts 'ad:vr' OPTION
do
case $OPTION in
a) # List all current config and let accept user input
echo "Selection:"
for (( i = 0 ; i <= $numarr ; i++ ))
do
echo -e "\t $i ) ${REMOTE_IP[$i]} ${REMOTE_DEST_PATH[$i]}"
done
echo -n "Please choose which file needed to transfer = "
read USR_OPT
if [ "$USR_OPT" == "" ] ; then
echo "No input! Bye"
exit
else
echo "User Input Option = $USR_OPT"
echo -n "Are you sure? [y|n]"
read USR_OPT_FLAG
case $USR_OPT_FLAG in
y|Y) USR_OPT=$USR_OPT
OPT_FLAG=1
;;
*) echo Bye
exit 0
;;
esac
fi
;;
d) # Date specified by user
# count the len of the arguement given
/usr/bin/expr $OPTARG - 0 >/dev/null 2>&1
INTCHECK=$?
if [ "$INTCHECK" == "0" ]; then
DATE_LEN=${#OPTARG}
if [ "$DATE_LEN" -ne 8 ]; then
echo "Wrong Date Format! = $OPTARG"
usage
exit
fi
echo "Transfer Log File with date = $OPTARG"
echo -n "Are you sure?[y/n]"
read USER_COMFIRMATION
case $USER_COMFIRMATION in
y|Y)
FILE_DATE=$OPTARG
;;
*)echo bye
exit
;;
esac
else
usage
exit 0
fi
;;
r) # real time mode
echo "Real Time mode enable"
REALTIME_ENABLE=1
;;
*) usage
;;
esac
done
fi
# Processing Part
if [ "$REALTIME_ENABLE" == 1 ]; then
for (( y = 0 ; y <= $numarr ; y++ ))
do
FILE_DATE=`date '+%Y%m%d'`
REALTIME_FUNC $y
done
else
if [ "$OPT_FLAG" == 1 ]; then
for y in $USR_OPT
do
SESSION_FUNC $y
done
else
for (( y = 0 ; y <= $numarr ; y++ ))
do
SESSION_FUNC $y
done
fi
fi
exit 0
No comments:
Post a Comment