Difference between revisions of "Scripts"

Latest revision as of 12:17, 4 September 2019

Flying replication

rsync -avz 192.168.1.122:/usr/local/apache/conf/ /usr/local/apache/conf/ --delete
rsync -avz 192.168.1.122:/var/named/ /var/named/ --delete
rsync -avz 192.168.1.122:/usr/local/cpanel/ /usr/local/cpanel/ --delete
rsync -avz 192.168.1.122:/var/cpanel/ /var/cpanel --delete

for x in `cat /etc/trueuserowners|grep -v "#"|cut -d: -f1`;do /usr/local/cpanel/bin/setsiteip -u $x `hostname -i` ;done;  

/scripts/rebuildhttpdconf

WHM replication to clean server

#!/bin/bash
rsync -avz 192.168.1.21:/etc/apache2/conf/ /etc/httpd/conf/
rsync -avz 192.168.1.21:/var/cpanel/ssl/installed/ /etc/ssl/certs/
rsync -avz 192.168.1.21:/etc/apache2/conf.d/includes/ /etc/httpd/conf.d/includes/
rsync -avz 192.168.1.21:/var/cpanel/ssl/cpanel/ /etc/ssl/certs/cpanel/
sed -i "s/192.168.1.21/`hostname -i`/g" /etc/httpd/conf/httpd.conf
sed -i "s/\/var\/cpanel\/ssl\/installed\//\/etc\/ssl\/certs\//g" /etc/httpd/conf/httpd.conf
sed -i "s/\/etc\/apache2\//\/etc\/httpd\//g" /etc/httpd/conf/httpd.conf
sed -i "s/RewriteMap LeechProtect/\#RewriteMap LeechProtect/g" /etc/httpd/conf/httpd.conf
sed -i "s/Mutex file:\/run\/apache2/\#Mutex file:\/run\/apache2/g" /etc/httpd/conf/httpd.conf
sed -i "s/apache2/httpd/g" /etc/httpd/conf/httpd.conf
sed -i "s/\/var\/cpanel\/ssl\/cpanel\//\/etc\/ssl\/certs\/cpanel\//g" /etc/httpd/conf/httpd.conf
sed -i "s/Listen \[\:\:\]\:80/\#Listen \[\:\:\]\:80/g" /etc/httpd/conf/httpd.conf

rsync -avz 192.168.1.21:/opt/cpanel/ea-php55/root/etc/php.ini /etc/
rsync -avz 192.168.1.21:/etc/passwd /tmp/passwd
rsync -avz 192.168.1.21:/etc/group /tmp/group

var1=`cat /etc/passwd |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'`
var2=`cat /tmp/passwd |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'`

c1=`cat /etc/passwd |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'|wc -l`
c2=`cat /tmp/passwd |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'|wc -l`

if [ $c2 -gt $c1 ]; then
       newone=`comm --nocheck-order -13 <(echo $var1) <(echo $var2)`
       echo $newone >> /etc/passwd
fi

var1=`cat /etc/group |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'`
var2=`cat /tmp/group |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'`

c1=`cat /etc/group |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'|wc -l`
c2=`cat /tmp/group |awk -F ':' '$3 > 1000 && $3 < 2000 {print $0}'|wc -l`

if [ $c2 -gt $c1 ]; then
       newone=`comm --nocheck-order -13 <(echo $var1) <(echo $var2)`
       echo $newone >> /etc/group
fi

WHM replicate suphp to clean server

scp /usr/lib64/httpd/modules/mod_suphp.so 172.18.0.6:/usr/lib64/httpd/modules/
scp /etc/httpd/conf.modules.d/90-suphp.conf 172.18.0.6:/etc/httpd/conf.modules.d/
scp /usr/lib64/httpd/modules/mod_suphp.so 172.18.0.6:/usr/lib64/httpd/modules/
scp /usr/sbin/suphp 172.18.0.6:/usr/sbin/
scp /etc/suphp.conf 172.18.0.6:/etc/
scp /etc/httpd/conf.d/php.conf 172.18.0.6:/etc/httpd/conf.d/
scp /etc/httpd/conf.modules.d/10-php.conf 172.18.0.6:/etc/httpd/conf.modules.d/

chown root.nobody /usr/sbin/suphp
chmod 4750 /usr/sbin/suphp

Sync nginx conf

#!/bin/bash

servs=(192.168.0.11 192.168.0.12 192.168.0.15 192.168.0.16 192.168.0.17 192.168.0.19 192.168.0.20 192.168.0.21 192.168.0.22 192.168.0.23 192.168.0.24 192.168.0.26 192.168.0.115 192.168.0.116 192.168.0.117 192.168.0.118 192.168.0.29 192.168.0.30)

for i in "${servs[@]}"
do
       echo $i
       scp nginx_conf/nginx.conf $i:/etc/nginx/
       scp nginx_conf/.htpasswd $i:/etc/nginx/
       scp -r nginx_conf/conf.d $i:/etc/nginx/
       ssh $i 'sed -i "s/Lxxx/`echo L\`hostname|cut -d "-" -f1|sed "s/web//"\``/g" /etc/nginx/conf.d/* && service nginx restart'
done

Manage CWM auto start and stop server

#!/bin/bash
. keys.sh


minservs=4
maxservs=8
locon=2500
hicon=8000

servs=($serv_01 $serv_02 $serv_03 $serv_04 $serv_05 $serv_06 $serv_07 $serv_08)
onservs=0
conns=0
#echo ${servs[@]}

for i in "${servs[@]}"
do
       readarray -t values < <(curl -s -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" "https://console.cloudwm.com/service/server/${i}"|/root/api/jq -r '(map(values))[]') # | egrep "power|ip"
       #echo ${values[6]} ##power
       #echo ${values[14]} |sed 's/\"//g' ##ips
       if [ ${values[6]} == 'on' ]; then 
               #onservs=$((onservs+1))
               ((onservs++))
               ip=$(echo ${values[14]} |tr -d \"\'\",\") ##ips
               curcon=$(ssh $ip "netstat -ant | grep :80 | wc -l")
               #echo $ip , $curcon
               conns=$((conns+curcon))
       fi
       #declare -p values
done

#echo "On servers: " $onservs
#echo "Connections: " $conns

mid=$((conns/onservs))
#echo "mid: " $mid

if [[ $mid > $hicon ]] && [[ $onservs < $maxservs ]]; then   ## power on +1
       #echo "hi"
       srv=${servs[onservs]}
       #echo $srv
       curl -s -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" -X PUT -d "power=on" "https://console.cloudwm.com/service/server/${srv}/power"

elif [[ $mid < $locon ]] && [[ $onservs > $minservs ]]; then   ## power off -1
       #echo "lo" $onservs
       srv=${servs[onservs-1]}
       #echo $srv
       curl -s -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" -X PUT -d "power=off" "https://console.cloudwm.com/service/server/${srv}/power"
fi

SQL dump

#!/bin/bash

retain=7                                ## days to keep
logdir="/var/www/html/sqlbackups/"      ## dumps folder
prename="db_Backup"                     ## name prefix
postname=".sql.gz"                      ## name suffix
mailaddr="eitani@gmail.com"

dumpcmd="mysqldump --all-databases"
date=`date +"%d%b%y"`
tmst='date +%Y%m%d-%H%M%S.%N'
outstr="[`$tmst`]: Dump started\n"

fullname=$logdir$prename"_"$date$postname

if [ -z "$logdir" ]; then
       outstr=$outstr"[`$tmst`]: Log directory not set\n"
       out=1
else
       outstr=$outstr"[`$tmst`]: Running dump $dumpcmd | gzip -c > $fullname\n"
       $dumpcmd | gzip -c > $fullname
       out=$?
fi

if [[ $out = 0 ]]; then 
       outstr=$outstr"[`$tmst`]: Dump done, file: $fullname\n"
       outstr=$outstr"[`$tmst`]: Deleting old backups (older than $retain days) from folder: $logdir\n"
       outstr=$outstr"[`$tmst`]: Files to be deleted: `find $logdir -type f -mtime +$retain `\n"
       `find $logdir -type f -mtime +$retain -exec rm {} \;`
else
       outstr=$outstr"[`$tmst`]: Dump failed\n"
fi

outstr=$outstr"[`$tmst`]: Script done\n"

#echo -e $outstr

mailtxt="Subject: SQLdump on webbit server\nFrom: script@webbit.web\nTo: $mailaddr\n\n$outstr"
echo -e $mailtxt | sendmail -t

Check DNS zones

#!/bin/bash

iparr=()
ipbad=()

tempfile="/tmp/zonetempfile.db"
backupdir="/root/oldzones"

getzones () {
       zones=($(find /var/named/*.db -printf '%f\n'|sed 's/\.db//g'))
       #echo $zones
}

findoffs () {
       fl=$1
       a=()
       #echo "File: " $fl
       #cat $fl | while read line
       #do
       while read -r line
       do
               #echo "line: "$line
               if [[ "${line:0:1}" == ";" ]] && [[ $line == *"IN A"* ]] ; then
                       addr=$(echo $line|awk '{print $4}')
                       #echo "adr:" $addr
                       checkip $addr
                       #echo done
                       changed+=($res)
               fi
       done < <(cat $fl)
       #done
}

checkip () {
       ipad=$1
       #echo $ipad
       if [[ "${iparr[@]}" =~ "i${ipad}i" ]]; then
       #if [[ "${iparr[@]}" == "${ipad}" ]]; then
               #echo "Found: " $ipad
               return 0
       elif [[ "${ipbad[@]}" =~ "i${ipad}i" ]]; then
               #echo "found bad: " $ipad
               return 1
       else
               #echo "Not found: " $ipad
               timeout 2 ping -c 1 $addr > /dev/null
               res=$?
               ii="i"$ipad"i"
               if  $res == 0 ; then
                       #echo $ii
                       #iparr+=($ipad)
                       iparr+=($ii)
                       #echo "add to good: "$ii
                       return 0
               else
                       ipbad+=($ii)
                       #echo "add to bad: "$ii ", old: "${ipbad[@]}
                       return 1
               fi
               #echo "no"
       fi
}

parsezone () {
       zone=$1
       #echo "Zone: " $zone
       fname="/var/named/$1.db"
       #echo "Name: " $fname
       parsed=($(named-checkzone -q -i none -s full -D $1 /var/named/$1.db | egrep "IN SOA|IN A"))
       #for i in `named-checkzone -q -i none -s full -D $1 /var/named/$1.db | egrep "IN SOA|IN A"`
       #do
       #       echo "u: " $i
       #done


       #echo "Parsed: " $parsed
       for i in ${parsed[@]}
       do
               #echo "z: " $zone "i: " $i
               if  $i == *"SOA"* ; then
                       ser=$(echo $i|awk '{print $7}')
                       #echo "Ser: " $ser
               elif  $i == *"IN A"* ; then
                       host=$(echo $i|awk '{print $1}')
                       addr=$(echo $i|awk '{print $5}')
                       #echo "Pinging: *"$addr"*"
                       #timeout 3 ping -c 1 $addr #> /dev/null
                       checkip $addr
                       res=$?
                       let changed+=($res)
                       if [[ $res > 0 ]]; then
                               toblock+=($addr)
                       fi
                       #echo "Host: " $host "Addr: " $addr "Res: " $res
               fi
       done
       #echo "Block: " $toblock
       #echo "Parsed: " ${parsed[@]}
       #sers=$(echo $parsed|grep SOA|awk '{print $7}')
       #sers=$(echo $parsed|grep SOA)
       #arec=$(echo $parsed|grep "IN A")
       #echo "ser: " $sers
       #echo ${arec[@]}
       #for i in "${arec[@]}"
       #do
       #       echo "As: " $i
       #done
}

updatezone () {
       uzone=$1
       zname=$2
       newser=$((ser+1))
       #echo "old ser" $ser
       #echo "new ser" $newser
       cp $zname /root/scripts/

       sed -i "s/$ser/$newser/" $zname   # update serial
       infile=$(cat $zname)
       #outfile="z.z"
       outfile=tempfile
       > $outfile
       #echo $infile
       for f1 in $infile
       do
               #echo " - "$f1
               newline=$f1
               if [[ "${f1:0:1}" == ";" ]] &&  $f1 == *"IN A"*  ; then
                       fip=$(echo $f1|awk '{print $4}')
                       #echo "add: "$fip
                       if ! [[ "${ipbad[@]}" =~ "i${fip}i" ]]; then
                               #echo "remove from bad" $fip
                               newline=$(echo $f1|sed -r 's/^.{1}//')
                               #echo "new: "$newline
                       fi
               elif  $f1 == *"IN A"*  ; then
                       fip=$(echo $f1|awk '{print $4}')
                       #echo "add: "$fip
                       if ! [[ "${iparr[@]}" =~ "i${fip}i" ]]; then
                               #echo "remove from good" $fip
                               newline=";"$f1
                               #echo "new: "$newline
                       fi
               fi
               echo $newline >> $outfile
       done
       cat $outfile > $zname
}

IFS='
'

#zone="eye-t.co.il"
getzones
#echo $zones

#if [ ! -d $backupdir ]; then
#       echo "nofolder"
#       mkdir $backupdir
#fi

curdate=$(date +"%Y%m%d-%H%M%S")

#echo $curdate

for zone in "${zones[@]}"
do
       changed=0
       toblock=()
       toallow=()

       parsezone $zone
       #echo $fname
       findoffs $fname
       #echo "change: " $changed
       #echo "good: " ${iparr[@]}
       #echo "bad: " ${ipbad[@]}
       if [[ $changed > 0 ]]; then
               if [ ! -d $backupdir/$curdate ]; then
                       mkdir $backupdir/$curdate -p
               fi
               echo "zone: " $zone "changed!!!"
               cp $fname $backupdir/$curdate/
               updatezone $zone $fname
       fi
done

Get all CFS users

After login, save the cookie.

Get all organizations:

curl --cookie "cwmid=001; PHPSESSID=xxx" "https://null.cloudwm.com/svc/fileserver/groups?size=500&sortBy=name&sortAsc=true&filter=&from="|jq -r '(map(values))[]'|grep "\"id"|cut -d"\"" -f4

Get users of an organization:

curl --cookie "cwmid=001; PHPSESSID=xxx" "https://null.cloudwm.com/svc/fileserver/group/$i"|jq -r '(map(values))[]'|grep "email"|cut -d"\"" -f4

Get full list of all users:

IFS=$'\n' ; for i in `curl --cookie "cwmid=001; PHPSESSID=xxx" "https://null.cloudwm.com/svc/fileserver/groups?size=500&sortBy=name&sortAsc=true&filter=&from="|jq -r '(map(values))[]'|grep "\"id"|cut -d"\"" -f4`; do z=$(echo $i|sed 's/ /%20/g') ; curl --cookie "cwmid=001; PHPSESSID=xxx" "https://null.cloudwm.com/svc/fileserver/group/$z"|jq -r '(map(values))[]'|grep "email"|cut -d"\"" -f4;done

Clone servers

#!/bin/bash
###########################################    README    ###########################################################################
## make sure to use followin jq binary: https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 
## before running the script change, parameters in settings and copy ssh key to all haproxies
## made for web servers only , modify the function if you wish to use for different balancing modes
#####################################################################################################################################

while [ -z $numbers ];do

read -p  "How many servers you want to clone?: " numbers

done

cloneServer () {

    ################ Settings ################
                src="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
                clientId="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
                secret="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
                haproxy1="xxx.xxx.xx.xx"
                haproxy2="xxx.xxx.xxx.xx"
                prefix="web"
                domainsuff="example.com"
                dc="IL"
                cpu="1D"
                billing="hourly"
                power="on"
                port="80" 
    ############### END SETTINGS #############

                NC='\033[0m'
                GREEN='\033[0;32m'
                YELLOW='\033[0;33m'
                
                last=$(curl -s -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" "https://api.cloudwm.com/service/servers" | jq '.[] | select( .datacenter | contains("'"${dc}"'"))' | jq '. | select( .name | startswith("'"${prefix}"'")) | .name' | cut -d'_' -f1 | grep -Eo '[0-9]{1,9}' | tail -1)

                name="${prefix}$((last+1)).${domainsuff}"

                lastname="${prefix}${last}.${domainsuff}"


                curl -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" -X POST -d "source=${src}&name=${name}&cpu=${cpu}&billing=${billing}&power=${power}" "https://console.clubvps.com/service/server"

                echo ""

                while [ -z "$newserverid" ] ;do

                newserverid=$(curl -s -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" "https://api.cloudwm.com/service/servers" | jq '.[] | select( .datacenter | contains("'"${dc}"'"))' | jq '. | select( .name | startswith("'"${name}"'")) | .id' | cut -d'"' -f2 | tail -1)

                sleep 5

                echo -e ${YELLOW}Waiting for new server named: ${GREEN}${name}${NC} ${YELLOW}to clone${NC}

                done

                newserverip=`curl -s -H "AuthClientId: ${clientId}" -H "AuthSecret: ${secret}" "https://api.cloudwm.com/service/server/${newserverid}" | jq '.networks[].ips[]' | cut -d'"' -f2`

                echo -e created new server named: ${GREEN}${name}${NC} with the ip: ${GREEN}${newserverip}${NC}

                echo -e ${GREEN}updating haproxies${NC}


                ssh root@${haproxy1} 'export '"lastname=$lastname"' && lastad=$(cat /etc/haproxy/haproxy.cfg | grep -e $lastname) && echo sed -i \"/${lastad}/a server '"${name}"' '"${newserverip}"':'"${port}"' check\" /etc/haproxy/haproxy.cfg > cmd.txt | cat cmd.txt|bash && service haproxy reload'
                ssh root@${haproxy1} 'export '"lastname=$lastname"' && lastad=$(cat /etc/haproxy/haproxy.cfg | grep -e $lastname) && echo sed -i \"/${lastad}/a server '"${name}"' '"${newserverip}"':'"${port}"' check\" /etc/haproxy/haproxy.cfg > cmd.txt | cat cmd.txt|bash && service haproxy reload'
                
                sleep 5
                echo -e ${GREEN}done${NC}



}

while [ $numbers -gt 0 ];
do

        cloneServer;

numbers=$(($numbers-1))

done

Fix Centos 5 Repos

rel=$(lsb_release -d | egrep -o '[0-9]\.[0-9]{1,2}')
file="/etc/yum.repos.d/CentOS-Base.repo"
sed -i.bak 's/mirror.centos.org\/centos\/$releasever/vault.centos.org\/'"$rel"'/g' $file
sed -i 's/\#baseurl/baseurl/g' $file
sed -i 's/mirror=*/\#mirrorlist=*/g' $file

Long version:

echo y | cp /etc/yum.repos.d/CentOS-Base.repo /etc/yum.repos.d/CentOS-Base.repo.bak
rel=$(lsb_release -d | egrep -o '[0-9]\.[0-9]{1,2}')
sed -i 's/\#baseurl=http:\/\/mirror.centos.org\/centos\/$releasever\/os\/$basearch\//baseurl=http:\/\/vault.centos.org\/'"$rel"'\/os\/$basearch/g' /etc/yum.repos.d/CentOS-Base.repo
sed -i 's/\#baseurl=http:\/\/mirror.centos.org\/centos\/$releasever\/updates\/$basearch\//baseurl=http:\/\/vault.centos.org\/'"$rel"'\/updates\/$basearch/g' /etc/yum.repos.d/CentOS-Base.repo
sed -i 's/\#baseurl=http:\/\/mirror.centos.org\/centos\/$releasever\/extras\/$basearch\//baseurl=http:\/\/vault.centos.org\/'"$rel"'\/extras\/$basearch/g' /etc/yum.repos.d/CentOS-Base.repo
sed -i 's/\#baseurl=http:\/\/mirror.centos.org\/centos\/$releasever\/centosplus\/$basearch\//baseurl=http:\/\/vault.centos.org\/'"$rel"'\/centosplus\/$basearch/g' /etc/yum.repos.d/CentOS-Base.repo
sed -i 's/\#baseurl=http:\/\/mirror.centos.org\/centos\/$releasever\/contrib\/$basearch\//baseurl=http:\/\/vault.centos.org\/'"$rel"'\/contrib\/$basearch/g' /etc/yum.repos.d/CentOS-Base.repo
sed -i 's/\mirror=*/\#mirrorlist=*/g' /etc/yum.repos.d/CentOS-Base.repo

Backup cpanel backups to Gdrive

#!/bin/bash
lastb=`ls -1tr /backup/|grep "\-"|tail -1`
tarb="/root/backup-${lastb}.tar"
dirb="/backup/${lastb}/"

bname='daily'
rtn=6

if [ `date '+%w'` == 0 ]; then
        bname='weekly'
        rtn=2
fi

if [ `date '+%d'` == 01 ]; then
        bname='monthly'
        rtn=2
fi


destb="/gdrive/backup/${bname}/"
#echo $destb

#echo $lastb
#ls -l /backup/|grep $lastb
tar -cf $tarb $dirb
gzip $tarb
#rsync -avz --progress backup-2019-08-01.tar.gz /gdrive/backup/
#rsync -avz ${tarb}.gz $destb
rsync -avz --progress ${tarb}.gz $destb

rm -f ${tarb}.gz
#date >> /root/backup.log
#find /$destb -type f -mtime 4 >> /root/backup.log
find $destb -type f|head -n -${rtn}|xargs rm -f
sleep 3
/usr/bin/rclone cleanup gdrive: