shell脚本并行模板

shell脚本并行模板

QQ群:397745473

shell脚本并行模板

1
2
3
4
5
参考: 
https://www.huaweicloud.com/zhishi/arc-3376272.html
https://ld246.com/article/1519530562281
http://ask.apelearn.com/m/question/17659
https://www.cnblogs.com/maxgongzuo/p/6414376.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
#!/bin/bash
ip=`cat iplist.txt|grep -v "#"|awk '{print $1}'`
dir='/usr/local/src'
answer="yes" #定义yes/no应答变量
passwd="123456" #服务器密码

thead_num=50
tmp_fifo_file="/tmp/$$.fifo"
mkfifo $tmp_fifo_file
exec 4<>$tmp_fifo_file
rm -f $tmp_fifo_file

for ((i=0;i<$thead_num;i++))>&4

for i in $ip
do read -u4 { expect <&4 }&
done
wait
exec 4>&-
exit 0

示例:使用 3 个并发,打印出 0-9 10 个数字,并且打印完之后会 sleep 1 秒.

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
#!/bin/sh

# 用户可定义的常量, 3个并发
THREAD=3

# 不需要改变的常量
TMPFILE=$$.fifo

# 公共代码开始
mkfifo ${TMPFILE}
exec 7<>${TMPFILE}
rm -f ${TMPFILE}


for (( i = 0; i < ${THREAD}; i++ ))
do
echo >&7
done


# 业务代码开始

function fun(){
echo $i
sleep 1
}

for(( i = 0; i < 10; i++ ))
do
read -u7
{
fun
echo >&7
} &
done

wait

# 关闭通道
exec 7>&-

echo "finished"

很有实战意义:

特点:

$(mktemp) # 建立一个随机的不重名的临时文件

for ((idx=0;idx<$PROCESS_NUM;idx++));
do
echo
done >&6

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
#!/bin/bash
####### env ########
first_taskid=1
PROCESS_NUM=3


####### func ########
function do_process {
local taskid=$2
[ $taskid -eq 1 ] && local gpuid=0
[ $taskid -eq 2 ] && local gpuid=1
[ $taskid -eq 3 ] && local gpuid=3
sh tran.sh $1 $taskid $gpuid &
hang $1 $taskid $gpuid
# wait
# echo >&6
}

function hang {
while true
do
sleep 1800
ps -ef|grep "sh tran.sh $1 $2 $3" |grep -v grep
[ $? -eq 0 ] && continue || break
done
echo >&6
}


######## fifo ########
FIFO_FILE=$(mktemp) # 建立一个随机的不重名的临时文件
rm $FIFO_FILE
mkfifo $FIFO_FILE
trap "rm $FIFO_FILE" 15
trap "rm $FIFO_FILE" 9
exec 6<>$FIFO_FILE

for ((idx=0;idx<$PROCESS_NUM;idx++));
do
echo
done >&6


######## main ##########
for i in `cat tmp.txt`
do
read -u6
taskid=$(( $first_taskid % $PROCESS_NUM ))
[ $taskid -eq 0 ] && taskid=$PROCESS_NUM
do_process $i $taskid &
((first_taskid++))
done


######## end #########
exec 6>&- # 关闭fd6
rm $FIFO_FILE

QQ群:397745473

知识星球