【移动应用开发技术】快网CDN怎么样统计脚本

【移动应用开发技术】快网CDN怎么样统计脚本

公司使用的快网的CDN，但是快网本身并不提供日志的分析统计，我这里还没有用elk，无奈只能先使用shell对日志进行分析统计，作为运维组对业务情况的一个大致的了解。脚本是对前一天的CDNLOG进行解压缩分析，最后将结果按类输出到指定目录，再将这些结果打包扔回CDNLOG目录，因为CDNLOG目录是要上传云存储的，一并保存。其实还有一个动作，是将这些内容作为参数传给python脚本给一个监控系统，出图便于查看，这里没有贴上。同样作为留底，不喜勿喷。#!/bin/bash

function

line()

{

cols=`tput

cols`

for

t

in

`seq

1

${cols}`

do

echo

-n

"="

done

}

function

global_define()

{

yesterday=`date

-d

"-1day"

+%F`

comp_path="/srv/cdnlog/downloads/${yesterday}/"

cdnlog_list=`ls

${comp_path}

|

sed

's#.gz##g'`

uncomp_path="/srv/cdnlog/uncomp/${yesterday}/"

user=`whoami`

backup_path="/srv/cdnlog/downloads/${yesterday}/"

}

function

global_check()

{

if

[

$user

!=

root

];then

echo

"Please

use

root

."

exit

1

fi

}

function

cycle()

{

for

n

in

$cdnlog_list

do

check

define

uncomp

segmentation

source_address

method

http_code

http_size

referer

retime

platform

tag

back

cdn_node

response_size

delete

done

}

function

check()

{

if

[

-d

${uncomp_path}${n}/

];then

rm

-rf

${uncomp_path}${n}/

fi

}

function

define()

{

gz_file="${n}.gz"

log_file="${n}.log"

log_path="/srv/cdnlog/uncomp/${yesterday}/${n}/"

}

function

uncomp()

{

mkdir

-p

${log_path}

gunzip

-c

${comp_path}${gz_file}

>

${log_path}${log_file}

}

function

segmentation()

{

log_file_count=`cat

${log_path}${log_file}

|

wc

-l`

awk

-vsour_addr=${log_path}sour_addr.tmp

-vmethod=${log_path}method.tmp

-vhttp_code=${log_path}http_code.tmp

-vhttp_size=${log_path}http_size.tmp

-vreferer=${log_path}referer.tmp

-vplatform=${log_path}platform.tmp

-vcdn_node=${log_path}cdn_node.tmp

-vresponse_size=${log_path}response_size.tmp

'{print

$1

>

sour_addr

;print

$6,$7

>

method

;print

$9

>

http_code

;print

$10

>

http_size

;print

$11

>

referer

;print

$(NF)

>

cdn_node

;print

$(NF-1)

>

response_size}'

${log_path}${log_file}

cat

${log_path}${log_file}

|

egrep

"FCACHE_HIT_DISK|FCACHE_HIT|FCACHE_MISS"

|

awk

-F

'FCACHE_HIT_DISK|FCACHE_HIT|FCACHE_MISS'

'{print

$2}'

>

${log_path}more.tmp

awk

-vretime1=${log_path}retime1.tmp

-vretime2=${log_path}retime2.tmp

-vretime3=${log_path}retime3.tmp

-vretime4=${log_path}retime4.tmp

-vretime5=${log_path}retime5.tmp

-vretime6=${log_path}retime6.tmp

-vretime7=${log_path}retime7.tmp

-vretime8=${log_path}retime8.tmp

-vretime9=${log_path}retime9.tmp

-vtag=${log_path}tag.tmp

-vback=${log_path}back.tmp

'{print

$1

>

retime1

;print

$2

>

retime2

;print

$3

>

retime3

;print

$4

>

retime4

;print

$5

>

retime5

;print

$6

>

retime6

;print

$7

>

retime7

;print

$8

>

retime8

;print

$9

>

retime9

;print

$10

>

tag

;print

$12

>

back

}'

${log_path}more.tmp

cat

${log_path}${log_file}

|

awk

-F

'[(|)]'

'{print

$2}'

>

${log_path}platform.tmp

}

function

source_address()

{

line

>>

${log_path}sour_addr.log

echo

"Source

Address

Top

30

:"

>>

${log_path}sour_addr.log

echo

""

>>

${log_path}sour_addr.log

cat

${log_path}sour_addr.tmp

|

sort

-r

|

uniq

-c

|

sort

-nr

|

head

-30

>>

${log_path}sour_addr.log

sour_addr_total=`cat

${log_path}sour_addr.tmp

|

sort

-r

|

uniq

-c

|

wc

-l

`

echo

""

>>

${log_path}sour_addr.log

echo

"Source

Address

Total

:

$sour_addr_total"

>>

${log_path}sour_addr.log

line

>>

${log_path}sour_addr.log

}

function

method()

{

line

>>

${log_path}method.log

echo

"Method

&

URL

Top

30

:"

>>

${log_path}method.log

echo

""

>>

${log_path}method.log

cat

${log_path}method.tmp

|

sed

's#"##g'

|

sort

-r

|

uniq

-c

|

sort

-nr

|

head

-30

>>

${log_path}method.log

line

>>

${log_path}method.log

}

function

http_code()

{

line

>>

${log_path}http_code.log

echo

"Http

Code

Top

:"

>>

${log_path}http_code.log

echo

""

>>

${log_path}http_code.log

http_code_list=`cat

${log_path}http_code.tmp

|

sort

-r

|

uniq

-c

|

sort

-nr

|

awk

'{print

$2}'`

cat

${log_path}http_code.tmp

|

sort

-r

|

uniq

-c

|

sort

-nr

>>

${log_path}http_code.log

echo

""

>>

${log_path}http_code.log

for

y

in

$http_code_list

do

count=`cat

${log_path}http_code.log

|

awk

-vncode=$y

'$2==ncode{print

$1}'`

rate=`echo

"$count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1*100/$2)}'`

echo

-e

"HTTP

$y

Count:

\t$count"

>>

${log_path}http_code.log

echo

-e

"HTTP

$y

Rate:

\t\t${rate}%"

>>

${log_path}http_code.log

echo

""

>>

${log_path}http_code.log

done

line

>>

${log_path}http_code.log

}

function

http_size()

{

size_max=`cat

${log_path}http_size.tmp

|

sort

-nr

|

head

-1`

size_min=`cat

${log_path}http_size.tmp

|

sort

-nr

|

tail

-1`

size_total_tmp=`cat

${log_path}http_size.tmp

|

awk

'{sum+=$1}END{print

sum}'`

size_total=`echo

$size_total_tmp

|

awk

'{printf

("%.0f\n",$1)}'`

size_avg=`echo

"$size_total

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2)}'`

line

>>

${log_path}http_size.log

echo

""

>>

${log_path}http_size.log

echo

-e

"Http

Size

Max

:

\t$size_max"

>>

${log_path}http_size.log

echo

-e

"Http

Size

Min

:

\t$size_min"

>>

${log_path}http_size.log

echo

-e

"Http

Size

Total

:

\t$size_total"

>>

${log_path}http_size.log

echo

-e

"Http

Size

Avg

:

\t$size_avg"

>>

${log_path}http_size.log

line

>>

${log_path}http_size.log

}

function

referer()

{

line

>>

${log_path}referer.log

echo

"Referer

Top

30

:"

>>

${log_path}referer.log

echo

""

>>

${log_path}referer.log

cat

${log_path}referer.tmp

|

sed

's#"##g'

|

sort

-r

|

uniq

-c

|

sort

-nr

|

head

-30

>>

${log_path}referer.log

line

>>

${log_path}referer.log

}

function

retime()

{

sed

-ie

'/-/d'

${log_path}retime{1..9}.tmp

for

i

in

`seq

1

9`

do

eval

retime${i}=`cat

${log_path}retime${i}.tmp

|

wc

-l`

eval

sum${i}=`cat

${log_path}retime${i}.tmp

|

awk

'{sum+=$1}END{print

sum}'

|

awk

'{printf

("%.2f\n",$1)}'`

eval

avg${i}=`eval

echo

"\\${retime${i}}

\\${sum$i}"

|

awk

'{printf

("%.2f\n",$2/$1)}'`

eval

max${i}=`cat

${log_path}retime${i}.tmp

|

sort

-nr

|

head

-1`

eval

min${i}=`cat

${log_path}retime${i}.tmp

|

sort

-nr

|

tail

-1`

done

for

k

in

`seq

1

9`

do

line

>>

${log_path}retime${k}.log

echo

"Retime${k}

:"

>>

${log_path}retime${k}.log

echo

""

>>

${log_path}retime${k}.log

echo

-en

"Max

\t:

\t"

>>

${log_path}retime${k}.log

eval

echo

"\${max$k}"

>>

${log_path}retime${k}.log

echo

-en

"Min

\t:

\t"

>>

${log_path}retime${k}.log

eval

echo

"\${min$k}"

>>

${log_path}retime${k}.log

echo

-en

"Avg

\t:

\t"

>>

${log_path}retime${k}.log

eval

echo

"\${avg$k}"

>>

${log_path}retime${k}.log

echo

-en

"Total

:

\t"

>>

${log_path}retime${k}.log

eval

echo

"\${sum$k}"

>>

${log_path}retime${k}.log

line

>>

${log_path}retime${k}.log

done

sed

-i

's#-nan#0.00#g'

${log_path}retime{1..9}.log

}

function

platform()

{

iphone_count=`cat

${log_path}platform.tmp

|

egrep

"iPhone"

|

wc

-l`

ipad_count=`cat

${log_path}platform.tmp

|

egrep

"iPad"

|

wc

-l`

mac_count=`cat

${log_path}platform.tmp

|

egrep

"Macintosh"

|

wc

-l`

android_count=`cat

${log_path}platform.tmp

|

egrep

"Android|Adr"

|

wc

-l`

windows_count=`cat

${log_path}platform.tmp

|

egrep

"Windows

NT|Windows

98|Windows

XP"

|

wc

-l`

windows_mobile_count=`cat

${log_path}platform.tmp

|

egrep

"Windows

Phone"

|

wc

-l`

java_count=`cat

${log_path}platform.tmp

|

egrep

"java"

|

wc

-l`

linux_count=`cat

${log_path}platform.tmp

|

egrep

"X11;

Linux|linux-gnu"

|

wc

-l`

iphone_rate=`echo

"$iphone_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

ipad_rate=`echo

"$ipad_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

mac_rate=`echo

"$mac_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

android_rate=`echo

"$android_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

windows_rate=`echo

"$windows_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

windows_mobile_rate=`echo

"$windows_mobile_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

java_rate=`echo

"$java_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

linux_rate=`echo

"$linux_count

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

line

>>

${log_path}platform.log

echo

"Platform

Info

:"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"iPhone

count

\t\t:

$iphone_count"

>>

${log_path}platform.log

echo

-e

"iPhone

rate

\t\t:

${iphone_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"iPad

count

\t\t:

$ipad_count"

>>

${log_path}platform.log

echo

-e

"iPad

rate

\t\t:

${ipad_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"Mac

count

\t\t:

$mac_count"

>>

${log_path}platform.log

echo

-e

"Mac

rate

\t\t:

${mac_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"Android

count

\t\t:

$android_count"

>>

${log_path}platform.log

echo

-e

"Android

rate

\t\t:

${android_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"Windows

count

\t\t:

$windows_count"

>>

${log_path}platform.log

echo

-e

"Windows

rate

\t\t:

${windows_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"Windows

mobile

count

\t:

$windows_mobile_count"

>>

${log_path}platform.log

echo

-e

"Windows

mobile

rate

\t:

${windows_mobile_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"Java

count

\t\t:

$java_count"

>>

${log_path}platform.log

echo

-e

"Java

rate

\t\t:

${java_rate}%"

>>

${log_path}platform.log

echo

""

>>

${log_path}platform.log

echo

-e

"Linux

count

\t\t:

$linux_count"

>>

${log_path}platform.log

echo

-e

"Linux

rate

\t\t:

${linux_rate}%"

>>

${log_path}platform.log

line

>>

${log_path}platform.log

}

function

tag()

{

tag_total=`cat

${log_path}more.tmp

|

wc

-l`

tag_suc=`cat

${log_path}more.tmp

|

awk

'{print

$10}'

|

grep

"1"

|

wc

-l`

tag_fail=`cat

${log_path}more.tmp

|

awk

'{print

$10}'

|

grep

"0"

|

wc

-l`

tag_suc_rate=`echo

"$tag_suc

$tag_total"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

tag_fal_rate=`echo

"$tag_fail

$tag_total"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

line

>>

${log_path}tag.log

echo

"Tag

of

Download

Info

:"

>>

${log_path}tag.log

echo

""

>>

${log_path}tag.log

echo

-e

"Download

Success

Count

\t\t:

$tag_suc"

>>

${log_path}tag.log

echo

-e

"Download

Success

Rate

\t\t:

${tag_suc_rate}%"

>>

${log_path}tag.log

echo

-e

"Download

Fail

Count

\t\t:

$tag_fail"

>>

${log_path}tag.log

echo

-e

"Download

Fail

Rate

\t\t:

${tag_fal_rate}%"

>>

${log_path}tag.log

line

>>

${log_path}tag.log

}

function

back()

{

back_total=`cat

${log_path}more.tmp

|

awk

'{print

$12}'

|

egrep

-v

"-"|

wc

-l`

back_rate=`echo

"$back_total

$log_file_count"

|

awk

'{printf

("%.2f\n",$1/$2*100)}'`

line

>>

${log_path}back.log

echo

"Back

Source

Info

:"

>>

${log_path}back.log

echo

""

>>

${log_path}back.log

echo

-e

"Back

Source

Count

\t:

$back_total"

>>

${log_path}back.log

echo

-e

"Back

Source

Rate

\t:

${back_rate}%"

>>

${