#!/usr/bin/python
#to get webcdn totaol statistics
# -*- coding: utf8 -*-
import MySQLdb
import sys
import os
def get_total_value(sql):
db = MySQLdb.connect(host='xxxx',user='xxxx',passwd='xxxx',db='xxxx')
cursor = db.cursor()
cursor.execute(sql)
try:
result = cursor.fetchone()[0]
except:
result = 0
cursor.close()
db.close()
return result
if __name__ == '__main__':
sql = ''
if sys.argv[1] == "network_traffic":
sql = "select round(sum(lastvalue)/(1024*1024),4) from hosts a, items b where key_ in ( 'net.if.out[eth1,bytes]','net.if.out[eth0,bytes]') and lower(host) like '%-cdn-cache%' and a.hostid = b.hostid"
elif sys.argv[1] == "nginx_traffic":
sql = "select sum(lastvalue) from hosts a, items b where key_ = 'log_webcdn_getvalue[traffic]' and lower(host) like '%cdn-cache%' and a.hostid = b.hostid"
elif sys.argv[1] == "2xxand3xx":
sql = "select sum(lastvalue) from hosts a, items b where key_ in ( 'log_webcdn_getvalue[200]','log_webcdn_getvalue[300]') and lower(host) like '%-cdn-cache%' and a.hostid = b.hostid"
elif sys.argv[1] == "4xxand5xx":
sql = "select sum(lastvalue) from hosts a, items b where key_ in ( 'log_webcdn_getvalue[four]','log_webcdn_getvalue[five]') and lower(host) like '%-cdn-cache%' and a.hostid = b.hostid"
elif sys.argv[1] == "network_ss":
sql = "select sum(lastvalue) from hosts a, items b where key_ = 'network_conn' and lower(host) like '%-cdn-cache%' and a.hostid = b.hostid"
else:
sys.exit(0)
# print sql
value = get_total_value(sql)
print value 然后是单台的分析脚本：#!/usr/bin/python
#coding=utf-8
from __future__ import division
import subprocess, signal,string
import codecs
import re
import os
import time, datetime
import sys
def show_usage():
print """
python nginx_log_wedcdn.py result_key
result_key could be:
average_bodysize, response_time, sum_count, count_success, four, 403, 404, 499, five, 500, 502, 503, 200, 300, requests_second
response_time_source, percentage_time_1, percentage_time_3, all
"""
def runCmd(command, timeout = 10):
start = datetime.datetime.now()
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
while process.poll() is None:
time.sleep(0.2)
now = datetime.datetime.now()
if (now - start).seconds > timeout:
os.kill(process.pid, signal.SIGKILL)
os.waitpid(-1, os.WNOHANG)
return None
return process.stdout.readlines()
def get_old_filename():
t = datetime.datetime.now() + datetime.timedelta(minutes = -5)
a = t.strftime('%Y-%m-%d-%H')
b = t.strftime('%M')
b = int(b)//5*5
if b < 10:
c = "0" + str(b)
else:
c = str(b)
d = "/log/nginx/old/" + a + "-%s.log.gz" % c
#print d
return d
def get_new_filename():
t = datetime.datetime.now() + datetime.timedelta(minutes = -5)
a = t.strftime('%Y-%m-%d-%H')
b = t.strftime('%M')
b = int(b)//5*5
if b < 10:
c = "0" + str(b)
else:
c = str(b)
d = "/log/nginx/old/" + a + "-%s.log" % c
#print d
return d
def get_new2_filename():
t = datetime.datetime.now() + datetime.timedelta(minutes = -5)
a = t.strftime('%Y-%m-%d-%H')
b = t.strftime('%M')
b = int(b)//5*5
if b < 10:
c = "0" + str(b)
else:
c = str(b)
d = "/log/nginx/new/" + a + "-%s.log" % c
#print d
return d
def average_flow():
flow = 0
flow1 = 0
flow_ppsucai = 0
flow_asimgs = 0
flow_static9 = 0
traffic = 0.0
traffic1 = 0.0
count = 0
count_sucai = 0
count_sucai_100 = 0
count_sucai_30_100 = 0
count_sucai_30 = 0
count_asimgs = 0
count_asimgs_100 = 0
count_asimgs_30_100 = 0
count_asimgs_30 = 0
count_static9 = 0
count_static9_100 = 0
count_static9_30_100 = 0
count_static9_30 = 0
sum_time = 0.0
sum_ppsucai_time = 0.0
sum_asimgs_time = 0.0
sum_static9_time = 0.0
sum_time_source = 0.0
count_200 = 0
count_300 = 0
count_success = 0
count_200_backup = 0
count_not_200_backup = 0
id_list_200 = [200,206]
id_list_300 = [300,301,302,303,304,305,306,307]
id_list_success = [200,206,300,301,302,303,304,305,306,307]
data_byte = 0
elapsed = 0.0
response_time = 0.0
response_time_source = 0.0
requests_second = 0.0
requests_second_sucai = 0.0
requests_second_asimgs = 0.0
list_time_1 = []
list_time_3 = []
list_ip_403 = []
list_ip_404 = []
list_ip_415 = []
list_ip_499 = []
list_ip_500 = []
list_ip_502 = []
list_ip_503 = []
server_list = ['"127.0.0.1:8080"','"127.0.0.1:8081"','"-"']
file_name = get_old_filename()
if os.path.isfile("%s" % file_name):
Writelog(file_name)
i = os.popen("/bin/zcat %s" % file_name).readlines()
#i = gzip.GzipFile("%s" % file_name).readlines()
else:
file_name = get_new_filename()
if os.path.isfile("%s" % file_name):
Writelog(file_name)
i = os.popen("/bin/cat %s" % file_name).readlines()
else:
#time.sleep(15)
file_name = get_new2_filename()
if os.path.isfile("%s" % file_name):
Writelog(file_name)
i = os.popen("/bin/cat %s" % file_name).readlines()
else:
os.popen("rm -f /tmp/exist.txt")
sys.exit(1)
for line in i:
count += 1
try:
domain_name = line.split()[1]
except:
pass
try:
web_code = int(line.split()[8])
except:
web_code = 888
try:
IP = str(line.split()[0])
except:
pass
try:
data_byte = int(line.split()[9])
#print "data", data_byte
except:
data_byte = 0.0001
try:
elapsed = float(line.split()[-1].strip('"'))
if elapsed == 0.000:
elapsed = 0.0001
except:
elapsed = 0.0001
try:
time_source = float(line.split()[-4].strip('"'))
except:
time_source = 0.0
try:
backup_server = str(line.split()[-3])
except:
pass
flow1 += data_byte
if web_code in id_list_success:
flow += data_byte
sum_time_source += time_source
if domain_name != "ppsucai.pptv.com":
sum_time += elapsed
else:
#print domain_name
sum_time += 0.000
if web_code in id_list_200:
#print web_code
count_200 += 1
if backup_server not in server_list:
#print web_code, backup_server
count_200_backup += 1
elif web_code == 200 and date_byte == 0:
#print line.split()[3].lstrip("[")
WriteURLInfo(line.split()[3].lstrip("["))
WriteURLInfo("\t")
WriteURLInfo(line.split()[10])
WriteURLInfo("\n")
elif web_code in id_list_300:
count_300 += 1
elif web_code == 403 and IP not in list_ip_403:
list_ip_403.append(IP)
#print "this is the sum 403 count:", IP, len(list_ip_403)
elif web_code == 404 and IP not in list_ip_404:
list_ip_404.append(IP)
#print "this is the sum 404 count:", IP, len(list_ip_404)
elif web_code == 415 and IP not in list_ip_415:
list_ip_415.append(IP)
#print "this is the sum 415 count:", IP, len(list_ip_415)
elif web_code == 499 and IP not in list_ip_499:
list_ip_499.append(IP)
#print "this is the sum 499 count:", IP, len(list_ip_499)
elif web_code == 500 and IP not in list_ip_500:
list_ip_500.append(IP)
#print "this is the sum 500 count:", IP, len(list_ip_500)
elif web_code == 502 and IP not in list_ip_502:
list_ip_502.append(IP)
#print "this is the sum 502 count:", IP, len(list_ip_502)
elif web_code == 503 and IP not in list_ip_503:
list_ip_503.append(IP)
#print "this is the sum 503 count:", IP, len(list_ip_503)
if web_code not in id_list_200 and backup_server not in server_list:
#print web_code, backup_server
count_not_200_backup += 1
if elapsed > 1.0 and web_code in id_list_success and IP not in list_time_1:
list_time_1.append(IP)
elif elapsed > 3.0 and web_code in id_list_success and IP not in list_time_3:
list_time_3.append(IP)
if domain_name == "ppsucai.pptv.com" and web_code in id_list_success:
download_speed_sucai = round(data_byte / elapsed / 1024, 2)
flow_ppsucai += data_byte
sum_ppsucai_time += elapsed
count_sucai += 1
if download_speed_sucai >= 100:
count_sucai_100 += 1
elif download_speed_sucai < 100 and download_speed_sucai >= 30:
count_sucai_30_100 += 1
else:
count_sucai_30 += 1
elif domain_name == "asimgs.pplive.cn" and web_code in id_list_success:
download_speed_asimgs = round(data_byte / elapsed / 1024, 2)
flow_asimgs += data_byte
sum_asimgs_time += elapsed
count_asimgs += 1
if download_speed_asimgs >= 100:
count_asimgs_100 += 1
elif download_speed_asimgs < 100 and download_speed_asimgs >= 30:
count_asimgs_30_100 += 1
else:
count_asimgs_30 += 1
elif domain_name == "static9.pplive.cn" and web_code in id_list_success:
download_speed_static9 = round(data_byte / elapsed / 1024, 2)
flow_static9 += data_byte
sum_static9_time += elapsed
count_static9 += 1
if download_speed_static9 >= 100:
count_static9_100 += 1
elif download_speed_static9 < 100 and download_speed_static9 >= 30:
count_static9_30_100 += 1
else:
count_static9_30 += 1
#else:
#break
try:
traffic = round((flow*1.07*8)/300/1024/1024, 2)
#traffic1 = round((flow1*1.07)/300/1024/1024, 2)
#print traffic, traffic1
#traffic1 = round(flow/sum_time/1024/1024, 2)
count_success = count_200 + count_300
response_time = round(sum_time/count_success, 2)
response_time_source = round(sum_time_source/count_success, 2)
requests_second = round(count_success/300, 2)
if sum_ppsucai_time == 0.0:
sum_ppsucai_time = 0.0001
if sum_asimgs_time == 0.0:
sum_asimgs_time = 0.0001
#print sum_static9_time
if sum_static9_time == 0.0:
sum_static9_time = 0.0001
traffic_ppsucai = round(flow_ppsucai/sum_ppsucai_time/1024, 2)
traffic_asimgs = round(flow_asimgs/sum_asimgs_time/1024, 2)
traffic_static9 = round(flow_static9/sum_static9_time/1024, 2)
#print "flow_static:", flow_static9, "traffic_static9", traffic_static9
average_bodysize = round((flow/count_success)/1024, 2)
percentage_time_1 = round(len(list_time_1)/count_success*100, 2)
percentage_time_3 = round(len(list_time_3)/count_success*100, 2)
if count_sucai == 0:
count_sucai = 0.0001
percentage_sucai_100 = round(count_sucai_100/count_sucai*100, 2)
percentage_sucai_30_100 = round(count_sucai_30_100/count_sucai*100, 2)
percentage_sucai_30 = round(count_sucai_30/count_sucai*100, 2)
if count_asimgs == 0:
count_asimgs = 0.0001
percentage_asimgs_100 = round(count_asimgs_100/count_asimgs*100, 2)
percentage_asimgs_30_100 = round(count_asimgs_30_100/count_asimgs*100, 2)
percentage_asimgs_30 = round(count_asimgs_30/count_asimgs*100, 2)
#print count_static9
if count_static9 == 0:
count_static9 = 0.0001
percentage_static9_100 = round(count_static9_100/count_static9*100, 2)
#print count_static9_100, "100", percentage_static9_100
percentage_static9_30_100 = round(count_static9_30_100/count_static9*100, 2)
#print count_static9_30_100, "30-100", percentage_static9_30_100
percentage_static9_30 = round(count_static9_30/count_static9*100, 2)
#print count_static9_30, "30", percentage_static9_30
requests_second_sucai = round(count_sucai/300, 2)
requests_second_asimgs = round(count_asimgs/300, 2)
requests_second_static9 = round(count_static9/300, 2)
#print requests_second_static9
#print count, "this is the count of 2xx_backup:", count_200_backup,"%", round(count_200_backup/count, 4),"this is the count of !2xx_backup:", count_not_200_backup, round(count_not_200_backup/count, 4)
percentage_200_backup = round(count_200_backup/count*100, 2)
percentage_not_200_backup = round(count_not_200_backup/count*100, 2)
return average_bodysize, response_time, count, count_success, len(list_ip_403), len(list_ip_404), len(list_ip_499), len(list_ip_500), len(list_ip_502), len(list_ip_503), count_200, count_300, requests_second, response_time_source, len(list_time_1), len(list_time_3), percentage_time_1, percentage_time_3,count_sucai,percentage_sucai_100, percentage_sucai_30_100, percentage_sucai_30, requests_second_sucai, count_asimgs, percentage_asimgs_100, percentage_asimgs_30_100, percentage_asimgs_30, requests_second_asimgs, traffic_ppsucai, traffic_asimgs, traffic, traffic_static9, count_static9, percentage_static9_100, percentage_static9_30_100, percentage_static9_30, requests_second_static9, percentage_200_backup, percentage_not_200_backup, len(list_ip_415)
except:
return 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
def log_files(pwd):
log_file_list = []
files = os.popen("ls %s" % pwd).readlines()
for x in files:
if x.strip().endswith("log"):
log_file_list.append(x.strip())
return log_file_list
def result_dic():
list = average_flow()
#print list
# print list
result = {}
result['average_bodysize'] = list[0]
result['response_time'] = list[1]
result['sum_count'] = list[2]
result['count_success'] = list[3]
result['four'] = list[4] + list[5] + list[6] + list[39]
# print 'four','=','%s' % list[4],'+','%s' % list[5],'+','%s' % list[6],'+','%s' % list[39], result['four']
result['403'] = list[4]
# print '403', result['403']
result['404'] = list[5]
# print '404', result['404']
result['499'] = list[6]
# print '499', result['499']
result['415'] = list[39]
# print '415', result['415']
result['five'] = list[7] + list[8] + list[9]
result['500'] = list[7]
result['502'] = list[8]
result['503'] = list[9]
result['200'] = list[10]
result['300'] = list[11]
result['requests_second'] = list[12]
result['response_time_source'] = list[13]
result['percentage_time_1'] = list[16]
result['percentage_time_3'] = list[17]
result['count_sucai'] = list[18]
result['percentage_sucai_100'] = list[19]
result['percentage_sucai_30_100'] = list[20]
result['percentage_sucai_30'] = list[21]
result['requests_second_sucai'] = list[22]
result['count_asimgs'] = list[23]
result['percentage_asimgs_100'] = list[24]
result['percentage_asimgs_30_100'] = list[25]
result['percentage_asimgs_30'] = list[26]
result['requests_second_asimgs'] = list[27]
result['traffic_ppsucai'] = list[28]
result['traffic_asimgs'] = list[29]
result['traffic'] = list[30]
result['traffic_static9'] = list[31]
result['count_static9'] = list[32]
result['percentage_static9_100'] = list[33]
result['percentage_static9_30_100'] = list[34]
result['percentage_static9_30'] = list[35]
result['requests_second_static9'] = list[36]
result['percentage_200_backup'] = list[37]
result['percentage_not_200_backup'] = list[38]
result['all'] = list
return result
def Writelog(msg):
o = open("/log/nginx/qos_result_new"+".log","aw")
o.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + ":" + msg + "\n")
o.close()
def WriteTmpInfo(msg):
o = open("/tmp/webcdnqos_result"+".txt","aw+")
o.write(msg+"\n")
o.close()
def WriteURLInfo(msg):
today = datetime.date.today()
o = open("/tmp/webcdnqos_url_%s" % today.strftime('%Y-%m-%d') + ".log","aw")
# o.write(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + " " +msg+"\n")
o.write(msg)
o.close()
if __name__ == "__main__":
if len(sys.argv) < 2:
show_usage()
os.popen("rm -f /tmp/exist.txt")
sys.exit(1)
else:
if os.path.isfile("/tmp/exist.txt"):
sys.exit(1)
else:
os.popen("echo 'hello' > /tmp/exist.txt")
result_key = sys.argv[1]
status = result_dic()
os.popen(">/tmp/webcdnqos_result.txt")
print status[result_key]
Writelog(str(status[result_key]))
for i in status.keys():
WriteTmpInfo(str(i)+"="+str(status[i]))
os.popen("rm -f /tmp/exist.txt")我正在寻找执行以下操作的正确语法(在Perl、Shell或Ruby中):#variabletoaccessthedatalinesappendedasafileEND_OF_SCRIPT_MARKERrawdatastartshereanditcontinues. 最佳答案 Perl用__DATA__做这个:#!/usr/bin/perlusestrict;usewarnings;while(){print;}__DATA__Texttoprintgoeshere 关于ruby-如何将脚
我有一个在Linux服务器上运行的ruby脚本。它不使用rails或任何东西。它基本上是一个命令行ruby脚本，可以像这样传递参数:./ruby_script.rbarg1arg2如何将参数抽象到配置文件(例如yaml文件或其他文件)中？您能否举例说明如何做到这一点？提前谢谢你。 最佳答案 首先，您可以运行一个写入YAML配置文件的独立脚本:require"yaml"File.write("path_to_yaml_file",[arg1,arg2].to_yaml)然后，在您的应用中阅读它:require"yaml"arg
//1.验证返回状态码是否是200pm.test("Statuscodeis200",function(){pm.response.to.have.status(200);});//2.验证返回body内是否含有某个值pm.test("Bodymatchesstring",function(){pm.expect(pm.response.text()).to.include("string_you_want_to_search");});//3.验证某个返回值是否是100pm.test("Yourtestname",function(){varjsonData=pm.response.json
有没有一种简单的方法可以判断ruby脚本是否已经在运行，然后适本地处理它？例如:我有一个名为really_long_script.rb的脚本。我让它每5分钟运行一次。当它运行时，我想看看之前运行的是否还在运行，然后停止第二个脚本的执行。有什么想法吗？ 最佳答案 ps是一种非常糟糕的方法，并且可能会出现竞争条件。传统的Unix/Linux方法是将PID写入文件(通常在/var/run中)并在启动时检查该文件是否存在。例如pid文件位于/var/run/myscript.pid然后你会在运行程序之前检查它是否存在。有一些技巧可以避免
我正在开发一个Ruby脚本，需要在没有Ruby解释器的情况下部署到系统上。它将需要在使用ELF格式的FreeBSD系统上运行。我知道有一个ruby2exe项目可以编译在Windows上运行的ruby脚本，但是在其他操作系统上这样做容易吗？甚至可能吗？ 最佳答案 您是否检查过Rubinius或JRuby是否允许您预编译您的代码？ 关于ruby-ruby脚本可以预编译成二进制文件吗？，我们在StackOverflow上找到一个类似的问题： https://
我目前可以将stdout重定向到ruby/rails中的字符串变量，只需在bash中运行命令并将结果设置为我的字符串变量，如下所示。val=%x[#{cmd}]其中cmd是表示bash命令的字符串。但是，这仅捕获stdout，因为我想捕获stderr并将其设置为ruby中的字符串——有什么想法吗？ 最佳答案 简单地重定向它:val=%x[#{cmd}2>&1]如果您只想从stderr捕获输出，请在将其复制到fd2后关闭stdout的文件描述符。val=%x[#{cmd}2>&1>/dev/null]
我们如何从ruby脚本返回值？#!/usr/bin/envrubya="test"a我们如何在Ubuntu终端或java或c中访问'a'的值？ 最佳答案 在ruby/python脚本中打印你的变量，然后可以通过示例从shell脚本中读取它:#!/bin/bashruby_var=$(rubymyrubyscript.rb)python_var=$(pythonmypythonscript.py)echo"$ruby_var"echo"$python_var"注意你的ruby/python脚本只打印这个变量(有更多复杂的方
我在跑Fastlane(适用于iOS的持续构建工具)以执行用于解密文件的自定义shell脚本。这是命令。sh"./decrypt.shENV['ENCRYPTION_P12']"我想不出将环境变量传递给该脚本的方法。显然，如果我将密码硬编码到脚本中，它就可以正常工作。sh"./decrypt.shmypwd"有什么建议吗？ 最佳答案 从直接Shell中扩展假设这里的sh是一个faSTLane命令，它以给定的参数作为脚本文本调用shell命令:#asafastlanedirectivesh'./decrypt.sh"$ENCRYPTI
我希望这些值匹配。当shell脚本由于某些错误条件而退出时(因此返回非零值)，它们不匹配。壳$?返回1，ruby$？返回256。>>%x[lskkr]ls:kkr:Nosuchfileordirectory=>"">>puts$?256=>nil>>exitHadoop:~Madcap$lskkrls:kkr:NosuchfileordirectoryHadoop:~Madcap$echo$?1 最佳答案 在Ruby中$?是一个Process::Status实例。打印$?等同于调用$?.to_s，这等同于$?.to_i.to_s(来
我有一个Ruby文件，我将它作为rubyfile.rb"parameters"运行。我更喜欢将它作为regtask参数运行，而不必每次都包含ruby和文件名。我希望它与ls处于同一级别。我将如何做到这一点？ 最佳答案 编辑你的文件，确保这是第一行，这样你的系统就知道如何执行你的文件:#!/usr/bin/envruby接下来，更改文件的权限以使其可执行:chmoda+xfile.rb最后，重命名并将其移动到将要执行的位置，而无需编写其完整路径:mkdir-p~/binmvfile.rb~/bin/regtask(如果~/bin存在，