机房UPS电池过热起火断电保护脚本

机房UPS电池过热起火断电保护脚本硬件对接配置TEMP_SENSOR_NODE=["RACK-UPS-01-BAT1-01","RACK-UPS-01-BAT1-02","RACK-UPS-01-BAT1-48","RACK-UPS-02-BAT2-01","RACK-UPS-02-BAT2-48","RACK-UPS-03-BAT3-01","RACK-UPS-03-BAT3-48"]#所有电池单节温度传感器节点IDSMOKE_SENSOR_ADDR=["UPS-BIN-01-01","UPS-BIN-01-02","UPS-BIN-02-01","UPS-BIN-02-02"]#电池仓双冗余烟感地址UPS_SNMP_CONF={"host":"0","community":"UPS-ADMIN-2024","version":"2c","port":161}#UPSSNMP通信参数AC_CONTROL_API="0/api/v1/precision_ac/control"#精密空调控制接口FIRE_CONTROL_API="0/api/v1/fire/control"#消防系统控制接口PDU_CONTROL_API="0/api/v1/pdu/control"#机柜PDU控制接口ALERT_CONTACT=["138XXXX1234","135XXXX6789","139XXXX0123"]#运维负责人、机房主管、应急专员电话DINGTALK_WEBHOOK="/robot/send?access_token=abcdef1234567890abcdef1234567890abcdef1234567890abcdef1234567890"#运维应急群机器人地址CAMERA_MAPPING={"UPS_BIN":101}#电池仓对应监控摄像头IDFACTORY_RESISTANCE=[10.2,10.1,10.3,10.2,10.1,10.3]24#每节电池出厂内阻基准值，单位mΩFACTORY_RESISTANCE=[10.2,10.1,10.3,10.2,10.1,10.3]24#每节电池出厂内阻基准值，单位mΩRACK_AC_MAPPING={"UPS01":1,"UPS02":1,"UPS03":2}#电池柜对应精密空调IDLOW_PRIORITY_CABINET=["TEST-01","TEST-02","OFFICE-01","OFFICE-02","NONCORE-01","NONCORE-02","NONCORE-03"]#分级下电优先级队列，靠前先下电REMOTE_STORAGE_PATH="/mnt/oss-backup/emergency-log/ups/"#应急日志及录像异地存储路径阈值配置SINGLE_BAT_TEMP_WARN=45#单节电池预警温度，单位℃SINGLE_BAT_TEMP_CRIT=55#单节电池应急触发温度，单位℃BIN_TEMP_CRIT=60#电池仓整体环境应急触发温度，单位℃BAT_INNER_RESISTANCE_WARN_RATE=1.5#内阻预警阈值，为出厂值倍数BAT_INNER_RESISTANCE_CRIT_RATE=2#内阻应急阈值，为出厂值倍数TEMP_RISE_RATE_CRIT=10#1分钟内温度骤升应急阈值，单位℃SMOKE_CONCENTRATION_CRIT=0.05#烟感浓度应急阈值，单位%obs/mCOLLECT_INTERVAL=10#数据采集周期，单位秒全局状态缓存LAST_TEMP_DATA={}ALERT_TRIGGERED=FalseAC_ORIGIN_SETTING={}OPERATE_LOG=[]基础工具函数defsend_phone_alert(contacts,content):formobileincontacts:requests.post("0/api/v1/alert/call",json={"mobile":mobile,"content":content})time.sleep(1)defsend_dingtalk_alert(webhook,title,content):requests.post(webhook,json={"msgtype":"markdown","markdown":{"title":title,"text":f"##{title}\n{content}\n触发时间：{time.strftime('%Y-%m-%d%H:%M:%S')}"},"at":{"atMobiles":ALERT_CONTACT,"isAtAll":True}})defrequest_sensor_data(node_id):try:res=requests.get(f"0/api/v1/sensor/temp?node_id={node_id}",timeout=2)returnres.json()exceptExceptionase:return{"code":-1,"msg":str(e)}defrequest_smoke_data(addr):try:res=requests.get(f"0/api/v1/sensor/smoke?addr={addr}",timeout=2)returnres.json()exceptExceptionase:return{"code":-1,"msg":str(e)}defrequest_ups_snmp(conf):try:res=netsnmp.snmpget(OID('..4.1.3.'),#旁路状态OID('..4.1.3.'),#电池组电压OID('..4.1.3.'),#电池剩余容量[OID(f'..4.1.3..3.{i+1}')foriinrange(144)],#144节单体内阻[OID(f'..4.1.3..3.{i+1}')foriinrange(144)],#144节单体内阻Version=conf["version"],Community=conf["community"],DestHost=conf["host"],RemotePort=conf["port"])return{"bypass_status":int(res[0]),"battery_voltage":float(res[1]),"battery_capacity":float(res[2]),"inner_resistance":[float(x)forxinres[3:]]}exceptExceptionase:return{"code":-1,"msg":str(e)}defwrite_audit_log(operate_type,data):log_content={"operate_type":operate_type,"timestamp":time.time(),"data":data}withopen(f"{REMOTE_STORAGE_PATH}audit_{time.strftime('%Y%m%d')}.log","a+",encoding="utf-8")asf:f.write(json.dumps(log_content,ensure_ascii=False)+"\n")数据采集函数defcollect_data():temp_data={}fornodeinTEMP_SENSOR_NODE:res=request_sensor_data(node)ifres.get("code")!=0:for_inrange(2):time.sleep(0.5)res=request_sensor_data(node)ifres.get("code")==0:breakifres.get("code")!=0:send_dingtalk_alert(DINGTALK_WEBHOOK,"传感器通信异常",f"电池温度传感器{node}连接失败，请排查硬件故障")continuetemp_data[node]=round(float(res.get("data",{}).get("temp",0)),1)smoke_data={}foraddrinSMOKE_SENSOR_ADDR:res=request_smoke_data(addr)ifres.get("code")==0:smoke_data[addr]=round(float(res.get("data",{}).get("concentration",0)),3)ups_data=request_ups_snmp(UPS_SNMP_CONF)returntemp_data,smoke_data,ups_data预警逻辑处理defwarn_process(temp_data,ups_data):globalAC_ORIGIN_SETTINGwarn_list=[]fornode,tempintemp_data.items():ifSINGLE_BAT_TEMP_WARN<=temp<SINGLE_BAT_TEMP_CRIT:warn_list.append(f"电池节点{node}当前温度{temp}℃，超过预警阈值{SINGLE_BAT_TEMP_WARN}℃")foridx,resistanceinenumerate(ups_data.get("inner_resistance",[])):ifresistance>=BAT_INNER_RESISTANCE_WARN_RATEFACTORY_RESISTANCE[idx]:ifresistance>=BAT_INNER_RESISTANCE_WARN_RATEFACTORY_RESISTANCE[idx]:warn_list.append(f"第{idx+1}节电池内阻{resistance}mΩ，超过出厂值{BAT_INNER_RESISTANCE_WARN_RATE}倍")ifwarn_list:send_dingtalk_alert(DINGTALK_WEBHOOK,"UPS电池预警","\n".join(warn_list))target_rack=set([node.split("-")[2]fornodeintemp_data.keys()iftemp_data[node]>=SINGLE_BAT_TEMP_WARN])forrackintarget_rack:ac_id=RACK_AC_MAPPING[rack]origin_ac_info=requests.get(f"{AC_CONTROL_API}?ac_id={ac_id}").json()["data"]AC_ORIGIN_SETTING[ac_id]={"temp":origin_ac_info["temp"],"wind_speed":origin_ac_info["wind_speed"]}requests.post(AC_CONTROL_API,json={"ac_id":ac_id,"temp":origin_ac_info["temp"]2,"wind_speed":"high"})OPERATE_LOG.append(f"{time.strftime('%Y-%m-%d%H:%M:%S')}已下调对应区域精密空调温度2℃，开大风量")time.sleep(600)new_temp_data=collect_data()[0]still_high=[nodefornode,tempinnew_temp_data.items()iftemp>=SINGLE_BAT_TEMP_WARN]ifstill_high:send_phone_alert(ALERT_CONTACT,"UPS电池二级预警，温度持续偏高，请立即前往现场排查")应急触发校验逻辑defcheck_crit_trigger(temp_data,smoke_data):globalLAST_TEMP_DATAtemp_crit=Falsefornode,tempintemp_data.items():iftemp>=SINGLE_BAT_TEMP_CRIT:his_count=0forhis_tempinLAST_TEMP_DATA.get(node,[]):ifhis_temp>=SINGLE_BAT_TEMP_CRIT:his_count+=1ifhis_count>=2:temp_crit=Truebreakbin_temp_list=[tempfornode,tempintemp_data.items()if"BIN"innode]ifbin_temp_listandmax(bin_temp_list)>=BIN_TEMP_CRIT:temp_crit=Truerise_crit=FalseifLAST_TEMP_DATA:fornode,tempintemp_data.items():ifnodeinLAST_TEMP_DATAandlen(LAST_TEMP_DATA[node])>=6:one_min_ago_temp=LAST_TEMP_DATA[node][-6]iftempone_min_ago_temp>=TEMP_RISE_RATE_CRIT:rise_crit=Truebreaksmoke_crit=Falseover_thresh_smoke=len([concforconcinsmoke_data.values()ifconc>=SMOKE_CONCENTRATION_CRIT])ifover_thresh_smoke>=2:smoke_crit=Truereturntemp_critorrise_critorsmoke_crit应急处置流程defemergency_process(temp_data,smoke_data,ups_data):globalALERT_TRIGGEREDifALERT_TRIGGERED:returnALERT_TRIGGERED=True多渠道告警推送send_phone_alert(ALERT_CONTACT,"UPS电池应急告警，已启动自动保护流程，请立即赶往机房现场")crit_info=[]fornode,tempintemp_data.items():iftemp>=SINGLE_BAT_TEMP_CRIT:crit_info.append(f"电池节点{node}温度{temp}℃")foraddr,concinsmoke_data.items():ifconc>=SMOKE_CONCENTRATION_CRIT:crit_info.append(f"烟感{addr}浓度{conc}%obs/m")send_dingtalk_alert(DINGTALK_WEBHOOK,"UPS电池应急触发","\n".join(crit_info))requests.post("0/api/v1/camera/stream/push",json={"camera_id":CAMERA_MAPPING["UPS_BIN"],"webhook":DINGTALK_WEBHOOK,"contacts":ALERT_CONTACT})requests.post("0/api/v1/camera/record/start",json={"camera_id":CAMERA_MAPPING["UPS_BIN"],"save_path":REMOTE_STORAGE_PATH})OPERATE_LOG.append(f"{time.strftime('%Y-%m-%d%H:%M:%S')}已推送现场监控流，启动录像异地存储")UPS供电回路隔离bypass_status=ups_data.get("bypass_status",0)ifbypass_status==0:netsnmp.snmpset(OID('..4.1.3.'),i=1,Version=UPS_SNMP_CONF["version"],Community=UPS_SNMP_CONF["community"],DestHost=UPS_SNMP_CONF["host"],RemotePort=UPS_SNMP_CONF["port"])time.sleep(3)new_ups_data=request_ups_snmp(UPS_SNMP_CONF)ifnew_ups_data.get("bypass_status",0)==1:netsnmp.snmpset(OID('..4.1.3.'),i=0,Version=UPS_SNMP_CONF["version"],Community=UPS_SNMP_CONF["community"],DestHost=UPS_SNMP_CONF["host"],RemotePort=UPS_SNMP_CONF["port"])send_dingtalk_alert(DINGTALK_WEBHOOK,"应急处置进展","UPS已切换至旁路供电，电池组输入空开已断开")OPERATE_LOG.append(f"{time.strftime('%Y-%m-%d%H:%M:%S')}UPS切旁路成功，电池组已隔离")else:send_dingtalk_alert(DINGTALK_WEBHOOK,"应急处置进展","旁路切换失败，启动非核心机柜分级下电")forcabinetinLOW_PRIORITY_CABINET:requests.post(PDU_CONTROL_API,json={"cabinet_id":cabinet,"operate":"off"})time.sleep(2)OPERATE_LOG.append(f"{time.strftime('%Y-%m-%d%H:%M:%S')}已断开机柜{cabinet}供电")remain_time=round((ups_data.get("battery_capacity",100)0.8)/15,1)remain_time=round((ups_data.get("battery_capacity",100)0.8)/15,1)send_phone_alert(ALERT_CONTACT,f"旁路切换失败，电池剩余供电时长约{remain_time}分钟，请立即启动应急发电机")else:netsnmp.snmpset(OID('..4.1.3.'),i=0,Version=UPS_SNMP_CONF["version"],Community=UPS_SNMP_CONF["community"],DestHost=UPS_SNMP_CONF["host"],RemotePort=UPS_SNMP_CONF["port"])send_dingtalk_alert(DINGTALK_WEBHOOK,"应急处置进展","电池组输入空开已断开")OPERATE_LOG.append(f"{time.strftime('%Y-%m-%d%H:%M:%S')}电池组已隔离")消防联动处置over_thresh_smoke=len([concforconcinsmoke_data.values()ifconc>=SMOKE_CONCENTRATION_CRIT])max_temp=max(temp_data.values())iftemp_dataelse0ifover_thresh_smoke>=2ormax_temp>=70:requests.post(FIRE_CONTROL_API,json={"zone":"UPS_BIN_ZONE","operate":"trigger_gas"})requests.post(FIRE_CONTROL_API,json={"zone":"UPS_BIN_ZONE","operate":"close_valve"})requests.post("0/api/v1/access/control",json={"door_id":"UPS_BIN_ENTRANCE","status":"always_open"})requests.post(FIRE_CONTROL_API,json={"zone":"UPS_BIN_ZONE","operate":"sound_light_alarm"})send_dingtalk_alert(DINGTALK_WEBHOOK,"消防联动进展","已启动UPS电池仓七氟丙烷灭火，区域风阀已关闭，门禁已解锁，声光告警已触发")OPERATE_LOG.append(f"{time.strftime('%Y-%m-%d%H:%M:%S')}消防联动已执行")审计日志落盘write_audit_log("UPS_EMERGENCY_TRIGGER",{"temp_data":temp_data,"smoke_data":smoke_data,"ups_data":ups_data,"operate_log":OPERATE_LOG})人工解除逻辑（运维确认误报或现场处置完成后触发）defmanual_reset():globalALERT_TRIGGERED,AC_ORIGIN_SETTING,OPERATE_LOGALERT_TRIGGERED=Falseforac_id,origin_se