first commit
This commit is contained in:
15
modules/__init__.py
Normal file
15
modules/__init__.py
Normal file
@@ -0,0 +1,15 @@
|
||||
"""
|
||||
ServerGuard 硬件检测模块
|
||||
|
||||
包含以下子模块:
|
||||
- system_info: 系统信息概览
|
||||
- cpu: CPU 检测与压力测试
|
||||
- memory: 内存检测与压力测试
|
||||
- storage: 存储设备检测
|
||||
- sensors: 电源与传感器监控
|
||||
- gpu: 显卡检测
|
||||
- log_analyzer: 日志分析
|
||||
"""
|
||||
|
||||
__version__ = "1.0.0"
|
||||
__author__ = "ServerGuard Team"
|
||||
518
modules/cpu.py
Normal file
518
modules/cpu.py
Normal file
@@ -0,0 +1,518 @@
|
||||
"""
|
||||
ServerGuard - CPU 检测与压力测试模块
|
||||
|
||||
检查 CPU 状态、温度、错误日志,并执行压力测试。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import (
|
||||
execute_command, check_command_exists, parse_key_value_output,
|
||||
safe_int, safe_float, require_root
|
||||
)
|
||||
|
||||
|
||||
def run_cpu_check(stress_test: bool = False, stress_duration: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
执行 CPU 检测。
|
||||
|
||||
Args:
|
||||
stress_test: 是否执行压力测试
|
||||
stress_duration: 压力测试持续时间(秒)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 检测结果
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"cpu_info": {},
|
||||
"temperature": {},
|
||||
"mce_errors": {},
|
||||
"load_average": {},
|
||||
"stress_test": {}
|
||||
}
|
||||
|
||||
try:
|
||||
# 获取基本信息
|
||||
result["cpu_info"] = get_cpu_details()
|
||||
|
||||
# 获取温度
|
||||
result["temperature"] = get_cpu_temperature()
|
||||
if result["temperature"].get("status") == "warning":
|
||||
result["status"] = "warning"
|
||||
|
||||
# 获取负载
|
||||
result["load_average"] = get_load_average()
|
||||
|
||||
# 检查 MCE 错误
|
||||
result["mce_errors"] = check_mce_errors()
|
||||
if result["mce_errors"].get("count", 0) > 0:
|
||||
result["status"] = "warning"
|
||||
|
||||
# 执行压力测试
|
||||
if stress_test:
|
||||
result["stress_test"] = run_cpu_stress_test(stress_duration)
|
||||
if not result["stress_test"].get("passed", False):
|
||||
result["status"] = "error"
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_cpu_details() -> Dict[str, Any]:
|
||||
"""获取 CPU 详细信息。"""
|
||||
info = {
|
||||
"model": "Unknown",
|
||||
"architecture": "Unknown",
|
||||
"cores": 0,
|
||||
"threads": 0,
|
||||
"current_frequency_mhz": 0,
|
||||
"bogomips": 0,
|
||||
"flags": []
|
||||
}
|
||||
|
||||
try:
|
||||
with open('/proc/cpuinfo', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
# 解析第一个 CPU 的信息
|
||||
cpu_sections = content.split('\n\n')
|
||||
if cpu_sections:
|
||||
first_cpu = cpu_sections[0]
|
||||
data = {}
|
||||
for line in first_cpu.split('\n'):
|
||||
if ':' in line:
|
||||
key, value = line.split(':', 1)
|
||||
data[key.strip()] = value.strip()
|
||||
|
||||
info["model"] = data.get('model name', 'Unknown')
|
||||
info["vendor"] = data.get('vendor_id', 'Unknown')
|
||||
info["architecture"] = data.get('cpu family', 'Unknown')
|
||||
info["bogomips"] = safe_float(data.get('bogomips', 0))
|
||||
|
||||
if 'flags' in data:
|
||||
info["flags"] = data['flags'].split()
|
||||
|
||||
# 统计核心数和线程数
|
||||
info["threads"] = content.count('processor\t:')
|
||||
info["cores"] = len(set(re.findall(r'physical id\t:\s*(\d+)', content)))
|
||||
if info["cores"] == 0:
|
||||
info["cores"] = info["threads"]
|
||||
|
||||
# 获取当前频率
|
||||
if os.path.exists('/proc/cpuinfo'):
|
||||
with open('/proc/cpuinfo', 'r') as f:
|
||||
for line in f:
|
||||
if 'cpu MHz' in line:
|
||||
info["current_frequency_mhz"] = safe_float(line.split(':')[1].strip())
|
||||
break
|
||||
|
||||
# 获取缩放频率信息
|
||||
freq_info = get_cpu_frequency_info()
|
||||
if freq_info:
|
||||
info["frequency_info"] = freq_info
|
||||
|
||||
except Exception as e:
|
||||
info["error"] = str(e)
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_cpu_frequency_info() -> Dict[str, Any]:
|
||||
"""获取 CPU 频率信息。"""
|
||||
info = {}
|
||||
|
||||
# 尝试从 cpufreq 获取
|
||||
cpu0_path = '/sys/devices/system/cpu/cpu0/cpufreq'
|
||||
if os.path.exists(cpu0_path):
|
||||
try:
|
||||
files = {
|
||||
"min_mhz": "scaling_min_freq",
|
||||
"max_mhz": "scaling_max_freq",
|
||||
"current_mhz": "scaling_cur_freq",
|
||||
"governor": "scaling_governor",
|
||||
"driver": "scaling_driver"
|
||||
}
|
||||
|
||||
for key, filename in files.items():
|
||||
filepath = os.path.join(cpu0_path, filename)
|
||||
if os.path.exists(filepath):
|
||||
with open(filepath, 'r') as f:
|
||||
value = f.read().strip()
|
||||
if 'freq' in filename:
|
||||
# 频率值通常以 kHz 存储
|
||||
info[key] = round(safe_int(value) / 1000, 2)
|
||||
else:
|
||||
info[key] = value
|
||||
except:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_cpu_temperature() -> Dict[str, Any]:
|
||||
"""获取 CPU 温度信息。"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"sensors": {},
|
||||
"current_c": None,
|
||||
"high_threshold_c": None,
|
||||
"critical_threshold_c": None
|
||||
}
|
||||
|
||||
temperatures = []
|
||||
|
||||
# 方法 1: 使用 sensors 命令 (lm-sensors)
|
||||
if check_command_exists('sensors'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['sensors', '-u'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
# 解析 sensors -u 输出
|
||||
current_chip = None
|
||||
current_adapter = None
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
line = line.strip()
|
||||
|
||||
# 检测芯片名称
|
||||
if line and not line.startswith('Adapter:') and not ':' in line and not line.startswith('temp'):
|
||||
current_chip = line.rstrip(':')
|
||||
result["sensors"][current_chip] = {}
|
||||
continue
|
||||
|
||||
if line.startswith('Adapter:'):
|
||||
current_adapter = line.split(':', 1)[1].strip()
|
||||
if current_chip:
|
||||
result["sensors"][current_chip]["adapter"] = current_adapter
|
||||
continue
|
||||
|
||||
# 解析温度输入值
|
||||
if 'temp' in line and '_input' in line:
|
||||
match = re.match(r'(temp\d+)_input:\s*([\d.]+)', line)
|
||||
if match:
|
||||
temp_name = match.group(1)
|
||||
temp_value = safe_float(match.group(2))
|
||||
|
||||
if current_chip:
|
||||
if temp_name not in result["sensors"][current_chip]:
|
||||
result["sensors"][current_chip][temp_name] = {}
|
||||
result["sensors"][current_chip][temp_name]["current"] = temp_value
|
||||
temperatures.append(temp_value)
|
||||
|
||||
# 解析高温阈值
|
||||
if 'temp' in line and '_max' in line:
|
||||
match = re.match(r'(temp\d+)_max:\s*([\d.]+)', line)
|
||||
if match:
|
||||
temp_name = match.group(1)
|
||||
temp_value = safe_float(match.group(2))
|
||||
if current_chip and temp_name in result["sensors"][current_chip]:
|
||||
result["sensors"][current_chip][temp_name]["high"] = temp_value
|
||||
|
||||
# 解析临界温度
|
||||
if 'temp' in line and '_crit' in line:
|
||||
match = re.match(r'(temp\d+)_crit:\s*([\d.]+)', line)
|
||||
if match:
|
||||
temp_name = match.group(1)
|
||||
temp_value = safe_float(match.group(2))
|
||||
if current_chip and temp_name in result["sensors"][current_chip]:
|
||||
result["sensors"][current_chip][temp_name]["critical"] = temp_value
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 2: 直接读取 thermal zone
|
||||
if not temperatures:
|
||||
try:
|
||||
thermal_path = '/sys/class/thermal'
|
||||
if os.path.exists(thermal_path):
|
||||
for zone in os.listdir(thermal_path):
|
||||
if zone.startswith('thermal_zone'):
|
||||
zone_path = os.path.join(thermal_path, zone)
|
||||
|
||||
# 读取类型
|
||||
type_file = os.path.join(zone_path, 'type')
|
||||
zone_type = 'unknown'
|
||||
if os.path.exists(type_file):
|
||||
with open(type_file, 'r') as f:
|
||||
zone_type = f.read().strip()
|
||||
|
||||
# 读取温度 (单位是毫摄氏度)
|
||||
temp_file = os.path.join(zone_path, 'temp')
|
||||
if os.path.exists(temp_file):
|
||||
with open(temp_file, 'r') as f:
|
||||
temp_mc = safe_int(f.read().strip())
|
||||
temp_c = temp_mc / 1000.0
|
||||
|
||||
if 'x86_pkg_temp' in zone_type or 'cpu' in zone_type.lower():
|
||||
result["sensors"][zone] = {
|
||||
"type": zone_type,
|
||||
"current": temp_c
|
||||
}
|
||||
temperatures.append(temp_c)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 3: 尝试从 hwmon 读取
|
||||
if not temperatures:
|
||||
try:
|
||||
hwmon_path = '/sys/class/hwmon'
|
||||
if os.path.exists(hwmon_path):
|
||||
for hwmon in os.listdir(hwmon_path):
|
||||
hwmon_dir = os.path.join(hwmon_path, hwmon)
|
||||
|
||||
# 读取名称
|
||||
name_file = os.path.join(hwmon_dir, 'name')
|
||||
if os.path.exists(name_file):
|
||||
with open(name_file, 'r') as f:
|
||||
name = f.read().strip()
|
||||
else:
|
||||
name = hwmon
|
||||
|
||||
# 查找温度输入
|
||||
for file in os.listdir(hwmon_dir):
|
||||
if file.startswith('temp') and file.endswith('_input'):
|
||||
temp_file = os.path.join(hwmon_dir, file)
|
||||
with open(temp_file, 'r') as f:
|
||||
temp_mc = safe_int(f.read().strip())
|
||||
temp_c = temp_mc / 1000.0
|
||||
|
||||
sensor_name = file.replace('_input', '')
|
||||
result["sensors"][f"{name}_{sensor_name}"] = {
|
||||
"current": temp_c
|
||||
}
|
||||
temperatures.append(temp_c)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 计算平均温度
|
||||
if temperatures:
|
||||
result["current_c"] = round(sum(temperatures) / len(temperatures), 1)
|
||||
result["max_c"] = round(max(temperatures), 1)
|
||||
|
||||
# 检查温度警告
|
||||
if result["max_c"] > 85:
|
||||
result["status"] = "warning"
|
||||
result["warning"] = f"CPU 温度过高: {result['max_c']}°C"
|
||||
else:
|
||||
result["status"] = "unknown"
|
||||
result["warning"] = "无法获取 CPU 温度信息"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_load_average() -> Dict[str, Any]:
|
||||
"""获取系统负载信息。"""
|
||||
result = {}
|
||||
|
||||
try:
|
||||
with open('/proc/loadavg', 'r') as f:
|
||||
load_data = f.read().strip().split()
|
||||
|
||||
if len(load_data) >= 3:
|
||||
result["1min"] = safe_float(load_data[0])
|
||||
result["5min"] = safe_float(load_data[1])
|
||||
result["15min"] = safe_float(load_data[2])
|
||||
|
||||
# 获取 CPU 核心数以计算相对负载
|
||||
num_cores = os.cpu_count() or 1
|
||||
result["cores"] = num_cores
|
||||
result["relative_1min"] = round(result["1min"] / num_cores, 2)
|
||||
result["relative_5min"] = round(result["5min"] / num_cores, 2)
|
||||
result["relative_15min"] = round(result["15min"] / num_cores, 2)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def check_mce_errors() -> Dict[str, Any]:
|
||||
"""检查 Machine Check Exception (MCE) 错误。"""
|
||||
result = {
|
||||
"count": 0,
|
||||
"errors": [],
|
||||
"status": "ok"
|
||||
}
|
||||
|
||||
# 方法 1: 检查 dmesg
|
||||
if check_command_exists('dmesg'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['dmesg'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
mce_keywords = ['Machine check events logged', 'Hardware Error', 'CMCI storm']
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
for keyword in mce_keywords:
|
||||
if keyword in line:
|
||||
result["count"] += 1
|
||||
if len(result["errors"]) < 10: # 限制错误数量
|
||||
result["errors"].append(line.strip())
|
||||
result["status"] = "warning"
|
||||
break
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 2: 检查 mcelog
|
||||
if check_command_exists('mcelog'):
|
||||
try:
|
||||
# 尝试读取 mcelog 输出
|
||||
_, stdout, _ = execute_command(
|
||||
['mcelog', '--client'],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
|
||||
if stdout.strip() and 'no machine check' not in stdout.lower():
|
||||
result["count"] += stdout.count('MCE')
|
||||
result["status"] = "warning"
|
||||
result["mcelog_available"] = True
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 3: 检查 /dev/mcelog
|
||||
if os.path.exists('/dev/mcelog'):
|
||||
result["mcelog_device"] = True
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@require_root
|
||||
def run_cpu_stress_test(duration: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
运行 CPU 压力测试。
|
||||
|
||||
Args:
|
||||
duration: 测试持续时间(秒)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 测试结果
|
||||
"""
|
||||
result = {
|
||||
"passed": False,
|
||||
"duration_seconds": duration,
|
||||
"cpu_cores": os.cpu_count() or 1,
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"max_temperature": None,
|
||||
"tool_used": None,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
# 使用 stress-ng 进行压力测试(首选)
|
||||
if check_command_exists('stress-ng'):
|
||||
result["tool_used"] = "stress-ng"
|
||||
try:
|
||||
result["start_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 获取测试前温度
|
||||
temp_before = get_cpu_temperature()
|
||||
|
||||
# 运行 stress-ng
|
||||
# --cpu 0 使用所有 CPU 核心
|
||||
# --timeout 指定超时时间
|
||||
# --metrics-brief 输出简要指标
|
||||
cmd = [
|
||||
'stress-ng',
|
||||
'--cpu', '0',
|
||||
'--timeout', str(duration),
|
||||
'--metrics-brief'
|
||||
]
|
||||
|
||||
_, stdout, stderr = execute_command(
|
||||
cmd,
|
||||
timeout=duration + 30, # 给一些额外时间
|
||||
check_returncode=False
|
||||
)
|
||||
|
||||
result["end_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 获取测试后温度
|
||||
temp_after = get_cpu_temperature()
|
||||
|
||||
# 分析输出
|
||||
output = stdout + stderr
|
||||
|
||||
# 检查是否有错误
|
||||
if 'error' in output.lower() or 'fail' in output.lower():
|
||||
result["passed"] = False
|
||||
result["errors"].append("压力测试过程中发现错误")
|
||||
else:
|
||||
result["passed"] = True
|
||||
|
||||
# 提取性能指标
|
||||
bogo_ops = re.search(r'stress-ng:\s+cpu:\s+(\d+)\s+bogo ops', output)
|
||||
if bogo_ops:
|
||||
result["bogo_ops"] = safe_int(bogo_ops.group(1))
|
||||
|
||||
bogo_ops_per_sec = re.search(r'(\d+\.\d+)\s+bogo ops per second', output)
|
||||
if bogo_ops_per_sec:
|
||||
result["bogo_ops_per_second"] = safe_float(bogo_ops_per_sec.group(1))
|
||||
|
||||
# 温度分析
|
||||
if temp_after.get("max_c"):
|
||||
result["max_temperature"] = temp_after["max_c"]
|
||||
if temp_after["max_c"] > 95:
|
||||
result["warnings"] = [f"测试期间温度过高: {temp_after['max_c']}°C"]
|
||||
|
||||
result["temperature_before"] = temp_before
|
||||
result["temperature_after"] = temp_after
|
||||
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["errors"].append(str(e))
|
||||
|
||||
# 备选: 使用 stress
|
||||
elif check_command_exists('stress'):
|
||||
result["tool_used"] = "stress"
|
||||
try:
|
||||
result["start_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
temp_before = get_cpu_temperature()
|
||||
|
||||
num_cores = os.cpu_count() or 1
|
||||
_, stdout, stderr = execute_command(
|
||||
['stress', '--cpu', str(num_cores), '--timeout', str(duration)],
|
||||
timeout=duration + 30,
|
||||
check_returncode=False
|
||||
)
|
||||
|
||||
result["end_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
temp_after = get_cpu_temperature()
|
||||
|
||||
result["passed"] = True
|
||||
result["temperature_before"] = temp_before
|
||||
result["temperature_after"] = temp_after
|
||||
|
||||
if temp_after.get("max_c"):
|
||||
result["max_temperature"] = temp_after["max_c"]
|
||||
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["errors"].append(str(e))
|
||||
|
||||
else:
|
||||
result["passed"] = False
|
||||
result["errors"].append("未找到压力测试工具 (stress-ng 或 stress)")
|
||||
result["note"] = "请安装 stress-ng 或 stress: yum install stress / apt install stress-ng"
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
print(json.dumps(run_cpu_check(stress_test=False), indent=2, ensure_ascii=False))
|
||||
497
modules/gpu.py
Normal file
497
modules/gpu.py
Normal file
@@ -0,0 +1,497 @@
|
||||
"""
|
||||
ServerGuard - 显卡检测模块
|
||||
|
||||
检测 GPU 信息、温度、驱动状态等。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import (
|
||||
execute_command, check_command_exists, parse_key_value_output,
|
||||
safe_int, safe_float, format_bytes
|
||||
)
|
||||
|
||||
|
||||
def run_gpu_check() -> Dict[str, Any]:
|
||||
"""
|
||||
执行 GPU 检测。
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 检测结果
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"gpus": [],
|
||||
"errors": []
|
||||
}
|
||||
|
||||
try:
|
||||
# 检测 NVIDIA GPU
|
||||
nvidia_gpus = check_nvidia_gpus()
|
||||
if nvidia_gpus:
|
||||
result["gpus"].extend(nvidia_gpus)
|
||||
|
||||
# 检测 AMD GPU
|
||||
amd_gpus = check_amd_gpus()
|
||||
if amd_gpus:
|
||||
result["gpus"].extend(amd_gpus)
|
||||
|
||||
# 检测 Intel GPU
|
||||
intel_gpus = check_intel_gpus()
|
||||
if intel_gpus:
|
||||
result["gpus"].extend(intel_gpus)
|
||||
|
||||
# 如果没有找到 GPU,使用 lspci 基础检测
|
||||
if not result["gpus"]:
|
||||
result["gpus"] = check_generic_gpus()
|
||||
|
||||
# 检查系统日志中的 GPU 错误
|
||||
result["dmesg_errors"] = check_gpu_dmesg_errors()
|
||||
|
||||
# 如果有错误,更新状态
|
||||
if result["dmesg_errors"]:
|
||||
result["status"] = "warning"
|
||||
|
||||
if not result["gpus"]:
|
||||
result["status"] = "unknown"
|
||||
result["note"] = "未检测到 GPU 设备"
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def check_nvidia_gpus() -> List[Dict[str, Any]]:
|
||||
"""检测 NVIDIA GPU。"""
|
||||
gpus = []
|
||||
|
||||
if not check_command_exists('nvidia-smi'):
|
||||
return gpus
|
||||
|
||||
try:
|
||||
# 获取 GPU 列表和基本信息
|
||||
_, stdout, _ = execute_command(
|
||||
['nvidia-smi', '--query-gpu=gpu_name,gpu_bus_id,pci.bus_id,driver_version,pstate,pcie.link.gen.max,pcie.link.gen.current',
|
||||
'--format=csv,noheader'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for i, line in enumerate(stdout.strip().split('\n')):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
parts = [p.strip() for p in line.split(',')]
|
||||
if len(parts) >= 4:
|
||||
gpu_info = {
|
||||
"vendor": "NVIDIA",
|
||||
"index": i,
|
||||
"name": parts[0],
|
||||
"bus_id": parts[1] if len(parts) > 1 else "unknown",
|
||||
"pci_bus_id": parts[2] if len(parts) > 2 else "unknown",
|
||||
"driver_version": parts[3],
|
||||
"pstate": parts[4] if len(parts) > 4 else "unknown",
|
||||
"pcie_max_gen": parts[5] if len(parts) > 5 else "unknown",
|
||||
"pcie_current_gen": parts[6] if len(parts) > 6 else "unknown"
|
||||
}
|
||||
|
||||
# 获取详细信息
|
||||
gpu_info.update(get_nvidia_gpu_details(i))
|
||||
gpus.append(gpu_info)
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return gpus
|
||||
|
||||
|
||||
def get_nvidia_gpu_details(gpu_index: int) -> Dict[str, Any]:
|
||||
"""获取单个 NVIDIA GPU 的详细信息。"""
|
||||
details = {}
|
||||
|
||||
try:
|
||||
# 获取温度和功耗
|
||||
_, stdout, _ = execute_command(
|
||||
['nvidia-smi', '--query-gpu=temperature.gpu,power.draw,power.limit,clocks.gr,clocks.mem,utilization.gpu,utilization.memory,memory.total,memory.used,memory.free,serial,uuid,vbios_version',
|
||||
'--format=csv,noheader,nounits', '-i', str(gpu_index)],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
parts = [p.strip() for p in stdout.split(',')]
|
||||
if len(parts) >= 10:
|
||||
details["temperature_c"] = safe_int(parts[0]) if parts[0] != '[Not Supported]' else None
|
||||
details["power_draw_w"] = safe_float(parts[1]) if parts[1] != '[Not Supported]' else None
|
||||
details["power_limit_w"] = safe_float(parts[2]) if parts[2] != '[Not Supported]' else None
|
||||
details["graphics_clock_mhz"] = safe_int(parts[3]) if parts[3] != '[Not Supported]' else None
|
||||
details["memory_clock_mhz"] = safe_int(parts[4]) if parts[4] != '[Not Supported]' else None
|
||||
details["gpu_utilization_percent"] = safe_int(parts[5]) if parts[5] != '[Not Supported]' else None
|
||||
details["memory_utilization_percent"] = safe_int(parts[6]) if parts[6] != '[Not Supported]' else None
|
||||
details["memory_total_mb"] = safe_int(parts[7]) if parts[7] != '[Not Supported]' else None
|
||||
details["memory_used_mb"] = safe_int(parts[8]) if parts[8] != '[Not Supported]' else None
|
||||
details["memory_free_mb"] = safe_int(parts[9]) if parts[9] != '[Not Supported]' else None
|
||||
|
||||
if len(parts) > 10:
|
||||
details["serial"] = parts[10] if parts[10] != '[Not Supported]' else None
|
||||
if len(parts) > 11:
|
||||
details["uuid"] = parts[11] if parts[11] != '[Not Supported]' else None
|
||||
if len(parts) > 12:
|
||||
details["vbios_version"] = parts[12] if parts[12] != '[Not Supported]' else None
|
||||
|
||||
# 获取 ECC 状态
|
||||
_, ecc_output, _ = execute_command(
|
||||
['nvidia-smi', '--query-gpu=ecc.mode.current,ecc.mode.pending,ecc.errors.corrected.volatile.total,ecc.errors.uncorrected.volatile.total',
|
||||
'--format=csv,noheader', '-i', str(gpu_index)],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
ecc_parts = [p.strip() for p in ecc_output.split(',')]
|
||||
if len(ecc_parts) >= 4:
|
||||
details["ecc_mode"] = ecc_parts[0] if ecc_parts[0] != '[Not Supported]' else None
|
||||
details["ecc_pending"] = ecc_parts[1] if ecc_parts[1] != '[Not Supported]' else None
|
||||
details["ecc_corrected_errors"] = safe_int(ecc_parts[2]) if ecc_parts[2] != '[Not Supported]' else 0
|
||||
details["ecc_uncorrected_errors"] = safe_int(ecc_parts[3]) if ecc_parts[3] != '[Not Supported]' else 0
|
||||
|
||||
# 获取进程信息
|
||||
_, proc_output, _ = execute_command(
|
||||
['nvidia-smi', 'pmon', '-s', 'um', '-c', '1', '-i', str(gpu_index)],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
|
||||
processes = []
|
||||
for line in proc_output.split('\n')[2:]: # 跳过表头
|
||||
if line.strip() and not line.startswith('#'):
|
||||
proc_parts = line.split()
|
||||
if len(proc_parts) >= 5:
|
||||
processes.append({
|
||||
"pid": proc_parts[1],
|
||||
"type": proc_parts[2],
|
||||
"sm_util": proc_parts[3],
|
||||
"mem_util": proc_parts[4]
|
||||
})
|
||||
|
||||
if processes:
|
||||
details["processes"] = processes
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return details
|
||||
|
||||
|
||||
def check_amd_gpus() -> List[Dict[str, Any]]:
|
||||
"""检测 AMD GPU。"""
|
||||
gpus = []
|
||||
|
||||
# 使用 radeontop 获取信息
|
||||
if check_command_exists('radeontop'):
|
||||
try:
|
||||
# radeontop 需要图形环境,使用 -d 参数输出到文件
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='r', suffix='.txt', delete=False) as f:
|
||||
dump_file = f.name
|
||||
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['radeontop', '-d', dump_file, '-l', '1'],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
|
||||
with open(dump_file, 'r') as f:
|
||||
output = f.read()
|
||||
|
||||
gpu_info = {"vendor": "AMD"}
|
||||
|
||||
# 解析 radeontop 输出
|
||||
for line in output.split('\n'):
|
||||
if 'GPU' in line and ':' in line:
|
||||
parts = line.split(':')
|
||||
if len(parts) == 2:
|
||||
key = parts[0].strip().lower().replace(' ', '_')
|
||||
value = parts[1].strip()
|
||||
gpu_info[key] = value
|
||||
|
||||
if gpu_info:
|
||||
gpus.append(gpu_info)
|
||||
|
||||
finally:
|
||||
if os.path.exists(dump_file):
|
||||
os.unlink(dump_file)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 尝试从 sysfs 获取 AMD GPU 信息
|
||||
try:
|
||||
for card in os.listdir('/sys/class/drm'):
|
||||
if card.startswith('card') and not card[-1].isdigit() or (card.startswith('card') and os.path.exists(f'/sys/class/drm/{card}/device/vendor')):
|
||||
vendor_path = f'/sys/class/drm/{card}/device/vendor'
|
||||
if os.path.exists(vendor_path):
|
||||
with open(vendor_path, 'r') as f:
|
||||
vendor_id = f.read().strip()
|
||||
|
||||
# AMD vendor ID 是 0x1002
|
||||
if vendor_id == '0x1002':
|
||||
gpu_info = {
|
||||
"vendor": "AMD",
|
||||
"card": card
|
||||
}
|
||||
|
||||
# 获取设备信息
|
||||
device_path = f'/sys/class/drm/{card}/device/device'
|
||||
if os.path.exists(device_path):
|
||||
with open(device_path, 'r') as f:
|
||||
gpu_info["device_id"] = f.read().strip()
|
||||
|
||||
# 获取驱动
|
||||
driver_path = f'/sys/class/drm/{card}/device/driver'
|
||||
if os.path.exists(driver_path):
|
||||
driver = os.path.basename(os.readlink(driver_path))
|
||||
gpu_info["driver"] = driver
|
||||
|
||||
# 获取温度
|
||||
temp_path = f'/sys/class/drm/{card}/device/hwmon/hwmon0/temp1_input'
|
||||
if os.path.exists(temp_path):
|
||||
with open(temp_path, 'r') as f:
|
||||
temp_mc = safe_int(f.read().strip())
|
||||
gpu_info["temperature_c"] = temp_mc / 1000.0
|
||||
|
||||
# 获取频率
|
||||
freq_path = f'/sys/class/drm/{card}/device/pp_dpm_sclk'
|
||||
if os.path.exists(freq_path):
|
||||
with open(freq_path, 'r') as f:
|
||||
gpu_info["core_clock_levels"] = f.read().strip()
|
||||
|
||||
gpus.append(gpu_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return gpus
|
||||
|
||||
|
||||
def check_intel_gpus() -> List[Dict[str, Any]]:
|
||||
"""检测 Intel GPU。"""
|
||||
gpus = []
|
||||
|
||||
# 从 sysfs 获取 Intel GPU 信息
|
||||
try:
|
||||
for card in os.listdir('/sys/class/drm'):
|
||||
if not card.startswith('card'):
|
||||
continue
|
||||
|
||||
vendor_path = f'/sys/class/drm/{card}/device/vendor'
|
||||
if not os.path.exists(vendor_path):
|
||||
continue
|
||||
|
||||
with open(vendor_path, 'r') as f:
|
||||
vendor_id = f.read().strip()
|
||||
|
||||
# Intel vendor ID 是 0x8086
|
||||
if vendor_id == '0x8086':
|
||||
gpu_info = {
|
||||
"vendor": "Intel",
|
||||
"card": card
|
||||
}
|
||||
|
||||
# 获取设备信息
|
||||
device_path = f'/sys/class/drm/{card}/device/device'
|
||||
if os.path.exists(device_path):
|
||||
with open(device_path, 'r') as f:
|
||||
gpu_info["device_id"] = f.read().strip()
|
||||
|
||||
# 获取驱动
|
||||
driver_path = f'/sys/class/drm/{card}/device/driver'
|
||||
if os.path.exists(driver_path):
|
||||
driver = os.path.basename(os.readlink(driver_path))
|
||||
gpu_info["driver"] = driver
|
||||
|
||||
# Intel GPU 通常集成,标记为集成显卡
|
||||
gpu_info["type"] = "integrated"
|
||||
|
||||
gpus.append(gpu_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return gpus
|
||||
|
||||
|
||||
def check_generic_gpus() -> List[Dict[str, Any]]:
|
||||
"""使用 lspci 进行通用 GPU 检测。"""
|
||||
gpus = []
|
||||
|
||||
if not check_command_exists('lspci'):
|
||||
return gpus
|
||||
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['lspci', '-nn'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if 'VGA' in line or '3D controller' in line or 'Display controller' in line:
|
||||
parts = line.split(': ', 1)
|
||||
if len(parts) == 2:
|
||||
bus_id = parts[0].split()[0]
|
||||
description = parts[1]
|
||||
|
||||
gpu_info = {
|
||||
"bus_id": bus_id,
|
||||
"description": description
|
||||
}
|
||||
|
||||
# 识别厂商
|
||||
desc_lower = description.lower()
|
||||
if 'nvidia' in desc_lower:
|
||||
gpu_info["vendor"] = "NVIDIA"
|
||||
elif 'amd' in desc_lower or 'ati' in desc_lower:
|
||||
gpu_info["vendor"] = "AMD"
|
||||
elif 'intel' in desc_lower:
|
||||
gpu_info["vendor"] = "Intel"
|
||||
else:
|
||||
gpu_info["vendor"] = "Unknown"
|
||||
|
||||
# 识别类型
|
||||
if 'VGA' in line:
|
||||
gpu_info["type"] = "vga"
|
||||
elif '3D controller' in line:
|
||||
gpu_info["type"] = "3d"
|
||||
elif 'Display controller' in line:
|
||||
gpu_info["type"] = "display"
|
||||
|
||||
# 获取详细信息
|
||||
try:
|
||||
_, detail, _ = execute_command(
|
||||
['lspci', '-v', '-s', bus_id],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
|
||||
# 提取驱动信息
|
||||
driver_match = re.search(r'Kernel driver in use:\s*(\S+)', detail)
|
||||
if driver_match:
|
||||
gpu_info["driver"] = driver_match.group(1)
|
||||
|
||||
# 提取模块信息
|
||||
modules_match = re.search(r'Kernel modules:\s*(.+)', detail)
|
||||
if modules_match:
|
||||
gpu_info["modules"] = modules_match.group(1).strip()
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
gpus.append(gpu_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return gpus
|
||||
|
||||
|
||||
def check_gpu_dmesg_errors() -> List[Dict[str, str]]:
|
||||
"""检查 dmesg 中的 GPU 相关错误。"""
|
||||
errors = []
|
||||
|
||||
if not check_command_exists('dmesg'):
|
||||
return errors
|
||||
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['dmesg'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
# GPU 相关错误关键词
|
||||
gpu_error_patterns = [
|
||||
r'GPU has fallen off the bus',
|
||||
r'NVRM: Xid',
|
||||
r'nvidia.*error',
|
||||
r'amdgpu.*error',
|
||||
r'i915.*error',
|
||||
r'GPU hang',
|
||||
r'ring.*timeout',
|
||||
r'Failed to load firmware',
|
||||
r'VRAM lost',
|
||||
r'gpu.*fault',
|
||||
r' thermal ',
|
||||
]
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
line_lower = line.lower()
|
||||
|
||||
# 检查是否包含 GPU 相关错误
|
||||
is_gpu_error = any(
|
||||
re.search(pattern, line, re.IGNORECASE)
|
||||
for pattern in gpu_error_patterns
|
||||
)
|
||||
|
||||
if is_gpu_error and ('error' in line_lower or 'fail' in line_lower or 'warn' in line_lower or 'Xid' in line):
|
||||
# 提取时间戳
|
||||
timestamp_match = re.match(r'\[\s*([\d.]+)\]', line)
|
||||
timestamp = timestamp_match.group(1) if timestamp_match else "unknown"
|
||||
|
||||
errors.append({
|
||||
"timestamp": timestamp,
|
||||
"message": line.strip()
|
||||
})
|
||||
|
||||
# 去重并限制数量
|
||||
seen = set()
|
||||
unique_errors = []
|
||||
for error in errors:
|
||||
msg = error["message"]
|
||||
if msg not in seen and len(unique_errors) < 20:
|
||||
seen.add(msg)
|
||||
unique_errors.append(error)
|
||||
|
||||
return unique_errors
|
||||
|
||||
except:
|
||||
return []
|
||||
|
||||
|
||||
def get_gpu_processes() -> List[Dict[str, Any]]:
|
||||
"""获取使用 GPU 的进程列表(仅 NVIDIA)。"""
|
||||
processes = []
|
||||
|
||||
if not check_command_exists('nvidia-smi'):
|
||||
return processes
|
||||
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['nvidia-smi', 'pmon', '-s', 'um', '-c', '1'],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
|
||||
lines = stdout.strip().split('\n')
|
||||
# 跳过前两行(表头)
|
||||
for line in lines[2:]:
|
||||
if line.strip() and not line.startswith('#'):
|
||||
parts = line.split()
|
||||
if len(parts) >= 8:
|
||||
processes.append({
|
||||
"gpu_index": safe_int(parts[0]),
|
||||
"pid": parts[1],
|
||||
"type": parts[2],
|
||||
"sm_util": parts[3],
|
||||
"mem_util": parts[4],
|
||||
"enc_util": parts[5],
|
||||
"dec_util": parts[6],
|
||||
"command": parts[7]
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
return processes
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
print(json.dumps(run_gpu_check(), indent=2, ensure_ascii=False))
|
||||
553
modules/log_analyzer.py
Normal file
553
modules/log_analyzer.py
Normal file
@@ -0,0 +1,553 @@
|
||||
"""
|
||||
ServerGuard - 日志分析模块
|
||||
|
||||
自动分析系统日志,查找硬件相关错误关键词。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import gzip
|
||||
from typing import Dict, Any, List, Optional
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import execute_command, check_command_exists, safe_int
|
||||
|
||||
|
||||
# 硬件错误关键词分类
|
||||
HARDWARE_ERROR_PATTERNS = {
|
||||
"cpu_errors": [
|
||||
r'Machine check events? logged',
|
||||
r'Hardware Error',
|
||||
r'CMCI storm',
|
||||
r'machine check',
|
||||
r'CPU\s*\d+.*temperature',
|
||||
r'thermal.*cpu',
|
||||
r'CPU.*throttl',
|
||||
r'core.*temp',
|
||||
r'CPU.*fault',
|
||||
r'uncorrectable',
|
||||
r'correctable.*error',
|
||||
],
|
||||
"memory_errors": [
|
||||
r'Hardware error.*memory',
|
||||
r'EDAC.*error',
|
||||
r'memory.*error',
|
||||
r'Memory.*parity',
|
||||
r'ECC.*error',
|
||||
r'ue\s+count',
|
||||
r'ce\s+count',
|
||||
r'Out of memory',
|
||||
r'oom-kill',
|
||||
r'page allocation failure',
|
||||
],
|
||||
"storage_errors": [
|
||||
r'I/O error',
|
||||
r'Buffer I/O error',
|
||||
r'blk_update_request',
|
||||
r'ata\d+.*error',
|
||||
r'SATA.*error',
|
||||
r'NVMe.*error',
|
||||
r'critical.*warning',
|
||||
r'disk error',
|
||||
r'block.*error',
|
||||
r'SMART.*failure',
|
||||
r'medium error',
|
||||
r'uncorrectable error',
|
||||
],
|
||||
"pci_errors": [
|
||||
r'PCIe.*error',
|
||||
r'pcieport.*error',
|
||||
r'PCI.*error',
|
||||
r'AER:\s*',
|
||||
r'Corrected error',
|
||||
r'Uncorrected error',
|
||||
r'Non-Fatal error',
|
||||
r'Fatal error',
|
||||
r'Unsupported Request',
|
||||
],
|
||||
"usb_errors": [
|
||||
r'usb.*error',
|
||||
r'USB.*over-current',
|
||||
r'usb.*disconnect',
|
||||
r'usb.*timeout',
|
||||
r'ehci.*error',
|
||||
r'xhci.*error',
|
||||
],
|
||||
"power_errors": [
|
||||
r'thermal.*shutdown',
|
||||
r'critical.*temperature',
|
||||
r'overheat',
|
||||
r'power.*fail',
|
||||
r'under.*voltage',
|
||||
r'over.*voltage',
|
||||
r'brownout',
|
||||
r'power.*button',
|
||||
],
|
||||
"kernel_panics": [
|
||||
r'Kernel panic',
|
||||
r'sysrq.*trigger',
|
||||
r'watchdog.*bug',
|
||||
r'softlockup',
|
||||
r'hardlockup',
|
||||
r'BUG:.*spinlock',
|
||||
r'BUG:.*scheduling',
|
||||
r'Oops:',
|
||||
r'Call Trace:',
|
||||
r'general protection fault',
|
||||
r'double fault',
|
||||
r'stack.*corruption',
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def analyze_logs() -> Dict[str, Any]:
|
||||
"""
|
||||
分析系统日志中的硬件错误。
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 分析结果
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"scan_time": datetime.now().isoformat(),
|
||||
"dmesg_analysis": {},
|
||||
"journal_analysis": {},
|
||||
"hardware_errors": {},
|
||||
"critical_events": [],
|
||||
"summary": {}
|
||||
}
|
||||
|
||||
try:
|
||||
# 分析 dmesg
|
||||
result["dmesg_analysis"] = analyze_dmesg()
|
||||
|
||||
# 分析 journalctl
|
||||
result["journal_analysis"] = analyze_journalctl()
|
||||
|
||||
# 汇总错误统计
|
||||
result["hardware_errors"] = summarize_errors(result)
|
||||
|
||||
# 识别关键事件
|
||||
result["critical_events"] = identify_critical_events(result)
|
||||
|
||||
# 生成摘要
|
||||
total_errors = sum(result["hardware_errors"].values())
|
||||
result["summary"] = {
|
||||
"total_errors_found": total_errors,
|
||||
"critical_events": len(result["critical_events"]),
|
||||
"recommend_action": total_errors > 0
|
||||
}
|
||||
|
||||
# 如果有错误,标记警告状态
|
||||
if total_errors > 0:
|
||||
result["status"] = "warning"
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def analyze_dmesg() -> Dict[str, Any]:
|
||||
"""分析 dmesg 输出。"""
|
||||
result = {
|
||||
"available": False,
|
||||
"error_counts": {key: 0 for key in HARDWARE_ERROR_PATTERNS.keys()},
|
||||
"recent_errors": [],
|
||||
"boot_errors": []
|
||||
}
|
||||
|
||||
if not check_command_exists('dmesg'):
|
||||
result["note"] = "dmesg 不可用"
|
||||
return result
|
||||
|
||||
try:
|
||||
# 获取 dmesg 输出
|
||||
_, stdout, _ = execute_command(
|
||||
['dmesg', '--time-format=iso'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
result["available"] = True
|
||||
|
||||
# 如果没有 --time-format 支持,使用标准格式
|
||||
if not stdout.strip():
|
||||
_, stdout, _ = execute_command(
|
||||
['dmesg'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
lines = stdout.split('\n')
|
||||
|
||||
# 分析每一行
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# 检查各类错误
|
||||
for error_type, patterns in HARDWARE_ERROR_PATTERNS.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, line, re.IGNORECASE):
|
||||
result["error_counts"][error_type] += 1
|
||||
|
||||
# 保存最近的一些错误
|
||||
if len(result["recent_errors"]) < 50:
|
||||
error_entry = {
|
||||
"type": error_type,
|
||||
"message": line.strip(),
|
||||
"pattern": pattern
|
||||
}
|
||||
if error_entry not in result["recent_errors"]:
|
||||
result["recent_errors"].append(error_entry)
|
||||
break
|
||||
|
||||
# 检查启动错误
|
||||
result["boot_errors"] = extract_boot_errors(lines)
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def analyze_journalctl() -> Dict[str, Any]:
|
||||
"""分析 journalctl 日志。"""
|
||||
result = {
|
||||
"available": False,
|
||||
"error_counts": {key: 0 for key in HARDWARE_ERROR_PATTERNS.keys()},
|
||||
"recent_errors": [],
|
||||
"boot_events": []
|
||||
}
|
||||
|
||||
if not check_command_exists('journalctl'):
|
||||
result["note"] = "journalctl 不可用"
|
||||
return result
|
||||
|
||||
try:
|
||||
# 获取最近 1000 行日志
|
||||
_, stdout, stderr = execute_command(
|
||||
['journalctl', '-n', '1000', '--no-pager', '-p', 'err'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
if 'No journal files were found' in stderr:
|
||||
result["note"] = "无 journal 文件"
|
||||
return result
|
||||
|
||||
result["available"] = True
|
||||
|
||||
lines = stdout.split('\n')
|
||||
|
||||
for line in lines:
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# 检查各类错误
|
||||
for error_type, patterns in HARDWARE_ERROR_PATTERNS.items():
|
||||
for pattern in patterns:
|
||||
if re.search(pattern, line, re.IGNORECASE):
|
||||
result["error_counts"][error_type] += 1
|
||||
|
||||
if len(result["recent_errors"]) < 50:
|
||||
error_entry = {
|
||||
"type": error_type,
|
||||
"message": line.strip()
|
||||
}
|
||||
if error_entry not in result["recent_errors"]:
|
||||
result["recent_errors"].append(error_entry)
|
||||
break
|
||||
|
||||
# 获取启动事件
|
||||
result["boot_events"] = get_journal_boot_events()
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def extract_boot_errors(lines: List[str]) -> List[Dict[str, str]]:
|
||||
"""提取启动过程中的错误。"""
|
||||
boot_errors = []
|
||||
in_boot = False
|
||||
|
||||
for line in lines:
|
||||
# 检测启动阶段
|
||||
if 'Linux version' in line or 'Command line:' in line:
|
||||
in_boot = True
|
||||
|
||||
if in_boot and ('error' in line.lower() or 'fail' in line.lower() or 'warn' in line.lower()):
|
||||
# 排除常见的非关键消息
|
||||
if not any(x in line.lower() for x in ['firmware', 'efi', 'acpi']):
|
||||
boot_errors.append({
|
||||
"stage": "boot",
|
||||
"message": line.strip()
|
||||
})
|
||||
|
||||
# 启动完成后停止
|
||||
if in_boot and ('systemd' in line and 'startup' in line):
|
||||
in_boot = False
|
||||
|
||||
return boot_errors[:20] # 限制数量
|
||||
|
||||
|
||||
def get_journal_boot_events() -> List[Dict[str, str]]:
|
||||
"""获取 journalctl 中的启动事件。"""
|
||||
events = []
|
||||
|
||||
try:
|
||||
# 获取当前启动的日志
|
||||
_, stdout, _ = execute_command(
|
||||
['journalctl', '-b', '0', '--no-pager', '-p', 'warning'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if 'error' in line.lower() or 'fail' in line.lower() or 'hardware' in line.lower():
|
||||
events.append({"message": line.strip()})
|
||||
|
||||
return events[:20]
|
||||
|
||||
except:
|
||||
return []
|
||||
|
||||
|
||||
def summarize_errors(analysis_result: Dict[str, Any]) -> Dict[str, int]:
|
||||
"""汇总错误统计。"""
|
||||
summary = {key: 0 for key in HARDWARE_ERROR_PATTERNS.keys()}
|
||||
|
||||
# 合并 dmesg 和 journalctl 的统计
|
||||
dmesg_counts = analysis_result.get("dmesg_analysis", {}).get("error_counts", {})
|
||||
journal_counts = analysis_result.get("journal_analysis", {}).get("error_counts", {})
|
||||
|
||||
for error_type in summary.keys():
|
||||
summary[error_type] = dmesg_counts.get(error_type, 0) + journal_counts.get(error_type, 0)
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def identify_critical_events(analysis_result: Dict[str, Any]) -> List[Dict[str, Any]]:
|
||||
"""识别需要立即关注的关键事件。"""
|
||||
critical_events = []
|
||||
|
||||
# 合并所有错误
|
||||
all_errors = []
|
||||
all_errors.extend(analysis_result.get("dmesg_analysis", {}).get("recent_errors", []))
|
||||
all_errors.extend(analysis_result.get("journal_analysis", {}).get("recent_errors", []))
|
||||
|
||||
# 定义关键错误模式
|
||||
critical_patterns = [
|
||||
(r'Kernel panic', 'kernel_panic', '内核崩溃'),
|
||||
(r'hardlockup', 'hard_lockup', 'CPU 硬死锁'),
|
||||
(r'softlockup', 'soft_lockup', 'CPU 软死锁'),
|
||||
(r'thermal.*shutdown', 'thermal_shutdown', '过热关机'),
|
||||
(r'Hardware Error', 'hardware_error', '硬件错误'),
|
||||
(r'Fatal.*PCIe', 'pcie_fatal', 'PCIe 致命错误'),
|
||||
(r'I/O error.*sector', 'disk_io_error', '磁盘 I/O 错误'),
|
||||
(r'Uncorrectable.*error', 'uncorrectable_error', '不可纠正错误'),
|
||||
(r'out of memory.*kill', 'oom_kill', 'OOM 进程杀死'),
|
||||
(r'GPU.*fallen.*bus', 'gpu_disconnect', 'GPU 断开连接'),
|
||||
]
|
||||
|
||||
for error in all_errors:
|
||||
message = error.get("message", "")
|
||||
for pattern, event_type, description in critical_patterns:
|
||||
if re.search(pattern, message, re.IGNORECASE):
|
||||
event = {
|
||||
"type": event_type,
|
||||
"description": description,
|
||||
"message": message[:200], # 限制长度
|
||||
"source": "dmesg" if error in analysis_result.get("dmesg_analysis", {}).get("recent_errors", []) else "journal"
|
||||
}
|
||||
|
||||
# 避免重复
|
||||
if event not in critical_events:
|
||||
critical_events.append(event)
|
||||
|
||||
return critical_events
|
||||
|
||||
|
||||
def get_kernel_panic_logs() -> List[Dict[str, str]]:
|
||||
"""专门查找内核崩溃信息。"""
|
||||
panics = []
|
||||
|
||||
# 检查 dmesg
|
||||
if check_command_exists('dmesg'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(['dmesg'], check_returncode=False, timeout=10)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if 'Kernel panic' in line or 'sysrq' in line.lower():
|
||||
panics.append({
|
||||
"source": "dmesg",
|
||||
"message": line.strip()
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
# 检查 journalctl
|
||||
if check_command_exists('journalctl'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['journalctl', '-k', '--no-pager', '-g', 'panic'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if 'panic' in line.lower():
|
||||
panics.append({
|
||||
"source": "journalctl",
|
||||
"message": line.strip()
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
return panics
|
||||
|
||||
|
||||
def get_hardware_error_logs() -> Dict[str, List[str]]:
|
||||
"""获取特定类型的硬件错误日志。"""
|
||||
result = {
|
||||
"mce_errors": [],
|
||||
"ecc_errors": [],
|
||||
"io_errors": [],
|
||||
"thermal_errors": []
|
||||
}
|
||||
|
||||
if check_command_exists('dmesg'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(['dmesg'], check_returncode=False, timeout=10)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
# MCE 错误
|
||||
if re.search(r'Machine check|CMCI|hardware error', line, re.IGNORECASE):
|
||||
result["mce_errors"].append(line.strip())
|
||||
|
||||
# ECC 错误
|
||||
if re.search(r'ECC|EDAC|memory error', line, re.IGNORECASE):
|
||||
result["ecc_errors"].append(line.strip())
|
||||
|
||||
# I/O 错误
|
||||
if re.search(r'I/O error|ata.*error|blk_update', line, re.IGNORECASE):
|
||||
result["io_errors"].append(line.strip())
|
||||
|
||||
# 热错误
|
||||
if re.search(r'thermal|overheat|critical temp', line, re.IGNORECASE):
|
||||
result["thermal_errors"].append(line.strip())
|
||||
except:
|
||||
pass
|
||||
|
||||
# 限制数量
|
||||
for key in result:
|
||||
result[key] = result[key][:20]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def search_logs_by_keyword(keyword: str, max_lines: int = 100) -> List[str]:
|
||||
"""
|
||||
根据关键词搜索日志。
|
||||
|
||||
Args:
|
||||
keyword: 搜索关键词
|
||||
max_lines: 最大返回行数
|
||||
|
||||
Returns:
|
||||
List[str]: 匹配的行列表
|
||||
"""
|
||||
results = []
|
||||
|
||||
# 搜索 dmesg
|
||||
if check_command_exists('dmesg'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['dmesg'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if keyword.lower() in line.lower():
|
||||
results.append(f"[dmesg] {line.strip()}")
|
||||
if len(results) >= max_lines:
|
||||
return results
|
||||
except:
|
||||
pass
|
||||
|
||||
# 搜索 journalctl
|
||||
if check_command_exists('journalctl'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['journalctl', '-n', str(max_lines * 2), '--no-pager'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if keyword.lower() in line.lower():
|
||||
results.append(f"[journal] {line.strip()}")
|
||||
if len(results) >= max_lines:
|
||||
return results
|
||||
except:
|
||||
pass
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def get_system_logs(since: Optional[str] = None, until: Optional[str] = None) -> Dict[str, Any]:
|
||||
"""
|
||||
获取系统日志。
|
||||
|
||||
Args:
|
||||
since: 开始时间 (格式: '2024-01-01 00:00:00')
|
||||
until: 结束时间
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 日志数据
|
||||
"""
|
||||
result = {
|
||||
"dmesg": "",
|
||||
"journalctl": "",
|
||||
"kern_log": ""
|
||||
}
|
||||
|
||||
# dmesg
|
||||
if check_command_exists('dmesg'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(['dmesg'], check_returncode=False, timeout=10)
|
||||
result["dmesg"] = stdout
|
||||
except:
|
||||
pass
|
||||
|
||||
# journalctl
|
||||
if check_command_exists('journalctl'):
|
||||
try:
|
||||
cmd = ['journalctl', '--no-pager', '-n', '5000']
|
||||
if since:
|
||||
cmd.extend(['--since', since])
|
||||
if until:
|
||||
cmd.extend(['--until', until])
|
||||
|
||||
_, stdout, _ = execute_command(cmd, check_returncode=False, timeout=15)
|
||||
result["journalctl"] = stdout
|
||||
except:
|
||||
pass
|
||||
|
||||
# /var/log/kern.log
|
||||
kern_log_path = '/var/log/kern.log'
|
||||
if os.path.exists(kern_log_path):
|
||||
try:
|
||||
with open(kern_log_path, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
lines = f.readlines()[-5000:] # 最后 5000 行
|
||||
result["kern_log"] = ''.join(lines)
|
||||
except:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
print(json.dumps(analyze_logs(), indent=2, ensure_ascii=False))
|
||||
577
modules/memory.py
Normal file
577
modules/memory.py
Normal file
@@ -0,0 +1,577 @@
|
||||
"""
|
||||
ServerGuard - 内存检测与压力测试模块
|
||||
|
||||
深度检测内存的读写错误和稳定性。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import (
|
||||
execute_command, check_command_exists, safe_int, safe_float,
|
||||
format_bytes, require_root
|
||||
)
|
||||
|
||||
|
||||
def run_memory_check(stress_test: bool = False, stress_duration: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
执行内存检测。
|
||||
|
||||
Args:
|
||||
stress_test: 是否执行压力测试
|
||||
stress_duration: 压力测试持续时间(秒)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 检测结果
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"summary": {},
|
||||
"dimm_info": [],
|
||||
"ecc_status": {},
|
||||
"edac_errors": {},
|
||||
"stress_test": {}
|
||||
}
|
||||
|
||||
try:
|
||||
# 获取内存摘要信息
|
||||
result["summary"] = get_memory_summary()
|
||||
|
||||
# 获取 DIMM 详细信息
|
||||
result["dimm_info"] = get_dimm_info()
|
||||
|
||||
# 检查 ECC 状态
|
||||
result["ecc_status"] = check_ecc_status()
|
||||
|
||||
# 检查 EDAC 错误
|
||||
result["edac_errors"] = check_edac_errors()
|
||||
if result["edac_errors"].get("total_errors", 0) > 0:
|
||||
result["status"] = "warning"
|
||||
|
||||
# 执行内存压力测试
|
||||
if stress_test:
|
||||
# 优先使用 memtester
|
||||
if check_command_exists('memtester'):
|
||||
result["stress_test"] = run_memtester(stress_duration)
|
||||
# 备选使用 stress-ng
|
||||
elif check_command_exists('stress-ng'):
|
||||
result["stress_test"] = run_memory_stress_ng(stress_duration)
|
||||
# 最后使用 stress
|
||||
elif check_command_exists('stress'):
|
||||
result["stress_test"] = run_memory_stress(stress_duration)
|
||||
else:
|
||||
result["stress_test"] = {
|
||||
"passed": False,
|
||||
"error": "未找到内存压力测试工具 (memtester/stress-ng/stress)"
|
||||
}
|
||||
|
||||
if not result["stress_test"].get("passed", False):
|
||||
result["status"] = "error"
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_memory_summary() -> Dict[str, Any]:
|
||||
"""获取内存摘要信息。"""
|
||||
result = {
|
||||
"total_bytes": 0,
|
||||
"total_gb": 0,
|
||||
"available_bytes": 0,
|
||||
"available_gb": 0,
|
||||
"used_bytes": 0,
|
||||
"used_gb": 0,
|
||||
"free_bytes": 0,
|
||||
"free_gb": 0,
|
||||
"buffers_bytes": 0,
|
||||
"cached_bytes": 0,
|
||||
"swap_total_bytes": 0,
|
||||
"swap_used_bytes": 0,
|
||||
"swap_free_bytes": 0
|
||||
}
|
||||
|
||||
try:
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
meminfo = f.read()
|
||||
|
||||
# 解析 meminfo
|
||||
patterns = {
|
||||
"total_bytes": r'MemTotal:\s+(\d+)',
|
||||
"free_bytes": r'MemFree:\s+(\d+)',
|
||||
"available_bytes": r'MemAvailable:\s+(\d+)',
|
||||
"buffers_bytes": r'Buffers:\s+(\d+)',
|
||||
"cached_bytes": r'Cached:\s+(\d+)',
|
||||
"swap_total_bytes": r'SwapTotal:\s+(\d+)',
|
||||
"swap_free_bytes": r'SwapFree:\s+(\d+)'
|
||||
}
|
||||
|
||||
for key, pattern in patterns.items():
|
||||
match = re.search(pattern, meminfo)
|
||||
if match:
|
||||
kb = safe_int(match.group(1))
|
||||
bytes_val = kb * 1024
|
||||
result[key] = bytes_val
|
||||
|
||||
# 同时设置 GB 版本
|
||||
gb_key = key.replace('bytes', 'gb')
|
||||
result[gb_key] = round(bytes_val / (1024**3), 2)
|
||||
|
||||
# 计算已用内存
|
||||
result["used_bytes"] = result["total_bytes"] - result["free_bytes"] - result["buffers_bytes"] - result["cached_bytes"]
|
||||
result["used_gb"] = round(result["used_bytes"] / (1024**3), 2)
|
||||
|
||||
# 计算交换空间使用情况
|
||||
result["swap_used_bytes"] = result["swap_total_bytes"] - result["swap_free_bytes"]
|
||||
result["swap_used_gb"] = round(result["swap_used_bytes"] / (1024**3), 2)
|
||||
result["swap_free_gb"] = round(result["swap_free_bytes"] / (1024**3), 2)
|
||||
|
||||
# 计算使用百分比
|
||||
if result["total_bytes"] > 0:
|
||||
result["usage_percent"] = round((result["used_bytes"] / result["total_bytes"]) * 100, 1)
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_dimm_info() -> List[Dict[str, Any]]:
|
||||
"""获取 DIMM(内存条)详细信息。"""
|
||||
dimms = []
|
||||
|
||||
if check_command_exists('dmidecode'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['dmidecode', '-t', 'memory'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
# 分割每个内存设备
|
||||
devices = stdout.split('Memory Device')
|
||||
|
||||
for device in devices[1:]: # 第一个是标题,跳过
|
||||
dimm = {}
|
||||
|
||||
# 解析各项属性
|
||||
patterns = {
|
||||
"array_handle": r'Array Handle:\s*(\S+)',
|
||||
"error_handle": r'Error Information Handle:\s*(\S+)',
|
||||
"total_width": r'Total Width:\s*(\d+)',
|
||||
"data_width": r'Data Width:\s*(\d+)',
|
||||
"size": r'Size:\s*(.*)',
|
||||
"form_factor": r'Form Factor:\s*(\S+)',
|
||||
"set": r'Set:\s*(\S+)',
|
||||
"locator": r'Locator:\s*(.+)',
|
||||
"bank_locator": r'Bank Locator:\s*(.+)',
|
||||
"type": r'Type:\s*(\S+)',
|
||||
"type_detail": r'Type Detail:\s*(.+)',
|
||||
"speed": r'Speed:\s*(.*)',
|
||||
"manufacturer": r'Manufacturer:\s*(\S+)',
|
||||
"serial_number": r'Serial Number:\s*(\S+)',
|
||||
"asset_tag": r'Asset Tag:\s*(\S+)',
|
||||
"part_number": r'Part Number:\s*(\S+)',
|
||||
"rank": r'Rank:\s*(\d+)',
|
||||
"configured_speed": r'Configured Memory Speed:\s*(.*)',
|
||||
"minimum_voltage": r'Minimum Voltage:\s*(.+)',
|
||||
"maximum_voltage": r'Maximum Voltage:\s*(.+)',
|
||||
"configured_voltage": r'Configured Voltage:\s*(.+)'
|
||||
}
|
||||
|
||||
for key, pattern in patterns.items():
|
||||
match = re.search(pattern, device, re.IGNORECASE)
|
||||
if match:
|
||||
value = match.group(1).strip()
|
||||
# 跳过无效值
|
||||
if value not in ['Not Specified', 'To be filled by O.E.M.', 'None', 'No Module Installed', 'Unknown']:
|
||||
dimm[key] = value
|
||||
|
||||
# 解析大小
|
||||
if 'size' in dimm:
|
||||
size_str = dimm['size']
|
||||
if 'MB' in size_str:
|
||||
dimm["size_mb"] = safe_int(size_str.replace('MB', '').strip())
|
||||
elif 'GB' in size_str:
|
||||
dimm["size_gb"] = safe_float(size_str.replace('GB', '').strip())
|
||||
dimm["size_mb"] = int(dimm["size_gb"] * 1024)
|
||||
elif 'No Module' in size_str:
|
||||
continue # 跳过空插槽
|
||||
|
||||
# 解析速度
|
||||
if 'speed' in dimm:
|
||||
speed_str = dimm['speed']
|
||||
if 'MT/s' in speed_str:
|
||||
dimm["speed_mts"] = safe_int(speed_str.replace('MT/s', '').strip())
|
||||
elif 'MHz' in speed_str:
|
||||
dimm["speed_mhz"] = safe_int(speed_str.replace('MHz', '').strip())
|
||||
|
||||
if dimm:
|
||||
dimms.append(dimm)
|
||||
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
return dimms
|
||||
|
||||
|
||||
def check_ecc_status() -> Dict[str, Any]:
|
||||
"""检查 ECC(错误校正码)内存状态。"""
|
||||
result = {
|
||||
"supported": False,
|
||||
"enabled": False,
|
||||
"mode": "unknown",
|
||||
"errors": 0
|
||||
}
|
||||
|
||||
# 方法 1: 检查 /proc/meminfo
|
||||
try:
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
if 'HardwareCorrupted' in content:
|
||||
result["supported"] = True
|
||||
match = re.search(r'HardwareCorrupted:\s+(\d+)\s+kB', content)
|
||||
if match:
|
||||
result["errors"] = safe_int(match.group(1))
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 2: 使用 dmidecode 检查内存类型
|
||||
if check_command_exists('dmidecode'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['dmidecode', '-t', 'memory'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
if 'ECC' in stdout or 'Error Correction' in stdout:
|
||||
result["supported"] = True
|
||||
|
||||
# 尝试提取 ECC 模式
|
||||
match = re.search(r'Error Correction Type:\s*(.+)', stdout)
|
||||
if match:
|
||||
result["mode"] = match.group(1).strip()
|
||||
result["enabled"] = result["mode"] != 'None'
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 3: 检查 EDAC
|
||||
edac_path = '/sys/devices/system/edac/mc'
|
||||
if os.path.exists(edac_path):
|
||||
result["edac_available"] = True
|
||||
try:
|
||||
# 检查每个内存控制器
|
||||
for mc in os.listdir(edac_path):
|
||||
if mc.startswith('mc'):
|
||||
mc_path = os.path.join(edac_path, mc)
|
||||
ce_file = os.path.join(mc_path, 'ce_count') # Correctable errors
|
||||
ue_file = os.path.join(mc_path, 'ue_count') # Uncorrectable errors
|
||||
|
||||
if os.path.exists(ce_file):
|
||||
with open(ce_file, 'r') as f:
|
||||
ce_count = safe_int(f.read().strip())
|
||||
result["correctable_errors"] = result.get("correctable_errors", 0) + ce_count
|
||||
|
||||
if os.path.exists(ue_file):
|
||||
with open(ue_file, 'r') as f:
|
||||
ue_count = safe_int(f.read().strip())
|
||||
result["uncorrectable_errors"] = result.get("uncorrectable_errors", 0) + ue_count
|
||||
except:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def check_edac_errors() -> Dict[str, Any]:
|
||||
"""检查 EDAC(Error Detection and Correction)错误。"""
|
||||
result = {
|
||||
"total_errors": 0,
|
||||
"correctable_errors": 0,
|
||||
"uncorrectable_errors": 0,
|
||||
"memory_controllers": []
|
||||
}
|
||||
|
||||
edac_path = '/sys/devices/system/edac/mc'
|
||||
|
||||
if not os.path.exists(edac_path):
|
||||
result["note"] = "EDAC 不可用"
|
||||
return result
|
||||
|
||||
try:
|
||||
for mc_name in os.listdir(edac_path):
|
||||
if not mc_name.startswith('mc'):
|
||||
continue
|
||||
|
||||
mc_path = os.path.join(edac_path, mc_name)
|
||||
mc_info = {"name": mc_name}
|
||||
|
||||
# 读取 CE 计数
|
||||
ce_file = os.path.join(mc_path, 'ce_count')
|
||||
if os.path.exists(ce_file):
|
||||
with open(ce_file, 'r') as f:
|
||||
ce = safe_int(f.read().strip())
|
||||
mc_info["correctable_errors"] = ce
|
||||
result["correctable_errors"] += ce
|
||||
|
||||
# 读取 UE 计数
|
||||
ue_file = os.path.join(mc_path, 'ue_count')
|
||||
if os.path.exists(ue_file):
|
||||
with open(ue_file, 'r') as f:
|
||||
ue = safe_int(f.read().strip())
|
||||
mc_info["uncorrectable_errors"] = ue
|
||||
result["uncorrectable_errors"] += ue
|
||||
|
||||
# 读取内存控制器信息
|
||||
info_files = ['mc_name', 'size_mb', 'mem_type', 'edac_mc_mode']
|
||||
for info_file in info_files:
|
||||
filepath = os.path.join(mc_path, info_file)
|
||||
if os.path.exists(filepath):
|
||||
with open(filepath, 'r') as f:
|
||||
mc_info[info_file] = f.read().strip()
|
||||
|
||||
result["memory_controllers"].append(mc_info)
|
||||
|
||||
result["total_errors"] = result["correctable_errors"] + result["uncorrectable_errors"]
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@require_root
|
||||
def run_memtester(duration: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
运行内存压力测试。
|
||||
|
||||
Args:
|
||||
duration: 测试持续时间(秒),实际 memtester 是基于大小而非时间
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 测试结果
|
||||
"""
|
||||
result = {
|
||||
"passed": False,
|
||||
"size_mb": 0,
|
||||
"iterations": 1,
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"duration_seconds": 0,
|
||||
"errors": [],
|
||||
"tests_run": []
|
||||
}
|
||||
|
||||
if not check_command_exists('memtester'):
|
||||
result["errors"].append("memtester 未安装")
|
||||
return result
|
||||
|
||||
try:
|
||||
# 计算测试内存大小
|
||||
# 留出一些内存给系统和 stress-ng 使用
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
content = f.read()
|
||||
|
||||
match = re.search(r'MemAvailable:\s+(\d+)', content)
|
||||
if match:
|
||||
available_mb = safe_int(match.group(1)) // 1024
|
||||
# 使用可用内存的 70%
|
||||
test_size_mb = max(64, int(available_mb * 0.7))
|
||||
else:
|
||||
test_size_mb = 256
|
||||
|
||||
result["size_mb"] = test_size_mb
|
||||
result["start_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
start_ts = time.time()
|
||||
|
||||
# 运行 memtester
|
||||
cmd = ['memtester', f'{test_size_mb}M', '1']
|
||||
|
||||
_, stdout, stderr = execute_command(
|
||||
cmd,
|
||||
timeout=max(300, test_size_mb), # 根据内存大小调整超时
|
||||
check_returncode=False
|
||||
)
|
||||
|
||||
result["end_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
result["duration_seconds"] = round(time.time() - start_ts, 2)
|
||||
|
||||
output = stdout + stderr
|
||||
result["raw_output"] = output[:2000] # 保存部分原始输出
|
||||
|
||||
# 分析结果
|
||||
if 'FAILURE' in output.upper():
|
||||
result["passed"] = False
|
||||
# 提取错误信息
|
||||
for line in output.split('\n'):
|
||||
if 'FAILURE' in line.upper() or 'error' in line.lower():
|
||||
result["errors"].append(line.strip())
|
||||
elif 'SUCCESS' in output.upper() or 'ok' in output.lower() or 'finished' in output.lower():
|
||||
result["passed"] = True
|
||||
else:
|
||||
# 检查是否完成所有测试
|
||||
if 'Done' in output or 'finished' in output.lower():
|
||||
result["passed"] = True
|
||||
else:
|
||||
result["passed"] = False
|
||||
result["errors"].append("测试可能未完成")
|
||||
|
||||
# 提取运行的测试
|
||||
test_names = [
|
||||
'Stuck Address', 'Random Value', 'Compare XOR',
|
||||
'Compare SUB', 'Compare MUL', 'Compare DIV',
|
||||
'Compare OR', 'Compare AND', 'Sequential Increment',
|
||||
'Solid Bits', 'Block Sequential', 'Checkerboard',
|
||||
'Bit Spread', 'Bit Flip', 'Walking Ones', 'Walking Zeroes'
|
||||
]
|
||||
|
||||
for test in test_names:
|
||||
if test in output:
|
||||
result["tests_run"].append(test)
|
||||
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["errors"].append(str(e))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@require_root
|
||||
def run_memory_stress_ng(duration: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
使用 stress-ng 进行内存压力测试。
|
||||
|
||||
Args:
|
||||
duration: 测试持续时间(秒)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 测试结果
|
||||
"""
|
||||
result = {
|
||||
"passed": False,
|
||||
"tool": "stress-ng",
|
||||
"duration_seconds": duration,
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
if not check_command_exists('stress-ng'):
|
||||
result["errors"].append("stress-ng 未安装")
|
||||
return result
|
||||
|
||||
try:
|
||||
result["start_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 运行 stress-ng 内存测试
|
||||
cmd = [
|
||||
'stress-ng',
|
||||
'--vm', '4', # 4 个 vm worker
|
||||
'--vm-bytes', '80%', # 每个 worker 使用 80% 可用内存
|
||||
'--vm-method', 'all', # 使用所有测试方法
|
||||
'--timeout', str(duration),
|
||||
'--metrics-brief'
|
||||
]
|
||||
|
||||
_, stdout, stderr = execute_command(
|
||||
cmd,
|
||||
timeout=duration + 30,
|
||||
check_returncode=False
|
||||
)
|
||||
|
||||
result["end_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
output = stdout + stderr
|
||||
|
||||
if 'error' in output.lower() or 'fail' in output.lower():
|
||||
result["passed"] = False
|
||||
else:
|
||||
result["passed"] = True
|
||||
|
||||
# 提取指标
|
||||
bogo_ops = re.search(r'stress-ng:\s+vm:\s+(\d+)\s+bogo ops', output)
|
||||
if bogo_ops:
|
||||
result["bogo_ops"] = safe_int(bogo_ops.group(1))
|
||||
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["errors"].append(str(e))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@require_root
|
||||
def run_memory_stress(duration: int = 300) -> Dict[str, Any]:
|
||||
"""
|
||||
使用 stress 进行内存压力测试(备选方案)。
|
||||
|
||||
Args:
|
||||
duration: 测试持续时间(秒)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 测试结果
|
||||
"""
|
||||
result = {
|
||||
"passed": False,
|
||||
"tool": "stress",
|
||||
"duration_seconds": duration,
|
||||
"start_time": None,
|
||||
"end_time": None,
|
||||
"workers": 4,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
if not check_command_exists('stress'):
|
||||
result["errors"].append("stress 未安装")
|
||||
return result
|
||||
|
||||
try:
|
||||
result["start_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# 运行 stress 内存测试
|
||||
# --vm: 内存分配 worker 数量
|
||||
# --vm-bytes: 每个 worker 分配的内存
|
||||
# --vm-keep: 保持内存占用
|
||||
# --timeout: 超时时间
|
||||
cmd = [
|
||||
'stress',
|
||||
'--vm', '4',
|
||||
'--vm-bytes', '80%',
|
||||
'--vm-keep',
|
||||
'--timeout', str(duration)
|
||||
]
|
||||
|
||||
_, stdout, stderr = execute_command(
|
||||
cmd,
|
||||
timeout=duration + 30,
|
||||
check_returncode=False
|
||||
)
|
||||
|
||||
result["end_time"] = time.strftime('%Y-%m-%d %H:%M:%S')
|
||||
|
||||
output = stdout + stderr
|
||||
|
||||
# stress 的成功退出码通常是 0
|
||||
# 如果有错误输出,可能是失败的
|
||||
if 'error' in output.lower() or 'fail' in output.lower():
|
||||
result["passed"] = False
|
||||
else:
|
||||
result["passed"] = True
|
||||
|
||||
except Exception as e:
|
||||
result["passed"] = False
|
||||
result["errors"].append(str(e))
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
print(json.dumps(run_memory_check(stress_test=False), indent=2, ensure_ascii=False))
|
||||
545
modules/sensors.py
Normal file
545
modules/sensors.py
Normal file
@@ -0,0 +1,545 @@
|
||||
"""
|
||||
ServerGuard - 电源与主板传感器监控模块
|
||||
|
||||
监控电源、主板传感器数据,包括温度、电压、风扇转速等。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import (
|
||||
execute_command, check_command_exists, parse_key_value_output,
|
||||
safe_int, safe_float, require_root
|
||||
)
|
||||
|
||||
|
||||
def run_sensors_check() -> Dict[str, Any]:
|
||||
"""
|
||||
执行传感器检测。
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 检测结果
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"lm_sensors": {},
|
||||
"ipmi_sensors": {},
|
||||
"thermal_zones": {},
|
||||
"power_supplies": {},
|
||||
"ipmi_sel": {}
|
||||
}
|
||||
|
||||
try:
|
||||
# 获取 lm-sensors 数据
|
||||
result["lm_sensors"] = get_lm_sensors_data()
|
||||
|
||||
# 获取 IPMI 传感器数据
|
||||
result["ipmi_sensors"] = get_ipmi_sensors_data()
|
||||
|
||||
# 获取 thermal zone 数据
|
||||
result["thermal_zones"] = get_thermal_zones()
|
||||
|
||||
# 获取电源信息
|
||||
result["power_supplies"] = get_power_supply_info()
|
||||
|
||||
# 获取 IPMI SEL 日志
|
||||
result["ipmi_sel"] = get_ipmi_sel_logs()
|
||||
|
||||
# 检查警告条件
|
||||
warnings = check_sensor_warnings(result)
|
||||
if warnings:
|
||||
result["warnings"] = warnings
|
||||
result["status"] = "warning"
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_lm_sensors_data() -> Dict[str, Any]:
|
||||
"""获取 lm-sensors 传感器数据。"""
|
||||
result = {
|
||||
"available": False,
|
||||
"chips": {}
|
||||
}
|
||||
|
||||
if not check_command_exists('sensors'):
|
||||
result["error"] = "lm-sensors 未安装"
|
||||
return result
|
||||
|
||||
try:
|
||||
# 检测传感器芯片
|
||||
_, stdout, _ = execute_command(
|
||||
['sensors', '-u'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
if not stdout.strip():
|
||||
result["error"] = "无传感器数据,可能需要运行 sensors-detect"
|
||||
return result
|
||||
|
||||
result["available"] = True
|
||||
|
||||
# 解析 sensors -u 输出
|
||||
current_chip = None
|
||||
current_adapter = None
|
||||
current_feature = None
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
line = line.rstrip()
|
||||
|
||||
if not line:
|
||||
continue
|
||||
|
||||
# 检测芯片名称行(以冒号结尾的非缩进行)
|
||||
if not line.startswith(' ') and line.endswith(':'):
|
||||
current_chip = line.rstrip(':')
|
||||
result["chips"][current_chip] = {
|
||||
"features": {}
|
||||
}
|
||||
current_feature = None
|
||||
continue
|
||||
|
||||
# 检测 Adapter 行
|
||||
if line.strip().startswith('Adapter:'):
|
||||
current_adapter = line.split(':', 1)[1].strip()
|
||||
if current_chip:
|
||||
result["chips"][current_chip]["adapter"] = current_adapter
|
||||
continue
|
||||
|
||||
# 检测功能名称行(缩进的非冒号结尾行)
|
||||
if line.startswith(' ') and not line.startswith(' ') and not line.endswith(':'):
|
||||
current_feature = line.strip().rstrip(':')
|
||||
if current_chip:
|
||||
result["chips"][current_chip]["features"][current_feature] = {}
|
||||
continue
|
||||
|
||||
# 检测属性行(四个空格缩进)
|
||||
if line.startswith(' ') and ':' in line and current_chip and current_feature:
|
||||
key_value = line.strip().split(':', 1)
|
||||
if len(key_value) == 2:
|
||||
key = key_value[0].strip()
|
||||
value_str = key_value[1].strip()
|
||||
|
||||
# 提取数值
|
||||
value_match = re.search(r'([\d.]+)', value_str)
|
||||
if value_match:
|
||||
value = safe_float(value_match.group(1))
|
||||
|
||||
feature_data = result["chips"][current_chip]["features"][current_feature]
|
||||
|
||||
# 分类存储
|
||||
if '_input' in key:
|
||||
feature_data["value"] = value
|
||||
elif '_max' in key:
|
||||
feature_data["max"] = value
|
||||
elif '_min' in key:
|
||||
feature_data["min"] = value
|
||||
elif '_crit' in key:
|
||||
feature_data["critical"] = value
|
||||
elif '_alarm' in key:
|
||||
feature_data["alarm"] = value > 0
|
||||
else:
|
||||
feature_data[key] = value
|
||||
|
||||
# 提取常用传感器的汇总数据
|
||||
result["summary"] = extract_sensor_summary(result["chips"])
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def extract_sensor_summary(chips: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""从传感器数据中提取常用指标的汇总。"""
|
||||
summary = {
|
||||
"temperatures": {},
|
||||
"voltages": {},
|
||||
"fans": {},
|
||||
"powers": {},
|
||||
"currents": {}
|
||||
}
|
||||
|
||||
for chip_name, chip_data in chips.items():
|
||||
for feature_name, feature_data in chip_data.get("features", {}).items():
|
||||
value = feature_data.get("value")
|
||||
if value is None:
|
||||
continue
|
||||
|
||||
feature_lower = feature_name.lower()
|
||||
|
||||
# 温度传感器
|
||||
if 'temp' in feature_lower or 'thermal' in feature_lower:
|
||||
# 提取传感器编号
|
||||
temp_match = re.search(r'temp(\d+)', feature_lower)
|
||||
if temp_match:
|
||||
temp_id = temp_match.group(1)
|
||||
summary["temperatures"][f"{chip_name}_temp{temp_id}"] = {
|
||||
"value": value,
|
||||
"max": feature_data.get("max"),
|
||||
"critical": feature_data.get("critical"),
|
||||
"alarm": feature_data.get("alarm", False)
|
||||
}
|
||||
|
||||
# 电压传感器
|
||||
elif 'in' in feature_lower or 'voltage' in feature_lower or 'vcc' in feature_lower:
|
||||
summary["voltages"][f"{chip_name}_{feature_name}"] = {
|
||||
"value": value,
|
||||
"min": feature_data.get("min"),
|
||||
"max": feature_data.get("max"),
|
||||
"alarm": feature_data.get("alarm", False)
|
||||
}
|
||||
|
||||
# 风扇转速
|
||||
elif 'fan' in feature_lower:
|
||||
fan_match = re.search(r'fan(\d+)', feature_lower)
|
||||
if fan_match:
|
||||
fan_id = fan_match.group(1)
|
||||
summary["fans"][f"{chip_name}_fan{fan_id}"] = {
|
||||
"rpm": value,
|
||||
"min": feature_data.get("min"),
|
||||
"alarm": feature_data.get("alarm", False)
|
||||
}
|
||||
|
||||
# 功率传感器
|
||||
elif 'power' in feature_lower or 'watt' in feature_lower:
|
||||
summary["powers"][f"{chip_name}_{feature_name}"] = {
|
||||
"value": value,
|
||||
"max": feature_data.get("max")
|
||||
}
|
||||
|
||||
# 电流传感器
|
||||
elif 'curr' in feature_lower or 'amp' in feature_lower:
|
||||
summary["currents"][f"{chip_name}_{feature_name}"] = {
|
||||
"value": value,
|
||||
"max": feature_data.get("max")
|
||||
}
|
||||
|
||||
return summary
|
||||
|
||||
|
||||
def get_ipmi_sensors_data() -> Dict[str, Any]:
|
||||
"""获取 IPMI 传感器数据。"""
|
||||
result = {
|
||||
"available": False,
|
||||
"sensors": {}
|
||||
}
|
||||
|
||||
if not check_command_exists('ipmitool'):
|
||||
result["note"] = "ipmitool 未安装"
|
||||
return result
|
||||
|
||||
try:
|
||||
# 检查 IPMI 是否可用
|
||||
_, stdout, stderr = execute_command(
|
||||
['ipmitool', 'sensor'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
if 'Could not open device' in stderr or 'Driver not found' in stderr:
|
||||
result["note"] = "IPMI 设备不可用"
|
||||
return result
|
||||
|
||||
result["available"] = True
|
||||
|
||||
# 解析传感器列表
|
||||
for line in stdout.split('\n'):
|
||||
if not line.strip() or '|' not in line:
|
||||
continue
|
||||
|
||||
parts = [p.strip() for p in line.split('|')]
|
||||
if len(parts) >= 4:
|
||||
sensor_name = parts[0]
|
||||
sensor_value = parts[1]
|
||||
sensor_unit = parts[2]
|
||||
sensor_status = parts[3]
|
||||
|
||||
result["sensors"][sensor_name] = {
|
||||
"value": sensor_value,
|
||||
"unit": sensor_unit,
|
||||
"status": sensor_status
|
||||
}
|
||||
|
||||
# 分类传感器
|
||||
result["categories"] = categorize_ipmi_sensors(result["sensors"])
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def categorize_ipmi_sensors(sensors: Dict[str, Any]) -> Dict[str, Dict[str, Any]]:
|
||||
"""将 IPMI 传感器分类。"""
|
||||
categories = {
|
||||
"temperatures": {},
|
||||
"voltages": {},
|
||||
"fans": {},
|
||||
"power": {},
|
||||
"currents": {},
|
||||
"other": {}
|
||||
}
|
||||
|
||||
for name, data in sensors.items():
|
||||
name_lower = name.lower()
|
||||
unit = data.get("unit", "").lower()
|
||||
|
||||
if 'temp' in name_lower or unit == 'degrees c':
|
||||
categories["temperatures"][name] = data
|
||||
elif 'volt' in name_lower or unit == 'volts' or 'vcc' in name_lower or '3.3v' in name_lower or '5v' in name_lower or '12v' in name_lower:
|
||||
categories["voltages"][name] = data
|
||||
elif 'fan' in name_lower or 'rpm' in unit:
|
||||
categories["fans"][name] = data
|
||||
elif 'power' in name_lower or 'watt' in unit:
|
||||
categories["power"][name] = data
|
||||
elif 'current' in name_lower or 'amp' in unit:
|
||||
categories["currents"][name] = data
|
||||
else:
|
||||
categories["other"][name] = data
|
||||
|
||||
return categories
|
||||
|
||||
|
||||
def get_thermal_zones() -> Dict[str, Any]:
|
||||
"""从 thermal zone 获取温度信息。"""
|
||||
result = {
|
||||
"zones": {},
|
||||
"policies": {}
|
||||
}
|
||||
|
||||
thermal_path = '/sys/class/thermal'
|
||||
|
||||
if not os.path.exists(thermal_path):
|
||||
return result
|
||||
|
||||
try:
|
||||
for zone_name in os.listdir(thermal_path):
|
||||
if not zone_name.startswith('thermal_zone'):
|
||||
continue
|
||||
|
||||
zone_path = os.path.join(thermal_path, zone_name)
|
||||
zone_info = {}
|
||||
|
||||
# 读取类型
|
||||
type_file = os.path.join(zone_path, 'type')
|
||||
if os.path.exists(type_file):
|
||||
with open(type_file, 'r') as f:
|
||||
zone_info["type"] = f.read().strip()
|
||||
|
||||
# 读取温度 (毫摄氏度转换为摄氏度)
|
||||
temp_file = os.path.join(zone_path, 'temp')
|
||||
if os.path.exists(temp_file):
|
||||
with open(temp_file, 'r') as f:
|
||||
temp_mc = safe_int(f.read().strip())
|
||||
zone_info["temperature_c"] = temp_mc / 1000.0
|
||||
|
||||
# 读取策略
|
||||
policy_file = os.path.join(zone_path, 'policy')
|
||||
if os.path.exists(policy_file):
|
||||
with open(policy_file, 'r') as f:
|
||||
zone_info["policy"] = f.read().strip()
|
||||
|
||||
# 读取临界温度
|
||||
trip_point_file = os.path.join(zone_path, 'trip_point_0_temp')
|
||||
if os.path.exists(trip_point_file):
|
||||
with open(trip_point_file, 'r') as f:
|
||||
zone_info["critical_temp_c"] = safe_int(f.read().strip()) / 1000.0
|
||||
|
||||
result["zones"][zone_name] = zone_info
|
||||
|
||||
# 读取 thermal 策略
|
||||
for policy_file in os.listdir('/sys/class/thermal'):
|
||||
if policy_file.startswith('cooling_device'):
|
||||
policy_path = os.path.join('/sys/class/thermal', policy_file)
|
||||
policy_info = {}
|
||||
|
||||
type_file = os.path.join(policy_path, 'type')
|
||||
if os.path.exists(type_file):
|
||||
with open(type_file, 'r') as f:
|
||||
policy_info["type"] = f.read().strip()
|
||||
|
||||
cur_state_file = os.path.join(policy_path, 'cur_state')
|
||||
if os.path.exists(cur_state_file):
|
||||
with open(cur_state_file, 'r') as f:
|
||||
policy_info["current_state"] = safe_int(f.read().strip())
|
||||
|
||||
max_state_file = os.path.join(policy_path, 'max_state')
|
||||
if os.path.exists(max_state_file):
|
||||
with open(max_state_file, 'r') as f:
|
||||
policy_info["max_state"] = safe_int(f.read().strip())
|
||||
|
||||
result["policies"][policy_file] = policy_info
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_power_supply_info() -> Dict[str, Any]:
|
||||
"""获取电源信息。"""
|
||||
result = {
|
||||
"supplies": []
|
||||
}
|
||||
|
||||
power_supply_path = '/sys/class/power_supply'
|
||||
|
||||
if not os.path.exists(power_supply_path):
|
||||
return result
|
||||
|
||||
try:
|
||||
for supply_name in os.listdir(power_supply_path):
|
||||
supply_path = os.path.join(power_supply_path, supply_name)
|
||||
supply_info = {"name": supply_name}
|
||||
|
||||
# 读取所有属性文件
|
||||
for attr in os.listdir(supply_path):
|
||||
attr_path = os.path.join(supply_path, attr)
|
||||
if os.path.isfile(attr_path):
|
||||
try:
|
||||
with open(attr_path, 'r') as f:
|
||||
value = f.read().strip()
|
||||
# 尝试转换为数字
|
||||
if value.isdigit():
|
||||
supply_info[attr] = safe_int(value)
|
||||
else:
|
||||
try:
|
||||
supply_info[attr] = safe_float(value)
|
||||
except:
|
||||
supply_info[attr] = value
|
||||
except:
|
||||
pass
|
||||
|
||||
result["supplies"].append(supply_info)
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_ipmi_sel_logs() -> Dict[str, Any]:
|
||||
"""获取 IPMI SEL(System Event Log)日志。"""
|
||||
result = {
|
||||
"available": False,
|
||||
"entries": [],
|
||||
"hardware_errors": [],
|
||||
"critical_events": []
|
||||
}
|
||||
|
||||
if not check_command_exists('ipmitool'):
|
||||
result["note"] = "ipmitool 未安装"
|
||||
return result
|
||||
|
||||
try:
|
||||
# 获取 SEL 列表
|
||||
_, stdout, stderr = execute_command(
|
||||
['ipmitool', 'sel', 'elist'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
if 'Could not open device' in stderr or 'Driver not found' in stderr:
|
||||
result["note"] = "IPMI 设备不可用"
|
||||
return result
|
||||
|
||||
result["available"] = True
|
||||
|
||||
# 解析 SEL 条目
|
||||
critical_keywords = ['critical', 'failure', 'error', 'thermal', 'voltage', 'power']
|
||||
hardware_keywords = ['memory', 'processor', 'hard drive', 'fan', 'power supply', 'temperature']
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if not line.strip():
|
||||
continue
|
||||
|
||||
# SEL 格式: ID | Date/Time | Source | Event
|
||||
parts = [p.strip() for p in line.split('|')]
|
||||
if len(parts) >= 4:
|
||||
entry = {
|
||||
"id": parts[0],
|
||||
"datetime": parts[1],
|
||||
"source": parts[2],
|
||||
"event": parts[3]
|
||||
}
|
||||
|
||||
result["entries"].append(entry)
|
||||
|
||||
# 检查是否为关键事件
|
||||
event_lower = entry["event"].lower()
|
||||
if any(kw in event_lower for kw in critical_keywords):
|
||||
result["critical_events"].append(entry)
|
||||
|
||||
# 检查是否为硬件错误
|
||||
if any(kw in event_lower for kw in hardware_keywords):
|
||||
result["hardware_errors"].append(entry)
|
||||
|
||||
result["total_entries"] = len(result["entries"])
|
||||
result["critical_count"] = len(result["critical_events"])
|
||||
result["hardware_error_count"] = len(result["hardware_errors"])
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def check_sensor_warnings(sensor_data: Dict[str, Any]) -> List[str]:
|
||||
"""检查传感器警告条件。"""
|
||||
warnings = []
|
||||
|
||||
# 检查 lm-sensors 告警
|
||||
lm_sensors = sensor_data.get("lm_sensors", {})
|
||||
summary = lm_sensors.get("summary", {})
|
||||
|
||||
# 温度告警
|
||||
for name, temp_data in summary.get("temperatures", {}).items():
|
||||
if temp_data.get("alarm"):
|
||||
warnings.append(f"温度传感器 {name} 告警: {temp_data.get('value')}°C")
|
||||
elif temp_data.get("value", 0) > 90:
|
||||
warnings.append(f"温度传感器 {name} 温度过高: {temp_data.get('value')}°C")
|
||||
|
||||
# 电压告警
|
||||
for name, volt_data in summary.get("voltages", {}).items():
|
||||
if volt_data.get("alarm"):
|
||||
warnings.append(f"电压传感器 {name} 告警: {volt_data.get('value')}V")
|
||||
|
||||
# 风扇告警
|
||||
for name, fan_data in summary.get("fans", {}).items():
|
||||
if fan_data.get("alarm"):
|
||||
warnings.append(f"风扇 {name} 告警: {fan_data.get('rpm')} RPM")
|
||||
elif fan_data.get("rpm", 0) == 0 and fan_data.get("min", 0) > 0:
|
||||
warnings.append(f"风扇 {name} 可能已停止: {fan_data.get('rpm')} RPM")
|
||||
|
||||
# 检查 IPMI 告警
|
||||
ipmi_sensors = sensor_data.get("ipmi_sensors", {})
|
||||
for name, data in ipmi_sensors.get("sensors", {}).items():
|
||||
status = data.get("status", "").lower()
|
||||
if status in ['critical', 'non-recoverable', 'warning']:
|
||||
warnings.append(f"IPMI 传感器 {name} 状态异常: {data.get('status')}")
|
||||
|
||||
# 检查 IPMI SEL 关键事件
|
||||
ipmi_sel = sensor_data.get("ipmi_sel", {})
|
||||
if ipmi_sel.get("critical_count", 0) > 0:
|
||||
warnings.append(f"IPMI SEL 中有 {ipmi_sel['critical_count']} 个关键事件")
|
||||
|
||||
# 检查 thermal zone 温度
|
||||
thermal_zones = sensor_data.get("thermal_zones", {})
|
||||
for zone_name, zone_data in thermal_zones.get("zones", {}).items():
|
||||
temp = zone_data.get("temperature_c", 0)
|
||||
critical = zone_data.get("critical_temp_c", 100)
|
||||
if temp > critical * 0.9: # 超过临界温度的 90%
|
||||
warnings.append(f"Thermal zone {zone_name} 温度接近临界值: {temp}°C (临界: {critical}°C)")
|
||||
|
||||
return warnings
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
print(json.dumps(run_sensors_check(), indent=2, ensure_ascii=False))
|
||||
602
modules/storage.py
Normal file
602
modules/storage.py
Normal file
@@ -0,0 +1,602 @@
|
||||
"""
|
||||
ServerGuard - 存储设备检测模块
|
||||
|
||||
检查硬盘/SSD 的健康状况、SMART 数据、RAID 状态。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import (
|
||||
execute_command, check_command_exists, parse_key_value_output,
|
||||
safe_int, safe_float, format_bytes, require_root
|
||||
)
|
||||
|
||||
|
||||
def run_storage_check() -> Dict[str, Any]:
|
||||
"""
|
||||
执行存储设备检测。
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 检测结果
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"devices": [],
|
||||
"raid_status": {},
|
||||
"io_stats": {}
|
||||
}
|
||||
|
||||
try:
|
||||
# 获取存储设备列表
|
||||
devices = get_storage_devices()
|
||||
|
||||
# 检测每个设备
|
||||
for device in devices:
|
||||
device_info = check_device(device)
|
||||
result["devices"].append(device_info)
|
||||
|
||||
# 如果有严重问题,标记警告状态
|
||||
if device_info.get("health") in ['FAILED', 'WARNING']:
|
||||
result["status"] = "warning"
|
||||
|
||||
# 检查 RAID 状态
|
||||
result["raid_status"] = check_raid_status()
|
||||
|
||||
# 获取 I/O 统计
|
||||
result["io_stats"] = get_io_statistics()
|
||||
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_storage_devices() -> List[Dict[str, str]]:
|
||||
"""获取存储设备列表。"""
|
||||
devices = []
|
||||
|
||||
# 方法 1: 使用 lsblk
|
||||
if check_command_exists('lsblk'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['lsblk', '-d', '-n', '-o', 'NAME,TYPE,ROTA', '-J'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
data = json.loads(stdout)
|
||||
for dev in data.get('blockdevices', []):
|
||||
if dev.get('type') == 'disk':
|
||||
devices.append({
|
||||
"name": dev['name'],
|
||||
"path": f"/dev/{dev['name']}",
|
||||
"type": "hdd" if dev.get('rota') else "ssd"
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
# 方法 2: 扫描 /sys/block
|
||||
if not devices:
|
||||
try:
|
||||
for name in os.listdir('/sys/block'):
|
||||
if name.startswith(('sd', 'hd', 'nvme', 'vd', 'xvd', 'mmcblk')):
|
||||
dev_type = "unknown"
|
||||
try:
|
||||
with open(f'/sys/block/{name}/queue/rotational', 'r') as f:
|
||||
dev_type = "hdd" if f.read().strip() == '1' else "ssd"
|
||||
except:
|
||||
pass
|
||||
|
||||
devices.append({
|
||||
"name": name,
|
||||
"path": f"/dev/{name}",
|
||||
"type": dev_type
|
||||
})
|
||||
except:
|
||||
pass
|
||||
|
||||
return devices
|
||||
|
||||
|
||||
def check_device(device: Dict[str, str]) -> Dict[str, Any]:
|
||||
"""检查单个存储设备。"""
|
||||
result = {
|
||||
"name": device["name"],
|
||||
"path": device["path"],
|
||||
"type": device.get("type", "unknown"),
|
||||
"model": "Unknown",
|
||||
"serial": "Unknown",
|
||||
"firmware": "Unknown",
|
||||
"size_bytes": 0,
|
||||
"size_human": "Unknown",
|
||||
"health": "UNKNOWN",
|
||||
"smart_status": {},
|
||||
"temperature_c": None,
|
||||
"power_on_hours": None,
|
||||
"start_stop_count": None,
|
||||
"reallocated_sectors": None,
|
||||
"pending_sectors": None,
|
||||
"test_result": None
|
||||
}
|
||||
|
||||
# 获取设备基本信息
|
||||
result.update(get_device_info(device["path"]))
|
||||
|
||||
# 获取 SMART 数据
|
||||
smart_data = get_smart_data(device["path"])
|
||||
result["smart_status"] = smart_data
|
||||
|
||||
# 分析健康状态
|
||||
result["health"] = analyze_health(smart_data)
|
||||
|
||||
# 提取关键属性
|
||||
if "attributes" in smart_data:
|
||||
attrs = smart_data["attributes"]
|
||||
|
||||
# 温度
|
||||
for temp_attr in ['194 Temperature_Celsius', '190 Airflow_Temperature_Cel', 'Temperature']:
|
||||
if temp_attr in attrs:
|
||||
temp_val = attrs[temp_attr].get('raw_value')
|
||||
if temp_val:
|
||||
result["temperature_c"] = safe_int(temp_val.split()[0])
|
||||
break
|
||||
|
||||
# 运行时间
|
||||
if '9 Power_On_Hours' in attrs:
|
||||
result["power_on_hours"] = safe_int(attrs['9 Power_On_Hours'].get('raw_value', 0))
|
||||
|
||||
# 启动次数
|
||||
if '4 Start_Stop_Count' in attrs:
|
||||
result["start_stop_count"] = safe_int(attrs['4 Start_Stop_Count'].get('raw_value', 0))
|
||||
|
||||
# 重映射扇区
|
||||
if '5 Reallocated_Sector_Ct' in attrs:
|
||||
result["reallocated_sectors"] = safe_int(attrs['5 Reallocated_Sector_Ct'].get('raw_value', 0))
|
||||
|
||||
# 待处理扇区
|
||||
if '197 Current_Pending_Sector' in attrs:
|
||||
result["pending_sectors"] = safe_int(attrs['197 Current_Pending_Sector'].get('raw_value', 0))
|
||||
|
||||
# NVMe 特殊处理
|
||||
if device["name"].startswith('nvme'):
|
||||
nvme_data = get_nvme_data(device["path"])
|
||||
result["nvme_data"] = nvme_data
|
||||
if nvme_data.get("temperature"):
|
||||
result["temperature_c"] = nvme_data["temperature"]
|
||||
if nvme_data.get("health"):
|
||||
result["health"] = nvme_data["health"]
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_device_info(device_path: str) -> Dict[str, Any]:
|
||||
"""获取设备基本信息。"""
|
||||
info = {}
|
||||
|
||||
# 使用 smartctl -i 获取信息
|
||||
if check_command_exists('smartctl'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['smartctl', '-i', device_path],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
patterns = {
|
||||
"model": r'Device Model:\s*(.+)',
|
||||
"serial": r'Serial Number:\s*(\S+)',
|
||||
"firmware": r'Firmware Version:\s*(\S+)',
|
||||
"size_human": r'User Capacity:\s*(.+)',
|
||||
"sector_size": r'Sector Size:\s*(.+)',
|
||||
"rotation_rate": r'Rotation Rate:\s*(.+)',
|
||||
"form_factor": r'Form Factor:\s*(.+)',
|
||||
"transport": r'Transport protocol:\s*(.+)'
|
||||
}
|
||||
|
||||
for key, pattern in patterns.items():
|
||||
match = re.search(pattern, stdout)
|
||||
if match:
|
||||
info[key] = match.group(1).strip()
|
||||
|
||||
# 提取容量字节数
|
||||
size_match = re.search(r'User Capacity:\s*[\d,]+\s*bytes\s*\[(\d+)\]', stdout)
|
||||
if size_match:
|
||||
info["size_bytes"] = safe_int(size_match.group(1))
|
||||
|
||||
# 是否为 SSD
|
||||
if 'Solid State Device' in stdout or 'Rotation Rate: Solid State Device' in stdout:
|
||||
info["is_ssd"] = True
|
||||
elif 'Rotation Rate' in stdout and 'Solid State' not in stdout:
|
||||
info["is_ssd"] = False
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 备用:从 /sys 获取大小
|
||||
if "size_bytes" not in info or info["size_bytes"] == 0:
|
||||
try:
|
||||
dev_name = os.path.basename(device_path)
|
||||
with open(f'/sys/block/{dev_name}/size', 'r') as f:
|
||||
sectors = safe_int(f.read().strip())
|
||||
info["size_bytes"] = sectors * 512
|
||||
info["size_human"] = format_bytes(info["size_bytes"])
|
||||
except:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_smart_data(device_path: str) -> Dict[str, Any]:
|
||||
"""获取 SMART 数据。"""
|
||||
result = {
|
||||
"supported": False,
|
||||
"enabled": False,
|
||||
"overall": "UNKNOWN",
|
||||
"attributes": {},
|
||||
"self_tests": []
|
||||
}
|
||||
|
||||
if not check_command_exists('smartctl'):
|
||||
result["error"] = "smartctl 未安装"
|
||||
return result
|
||||
|
||||
try:
|
||||
# 检查 SMART 支持
|
||||
_, stdout, _ = execute_command(
|
||||
['smartctl', '-i', device_path],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
if 'SMART support is: Available' in stdout:
|
||||
result["supported"] = True
|
||||
if 'SMART support is: Enabled' in stdout:
|
||||
result["enabled"] = True
|
||||
|
||||
# 获取所有 SMART 数据
|
||||
_, stdout, _ = execute_command(
|
||||
['smartctl', '-a', device_path],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
# 解析整体健康状态
|
||||
if 'PASSED' in stdout or 'OK' in stdout:
|
||||
result["overall"] = "PASSED"
|
||||
elif 'FAILED' in stdout:
|
||||
result["overall"] = "FAILED"
|
||||
|
||||
# 解析 SMART 属性表 (ATA 设备)
|
||||
if 'ID#' in stdout and 'ATTRIBUTE_NAME' in stdout:
|
||||
lines = stdout.split('\n')
|
||||
in_attributes = False
|
||||
|
||||
for line in lines:
|
||||
if 'ID#' in line and 'ATTRIBUTE_NAME' in line:
|
||||
in_attributes = True
|
||||
continue
|
||||
|
||||
if in_attributes:
|
||||
if not line.strip() or line.startswith('SMART'):
|
||||
break
|
||||
|
||||
# 解析属性行
|
||||
# 格式: ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
|
||||
parts = line.split()
|
||||
if len(parts) >= 10:
|
||||
attr_id = parts[0]
|
||||
attr_name = parts[1]
|
||||
attr_key = f"{attr_id} {attr_name}"
|
||||
|
||||
result["attributes"][attr_key] = {
|
||||
"flag": parts[2],
|
||||
"value": safe_int(parts[3]),
|
||||
"worst": safe_int(parts[4]),
|
||||
"thresh": safe_int(parts[5]),
|
||||
"type": parts[6],
|
||||
"updated": parts[7],
|
||||
"when_failed": parts[8] if parts[8] != '-' else None,
|
||||
"raw_value": ' '.join(parts[9:])
|
||||
}
|
||||
|
||||
# 解析自检日志
|
||||
if 'SMART Self-test log' in stdout:
|
||||
self_test_section = False
|
||||
for line in stdout.split('\n'):
|
||||
if 'SMART Self-test log' in line:
|
||||
self_test_section = True
|
||||
continue
|
||||
if self_test_section and line.strip() and not line.startswith('SMART'):
|
||||
if '#' in line:
|
||||
result["self_tests"].append(line.strip())
|
||||
|
||||
# 解析错误日志
|
||||
if 'SMART Error Log' in stdout:
|
||||
error_match = re.search(r'Error (\d+)\s+occurred at', stdout)
|
||||
if error_match:
|
||||
result["error_count"] = safe_int(error_match.group(1))
|
||||
|
||||
except Exception as e:
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_nvme_data(device_path: str) -> Dict[str, Any]:
|
||||
"""获取 NVMe 设备特有数据。"""
|
||||
result = {
|
||||
"health": "UNKNOWN",
|
||||
"temperature": None,
|
||||
"available_spare": None,
|
||||
"percentage_used": None,
|
||||
"data_units_read": None,
|
||||
"data_units_written": None,
|
||||
"host_reads": None,
|
||||
"host_writes": None
|
||||
}
|
||||
|
||||
if not check_command_exists('nvme'):
|
||||
return result
|
||||
|
||||
try:
|
||||
# 获取 SMART 日志
|
||||
_, stdout, _ = execute_command(
|
||||
['nvme', 'smart-log', device_path],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
# 解析关键指标
|
||||
temp_match = re.search(r'temperature\s*:\s*(\d+)', stdout)
|
||||
if temp_match:
|
||||
result["temperature"] = safe_int(temp_match.group(1)) - 273 # 转换为摄氏度
|
||||
|
||||
spare_match = re.search(r'available spare\s*:\s*(\d+)%', stdout)
|
||||
if spare_match:
|
||||
result["available_spare"] = safe_int(spare_match.group(1))
|
||||
|
||||
used_match = re.search(r'percentage used\s*:\s*(\d+)%', stdout)
|
||||
if used_match:
|
||||
result["percentage_used"] = safe_int(used_match.group(1))
|
||||
|
||||
# 评估健康状态
|
||||
if result["percentage_used"] is not None:
|
||||
if result["percentage_used"] < 90:
|
||||
result["health"] = "PASSED"
|
||||
else:
|
||||
result["health"] = "WARNING"
|
||||
|
||||
if result["available_spare"] is not None and result["available_spare"] < 10:
|
||||
result["health"] = "WARNING"
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def analyze_health(smart_data: Dict[str, Any]) -> str:
|
||||
"""分析设备健康状态。"""
|
||||
if not smart_data.get("supported"):
|
||||
return "UNKNOWN"
|
||||
|
||||
if smart_data.get("overall") == "FAILED":
|
||||
return "FAILED"
|
||||
|
||||
# 检查关键属性
|
||||
attrs = smart_data.get("attributes", {})
|
||||
|
||||
critical_attrs = {
|
||||
'5 Reallocated_Sector_Ct': 'reallocated_sectors',
|
||||
'197 Current_Pending_Sector': 'pending_sectors',
|
||||
'198 Offline_Uncorrectable': 'offline_uncorrectable',
|
||||
'196 Reallocation_Event_Count': 'reallocation_events'
|
||||
}
|
||||
|
||||
for attr_name, description in critical_attrs.items():
|
||||
if attr_name in attrs:
|
||||
raw_value = attrs[attr_name].get('raw_value', '0')
|
||||
value = safe_int(raw_value.split()[0])
|
||||
if value > 0:
|
||||
return "WARNING"
|
||||
|
||||
# 检查温度
|
||||
for temp_attr in ['194 Temperature_Celsius', '190 Airflow_Temperature_Cel']:
|
||||
if temp_attr in attrs:
|
||||
temp = attrs[temp_attr].get('value', 0)
|
||||
if temp > 60: # 温度阈值
|
||||
return "WARNING"
|
||||
|
||||
return "PASSED"
|
||||
|
||||
|
||||
def check_raid_status() -> Dict[str, Any]:
|
||||
"""检查 RAID 阵列状态。"""
|
||||
result = {
|
||||
"raid_available": False,
|
||||
"controllers": [],
|
||||
"arrays": []
|
||||
}
|
||||
|
||||
# 检查软件 RAID (mdadm)
|
||||
if check_command_exists('mdadm'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['mdadm', '--detail', '--scan'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
if stdout.strip():
|
||||
result["software_raid"] = True
|
||||
result["mdadm_config"] = stdout.strip()
|
||||
|
||||
# 获取详细信息
|
||||
_, detail, _ = execute_command(
|
||||
['cat', '/proc/mdstat'],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
result["mdstat"] = detail
|
||||
|
||||
# 解析每个阵列
|
||||
for line in detail.split('\n'):
|
||||
if line.startswith('md'):
|
||||
parts = line.split()
|
||||
array_info = {
|
||||
"name": parts[0],
|
||||
"status": "active" if "active" in line else "inactive"
|
||||
}
|
||||
|
||||
# 检查是否有降级
|
||||
if '_' in line or 'recovery' in line:
|
||||
array_info["degraded"] = True
|
||||
result["status"] = "warning"
|
||||
|
||||
result["arrays"].append(array_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 检查硬件 RAID (MegaCli/storcli)
|
||||
if check_command_exists('storcli'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['storcli', '/c0', 'show'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
result["hardware_raid"] = True
|
||||
result["controller_type"] = "LSI/Broadcom"
|
||||
result["storcli_output"] = stdout[:500] # 保存部分输出
|
||||
except:
|
||||
pass
|
||||
elif check_command_exists('MegaCli'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['MegaCli', '-AdpAllInfo', '-aALL'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
result["hardware_raid"] = True
|
||||
result["controller_type"] = "LSI"
|
||||
result["megacli_output"] = stdout[:500]
|
||||
except:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_io_statistics() -> Dict[str, Any]:
|
||||
"""获取 I/O 统计信息。"""
|
||||
result = {}
|
||||
|
||||
# 从 /proc/diskstats 获取
|
||||
try:
|
||||
with open('/proc/diskstats', 'r') as f:
|
||||
for line in f:
|
||||
parts = line.split()
|
||||
if len(parts) >= 14:
|
||||
device = parts[2]
|
||||
# 只关注物理磁盘
|
||||
if device.startswith(('sd', 'hd', 'nvme', 'vd')) and not device[-1].isdigit():
|
||||
result[device] = {
|
||||
"reads_completed": safe_int(parts[3]),
|
||||
"reads_merged": safe_int(parts[4]),
|
||||
"sectors_read": safe_int(parts[5]),
|
||||
"time_reading_ms": safe_int(parts[6]),
|
||||
"writes_completed": safe_int(parts[7]),
|
||||
"writes_merged": safe_int(parts[8]),
|
||||
"sectors_written": safe_int(parts[9]),
|
||||
"time_writing_ms": safe_int(parts[10]),
|
||||
"ios_in_progress": safe_int(parts[11]),
|
||||
"time_doing_ios_ms": safe_int(parts[12]),
|
||||
"weighted_time_ios_ms": safe_int(parts[13])
|
||||
}
|
||||
except:
|
||||
pass
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@require_root
|
||||
def run_io_test(device_path: str, test_size_mb: int = 100) -> Dict[str, Any]:
|
||||
"""
|
||||
运行简单的 I/O 性能测试。
|
||||
|
||||
Args:
|
||||
device_path: 设备路径
|
||||
test_size_mb: 测试大小(MB)
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 测试结果
|
||||
"""
|
||||
result = {
|
||||
"passed": False,
|
||||
"device": device_path,
|
||||
"test_size_mb": test_size_mb,
|
||||
"read_speed_mbps": None,
|
||||
"write_speed_mbps": None,
|
||||
"errors": []
|
||||
}
|
||||
|
||||
# 使用 fio 进行测试
|
||||
if check_command_exists('fio'):
|
||||
try:
|
||||
import tempfile
|
||||
|
||||
with tempfile.NamedTemporaryFile(mode='w', suffix='.fio', delete=False) as f:
|
||||
fio_config = f"""
|
||||
[global]
|
||||
directory=/tmp
|
||||
filename=serverguard_test
|
||||
direct=1
|
||||
size={test_size_mb}M
|
||||
unlink=1
|
||||
|
||||
[seq_read]
|
||||
stonewall
|
||||
rw=read
|
||||
bs=1M
|
||||
|
||||
[seq_write]
|
||||
stonewall
|
||||
rw=write
|
||||
bs=1M
|
||||
"""
|
||||
f.write(fio_config)
|
||||
fio_file = f.name
|
||||
|
||||
try:
|
||||
_, stdout, stderr = execute_command(
|
||||
['fio', fio_file, '--output-format=json'],
|
||||
timeout=120,
|
||||
check_returncode=False
|
||||
)
|
||||
|
||||
data = json.loads(stdout)
|
||||
|
||||
for job in data.get('jobs', []):
|
||||
job_name = job.get('jobname', '')
|
||||
read_bw = job.get('read', {}).get('bw', 0) / 1024 # 转换为 MB/s
|
||||
write_bw = job.get('write', {}).get('bw', 0) / 1024
|
||||
|
||||
if 'read' in job_name.lower() and read_bw > 0:
|
||||
result["read_speed_mbps"] = round(read_bw, 2)
|
||||
if 'write' in job_name.lower() and write_bw > 0:
|
||||
result["write_speed_mbps"] = round(write_bw, 2)
|
||||
|
||||
result["passed"] = True
|
||||
|
||||
finally:
|
||||
os.unlink(fio_file)
|
||||
|
||||
except Exception as e:
|
||||
result["errors"].append(str(e))
|
||||
else:
|
||||
result["errors"].append("fio 未安装")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
import json
|
||||
print(json.dumps(run_storage_check(), indent=2, ensure_ascii=False))
|
||||
476
modules/system_info.py
Normal file
476
modules/system_info.py
Normal file
@@ -0,0 +1,476 @@
|
||||
"""
|
||||
ServerGuard - 系统信息概览模块
|
||||
|
||||
收集服务器的硬件和操作系统基本信息。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import platform
|
||||
from typing import Dict, Any, List, Optional
|
||||
|
||||
import sys
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from utils import (
|
||||
execute_command, parse_key_value_output, check_command_exists,
|
||||
safe_int, safe_float, format_bytes
|
||||
)
|
||||
|
||||
|
||||
def get_system_info() -> Dict[str, Any]:
|
||||
"""
|
||||
获取系统硬件和操作系统信息。
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 系统信息字典
|
||||
"""
|
||||
result = {
|
||||
"status": "success",
|
||||
"os": {},
|
||||
"cpu": {},
|
||||
"memory": {},
|
||||
"motherboard": {},
|
||||
"storage": [],
|
||||
"network": [],
|
||||
"gpu": []
|
||||
}
|
||||
|
||||
try:
|
||||
result["os"] = get_os_info()
|
||||
result["cpu"] = get_cpu_info()
|
||||
result["memory"] = get_memory_info()
|
||||
result["motherboard"] = get_motherboard_info()
|
||||
result["storage"] = get_storage_list()
|
||||
result["network"] = get_network_info()
|
||||
result["gpu"] = get_gpu_list()
|
||||
except Exception as e:
|
||||
result["status"] = "error"
|
||||
result["error"] = str(e)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def get_os_info() -> Dict[str, str]:
|
||||
"""获取操作系统信息。"""
|
||||
info = {
|
||||
"platform": platform.system(),
|
||||
"release": platform.release(),
|
||||
"version": platform.version(),
|
||||
"machine": platform.machine(),
|
||||
"processor": platform.processor()
|
||||
}
|
||||
|
||||
# 尝试获取 Linux 发行版信息
|
||||
if os.path.exists('/etc/os-release'):
|
||||
try:
|
||||
with open('/etc/os-release', 'r') as f:
|
||||
for line in f:
|
||||
if line.startswith('PRETTY_NAME='):
|
||||
info["distribution"] = line.split('=', 1)[1].strip().strip('"')
|
||||
break
|
||||
except:
|
||||
pass
|
||||
|
||||
# 获取主机名
|
||||
try:
|
||||
_, hostname, _ = execute_command(['hostname'], check_returncode=False)
|
||||
info["hostname"] = hostname.strip()
|
||||
except:
|
||||
info["hostname"] = "unknown"
|
||||
|
||||
# 获取 uptime
|
||||
try:
|
||||
with open('/proc/uptime', 'r') as f:
|
||||
uptime_seconds = float(f.readline().split()[0])
|
||||
days = int(uptime_seconds // 86400)
|
||||
hours = int((uptime_seconds % 86400) // 3600)
|
||||
minutes = int((uptime_seconds % 3600) // 60)
|
||||
info["uptime"] = f"{days}天 {hours}小时 {minutes}分钟"
|
||||
except:
|
||||
info["uptime"] = "unknown"
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_cpu_info() -> Dict[str, Any]:
|
||||
"""获取 CPU 信息。"""
|
||||
info = {
|
||||
"model": "Unknown",
|
||||
"vendor": "Unknown",
|
||||
"architecture": "Unknown",
|
||||
"cores": 0,
|
||||
"threads": 0,
|
||||
"frequency_mhz": 0,
|
||||
"cache_size_kb": {}
|
||||
}
|
||||
|
||||
# 从 /proc/cpuinfo 获取
|
||||
try:
|
||||
cpu_data = {}
|
||||
with open('/proc/cpuinfo', 'r') as f:
|
||||
for line in f:
|
||||
if ':' in line:
|
||||
key, value = line.split(':', 1)
|
||||
cpu_data[key.strip()] = value.strip()
|
||||
|
||||
info["model"] = cpu_data.get('model name', 'Unknown')
|
||||
info["vendor"] = cpu_data.get('vendor_id', 'Unknown')
|
||||
info["architecture"] = cpu_data.get('cpu architecture', platform.machine())
|
||||
info["cores"] = safe_int(cpu_data.get('cpu cores', 0))
|
||||
info["threads"] = safe_int(cpu_data.get('siblings', 0))
|
||||
info["frequency_mhz"] = safe_int(cpu_data.get('cpu MHz', 0))
|
||||
|
||||
# 缓存信息
|
||||
if 'cache size' in cpu_data:
|
||||
cache = cpu_data['cache size']
|
||||
info["cache_size_kb"] = {"general": cache}
|
||||
except Exception as e:
|
||||
pass
|
||||
|
||||
# 使用 lscpu 获取更详细的信息
|
||||
if check_command_exists('lscpu'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(['lscpu'], check_returncode=False, timeout=10)
|
||||
lscpu_data = parse_key_value_output(stdout)
|
||||
|
||||
if 'Model name' in lscpu_data:
|
||||
info["model"] = lscpu_data['Model name']
|
||||
if 'Architecture' in lscpu_data:
|
||||
info["architecture"] = lscpu_data['Architecture']
|
||||
if 'CPU(s)' in lscpu_data:
|
||||
info["threads"] = safe_int(lscpu_data['CPU(s)'])
|
||||
if 'Core(s) per socket' in lscpu_data and 'Socket(s)' in lscpu_data:
|
||||
cores_per_socket = safe_int(lscpu_data['Core(s) per socket'])
|
||||
sockets = safe_int(lscpu_data['Socket(s)'])
|
||||
info["cores"] = cores_per_socket * sockets
|
||||
if 'CPU max MHz' in lscpu_data:
|
||||
info["max_frequency_mhz"] = safe_float(lscpu_data['CPU max MHz'])
|
||||
if 'CPU min MHz' in lscpu_data:
|
||||
info["min_frequency_mhz"] = safe_float(lscpu_data['CPU min MHz'])
|
||||
if 'Virtualization' in lscpu_data:
|
||||
info["virtualization"] = lscpu_data['Virtualization']
|
||||
except:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_memory_info() -> Dict[str, Any]:
|
||||
"""获取内存信息。"""
|
||||
info = {
|
||||
"total_gb": 0,
|
||||
"available_gb": 0,
|
||||
"slots_total": 0,
|
||||
"slots_used": 0,
|
||||
"slots": [],
|
||||
"type": "Unknown",
|
||||
"speed_mhz": 0,
|
||||
"ecc_supported": False
|
||||
}
|
||||
|
||||
# 从 /proc/meminfo 获取总内存
|
||||
try:
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
for line in f:
|
||||
if line.startswith('MemTotal:'):
|
||||
kb = safe_int(line.split()[1])
|
||||
info["total_gb"] = round(kb / 1024 / 1024, 2)
|
||||
elif line.startswith('MemAvailable:'):
|
||||
kb = safe_int(line.split()[1])
|
||||
info["available_gb"] = round(kb / 1024 / 1024, 2)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 使用 dmidecode 获取详细内存信息
|
||||
if check_command_exists('dmidecode'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['dmidecode', '-t', 'memory'],
|
||||
check_returncode=False, timeout=15
|
||||
)
|
||||
|
||||
memory_devices = stdout.split('Memory Device')
|
||||
slots = []
|
||||
|
||||
for device in memory_devices[1:]: # 第一个是标题,跳过
|
||||
slot = {}
|
||||
|
||||
# 解析各项属性
|
||||
size_match = re.search(r'Size:\s*(\d+)\s*MB', device)
|
||||
if size_match:
|
||||
slot["size_gb"] = round(safe_int(size_match.group(1)) / 1024, 2)
|
||||
|
||||
type_match = re.search(r'Type:\s*(DDR\d+)', device)
|
||||
if type_match:
|
||||
slot["type"] = type_match.group(1)
|
||||
info["type"] = type_match.group(1)
|
||||
|
||||
speed_match = re.search(r'Speed:\s*(\d+)\s*MT/s', device)
|
||||
if speed_match:
|
||||
slot["speed_mhz"] = safe_int(speed_match.group(1))
|
||||
|
||||
manufacturer_match = re.search(r'Manufacturer:\s*(\S+)', device)
|
||||
if manufacturer_match:
|
||||
slot["manufacturer"] = manufacturer_match.group(1)
|
||||
|
||||
locator_match = re.search(r'Locator:\s*(.+)', device)
|
||||
if locator_match:
|
||||
slot["locator"] = locator_match.group(1).strip()
|
||||
|
||||
if slot and slot.get("size_gb", 0) > 0:
|
||||
slots.append(slot)
|
||||
|
||||
info["slots"] = slots
|
||||
info["slots_used"] = len(slots)
|
||||
|
||||
# 计算总插槽数
|
||||
array_match = re.search(r'Number Of Devices:\s*(\d+)', stdout)
|
||||
if array_match:
|
||||
info["slots_total"] = safe_int(array_match.group(1))
|
||||
else:
|
||||
info["slots_total"] = len(slots)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 使用 free 命令作为备用
|
||||
if info["total_gb"] == 0 and check_command_exists('free'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(['free', '-m'], check_returncode=False)
|
||||
lines = stdout.strip().split('\n')
|
||||
if len(lines) > 1:
|
||||
parts = lines[1].split()
|
||||
if len(parts) >= 2:
|
||||
info["total_gb"] = round(safe_int(parts[1]) / 1024, 2)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 检查 ECC 支持
|
||||
try:
|
||||
with open('/proc/meminfo', 'r') as f:
|
||||
content = f.read()
|
||||
if 'HardwareCorrupted' in content:
|
||||
info["ecc_supported"] = True
|
||||
except:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_motherboard_info() -> Dict[str, str]:
|
||||
"""获取主板信息。"""
|
||||
info = {
|
||||
"manufacturer": "Unknown",
|
||||
"product_name": "Unknown",
|
||||
"version": "Unknown",
|
||||
"serial_number": "Unknown",
|
||||
"bios_vendor": "Unknown",
|
||||
"bios_version": "Unknown",
|
||||
"bios_date": "Unknown"
|
||||
}
|
||||
|
||||
if check_command_exists('dmidecode'):
|
||||
try:
|
||||
# 获取主板信息
|
||||
_, stdout, _ = execute_command(
|
||||
['dmidecode', '-t', 'baseboard'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
patterns = {
|
||||
"manufacturer": r'Manufacturer:\s*(.+)',
|
||||
"product_name": r'Product Name:\s*(.+)',
|
||||
"version": r'Version:\s*(.+)',
|
||||
"serial_number": r'Serial Number:\s*(.+)'
|
||||
}
|
||||
|
||||
for key, pattern in patterns.items():
|
||||
match = re.search(pattern, stdout)
|
||||
if match:
|
||||
value = match.group(1).strip()
|
||||
if value not in ['Not Specified', 'To be filled by O.E.M.', 'None']:
|
||||
info[key] = value
|
||||
|
||||
# 获取 BIOS 信息
|
||||
_, stdout, _ = execute_command(
|
||||
['dmidecode', '-t', 'bios'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
bios_patterns = {
|
||||
"bios_vendor": r'Vendor:\s*(.+)',
|
||||
"bios_version": r'Version:\s*(.+)',
|
||||
"bios_date": r'Release Date:\s*(.+)'
|
||||
}
|
||||
|
||||
for key, pattern in bios_patterns.items():
|
||||
match = re.search(pattern, stdout)
|
||||
if match:
|
||||
info[key] = match.group(1).strip()
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def get_storage_list() -> List[Dict[str, Any]]:
|
||||
"""获取存储设备列表。"""
|
||||
devices = []
|
||||
|
||||
# 使用 lsblk 获取块设备列表
|
||||
if check_command_exists('lsblk'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['lsblk', '-d', '-o', 'NAME,SIZE,TYPE,MODEL,VENDOR,ROTA', '-n', '-J'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
import json
|
||||
data = json.loads(stdout)
|
||||
|
||||
for device in data.get('blockdevices', []):
|
||||
dev_info = {
|
||||
"name": device.get('name', 'unknown'),
|
||||
"path": f"/dev/{device.get('name', 'unknown')}",
|
||||
"size": device.get('size', 'unknown'),
|
||||
"type": device.get('type', 'unknown'),
|
||||
"model": device.get('model', 'unknown'),
|
||||
"vendor": device.get('vendor', 'unknown'),
|
||||
"is_rotational": device.get('rota', True)
|
||||
}
|
||||
devices.append(dev_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
# 备用方法:直接读取 /sys/block
|
||||
if not devices:
|
||||
try:
|
||||
for name in os.listdir('/sys/block'):
|
||||
if name.startswith(('sd', 'hd', 'nvme', 'vd')):
|
||||
dev_info = {"name": name, "path": f"/dev/{name}"}
|
||||
|
||||
# 尝试读取大小
|
||||
try:
|
||||
with open(f'/sys/block/{name}/size', 'r') as f:
|
||||
sectors = safe_int(f.read().strip())
|
||||
size_bytes = sectors * 512
|
||||
dev_info["size"] = format_bytes(size_bytes)
|
||||
except:
|
||||
dev_info["size"] = "unknown"
|
||||
|
||||
# 判断是否为 SSD
|
||||
try:
|
||||
with open(f'/sys/block/{name}/queue/rotational', 'r') as f:
|
||||
dev_info["is_rotational"] = f.read().strip() == '1'
|
||||
dev_info["type"] = 'hdd' if dev_info["is_rotational"] else 'ssd'
|
||||
except:
|
||||
dev_info["type"] = 'unknown'
|
||||
|
||||
devices.append(dev_info)
|
||||
except:
|
||||
pass
|
||||
|
||||
return devices
|
||||
|
||||
|
||||
def get_network_info() -> List[Dict[str, Any]]:
|
||||
"""获取网络接口信息。"""
|
||||
interfaces = []
|
||||
|
||||
# 使用 ip 命令
|
||||
if check_command_exists('ip'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['ip', '-j', 'link', 'show'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
import json
|
||||
data = json.loads(stdout)
|
||||
|
||||
for iface in data:
|
||||
iface_info = {
|
||||
"name": iface.get('ifname', 'unknown'),
|
||||
"state": iface.get('operstate', 'unknown'),
|
||||
"mac_address": iface.get('address', 'unknown'),
|
||||
"type": iface.get('link_type', 'unknown')
|
||||
}
|
||||
|
||||
# 获取 IP 地址
|
||||
if 'addr_info' in iface:
|
||||
ips = []
|
||||
for addr in iface['addr_info']:
|
||||
if addr.get('family') == 'inet':
|
||||
ips.append(f"{addr.get('local')}/{addr.get('prefixlen', '')}")
|
||||
if ips:
|
||||
iface_info["ip_addresses"] = ips
|
||||
|
||||
interfaces.append(iface_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return interfaces
|
||||
|
||||
|
||||
def get_gpu_list() -> List[Dict[str, Any]]:
|
||||
"""获取显卡列表。"""
|
||||
gpus = []
|
||||
|
||||
# 使用 lspci 查找 VGA 和 3D 控制器
|
||||
if check_command_exists('lspci'):
|
||||
try:
|
||||
_, stdout, _ = execute_command(
|
||||
['lspci', '-nn'],
|
||||
check_returncode=False, timeout=10
|
||||
)
|
||||
|
||||
for line in stdout.split('\n'):
|
||||
if 'VGA' in line or '3D controller' in line or 'Display controller' in line:
|
||||
# 提取设备信息
|
||||
parts = line.split(': ', 1)
|
||||
if len(parts) == 2:
|
||||
bus_id = parts[0].split()[0]
|
||||
description = parts[1]
|
||||
|
||||
gpu_info = {
|
||||
"bus_id": bus_id,
|
||||
"description": description,
|
||||
"type": "integrated" if "Intel" in description else "discrete"
|
||||
}
|
||||
|
||||
# 尝试获取更详细的信息
|
||||
try:
|
||||
_, detail, _ = execute_command(
|
||||
['lspci', '-v', '-s', bus_id],
|
||||
check_returncode=False, timeout=5
|
||||
)
|
||||
|
||||
# 提取驱动信息
|
||||
driver_match = re.search(r'Kernel driver in use:\s*(\S+)', detail)
|
||||
if driver_match:
|
||||
gpu_info["driver"] = driver_match.group(1)
|
||||
|
||||
# 提取模块信息
|
||||
modules_match = re.search(r'Kernel modules:\s*(.+)', detail)
|
||||
if modules_match:
|
||||
gpu_info["modules"] = modules_match.group(1).strip()
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
gpus.append(gpu_info)
|
||||
|
||||
except:
|
||||
pass
|
||||
|
||||
return gpus
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 测试模块
|
||||
import json
|
||||
print(json.dumps(get_system_info(), indent=2, ensure_ascii=False))
|
||||
Reference in New Issue
Block a user