603 lines
19 KiB
Python
603 lines
19 KiB
Python
"""
|
||
ServerGuard - 存储设备检测模块
|
||
|
||
检查硬盘/SSD 的健康状况、SMART 数据、RAID 状态。
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import json
|
||
from typing import Dict, Any, List, Optional
|
||
|
||
import sys
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
from utils import (
|
||
execute_command, check_command_exists, parse_key_value_output,
|
||
safe_int, safe_float, format_bytes, require_root
|
||
)
|
||
|
||
|
||
def run_storage_check() -> Dict[str, Any]:
|
||
"""
|
||
执行存储设备检测。
|
||
|
||
Returns:
|
||
Dict[str, Any]: 检测结果
|
||
"""
|
||
result = {
|
||
"status": "success",
|
||
"devices": [],
|
||
"raid_status": {},
|
||
"io_stats": {}
|
||
}
|
||
|
||
try:
|
||
# 获取存储设备列表
|
||
devices = get_storage_devices()
|
||
|
||
# 检测每个设备
|
||
for device in devices:
|
||
device_info = check_device(device)
|
||
result["devices"].append(device_info)
|
||
|
||
# 如果有严重问题，标记警告状态
|
||
if device_info.get("health") in ['FAILED', 'WARNING']:
|
||
result["status"] = "warning"
|
||
|
||
# 检查 RAID 状态
|
||
result["raid_status"] = check_raid_status()
|
||
|
||
# 获取 I/O 统计
|
||
result["io_stats"] = get_io_statistics()
|
||
|
||
except Exception as e:
|
||
result["status"] = "error"
|
||
result["error"] = str(e)
|
||
|
||
return result
|
||
|
||
|
||
def get_storage_devices() -> List[Dict[str, str]]:
|
||
"""获取存储设备列表。"""
|
||
devices = []
|
||
|
||
# 方法 1: 使用 lsblk
|
||
if check_command_exists('lsblk'):
|
||
try:
|
||
_, stdout, _ = execute_command(
|
||
['lsblk', '-d', '-n', '-o', 'NAME,TYPE,ROTA', '-J'],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
|
||
data = json.loads(stdout)
|
||
for dev in data.get('blockdevices', []):
|
||
if dev.get('type') == 'disk':
|
||
devices.append({
|
||
"name": dev['name'],
|
||
"path": f"/dev/{dev['name']}",
|
||
"type": "hdd" if dev.get('rota') else "ssd"
|
||
})
|
||
except:
|
||
pass
|
||
|
||
# 方法 2: 扫描 /sys/block
|
||
if not devices:
|
||
try:
|
||
for name in os.listdir('/sys/block'):
|
||
if name.startswith(('sd', 'hd', 'nvme', 'vd', 'xvd', 'mmcblk')):
|
||
dev_type = "unknown"
|
||
try:
|
||
with open(f'/sys/block/{name}/queue/rotational', 'r') as f:
|
||
dev_type = "hdd" if f.read().strip() == '1' else "ssd"
|
||
except:
|
||
pass
|
||
|
||
devices.append({
|
||
"name": name,
|
||
"path": f"/dev/{name}",
|
||
"type": dev_type
|
||
})
|
||
except:
|
||
pass
|
||
|
||
return devices
|
||
|
||
|
||
def check_device(device: Dict[str, str]) -> Dict[str, Any]:
|
||
"""检查单个存储设备。"""
|
||
result = {
|
||
"name": device["name"],
|
||
"path": device["path"],
|
||
"type": device.get("type", "unknown"),
|
||
"model": "Unknown",
|
||
"serial": "Unknown",
|
||
"firmware": "Unknown",
|
||
"size_bytes": 0,
|
||
"size_human": "Unknown",
|
||
"health": "UNKNOWN",
|
||
"smart_status": {},
|
||
"temperature_c": None,
|
||
"power_on_hours": None,
|
||
"start_stop_count": None,
|
||
"reallocated_sectors": None,
|
||
"pending_sectors": None,
|
||
"test_result": None
|
||
}
|
||
|
||
# 获取设备基本信息
|
||
result.update(get_device_info(device["path"]))
|
||
|
||
# 获取 SMART 数据
|
||
smart_data = get_smart_data(device["path"])
|
||
result["smart_status"] = smart_data
|
||
|
||
# 分析健康状态
|
||
result["health"] = analyze_health(smart_data)
|
||
|
||
# 提取关键属性
|
||
if "attributes" in smart_data:
|
||
attrs = smart_data["attributes"]
|
||
|
||
# 温度
|
||
for temp_attr in ['194 Temperature_Celsius', '190 Airflow_Temperature_Cel', 'Temperature']:
|
||
if temp_attr in attrs:
|
||
temp_val = attrs[temp_attr].get('raw_value')
|
||
if temp_val:
|
||
result["temperature_c"] = safe_int(temp_val.split()[0])
|
||
break
|
||
|
||
# 运行时间
|
||
if '9 Power_On_Hours' in attrs:
|
||
result["power_on_hours"] = safe_int(attrs['9 Power_On_Hours'].get('raw_value', 0))
|
||
|
||
# 启动次数
|
||
if '4 Start_Stop_Count' in attrs:
|
||
result["start_stop_count"] = safe_int(attrs['4 Start_Stop_Count'].get('raw_value', 0))
|
||
|
||
# 重映射扇区
|
||
if '5 Reallocated_Sector_Ct' in attrs:
|
||
result["reallocated_sectors"] = safe_int(attrs['5 Reallocated_Sector_Ct'].get('raw_value', 0))
|
||
|
||
# 待处理扇区
|
||
if '197 Current_Pending_Sector' in attrs:
|
||
result["pending_sectors"] = safe_int(attrs['197 Current_Pending_Sector'].get('raw_value', 0))
|
||
|
||
# NVMe 特殊处理
|
||
if device["name"].startswith('nvme'):
|
||
nvme_data = get_nvme_data(device["path"])
|
||
result["nvme_data"] = nvme_data
|
||
if nvme_data.get("temperature"):
|
||
result["temperature_c"] = nvme_data["temperature"]
|
||
if nvme_data.get("health"):
|
||
result["health"] = nvme_data["health"]
|
||
|
||
return result
|
||
|
||
|
||
def get_device_info(device_path: str) -> Dict[str, Any]:
|
||
"""获取设备基本信息。"""
|
||
info = {}
|
||
|
||
# 使用 smartctl -i 获取信息
|
||
if check_command_exists('smartctl'):
|
||
try:
|
||
_, stdout, _ = execute_command(
|
||
['smartctl', '-i', device_path],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
|
||
patterns = {
|
||
"model": r'Device Model:\s*(.+)',
|
||
"serial": r'Serial Number:\s*(\S+)',
|
||
"firmware": r'Firmware Version:\s*(\S+)',
|
||
"size_human": r'User Capacity:\s*(.+)',
|
||
"sector_size": r'Sector Size:\s*(.+)',
|
||
"rotation_rate": r'Rotation Rate:\s*(.+)',
|
||
"form_factor": r'Form Factor:\s*(.+)',
|
||
"transport": r'Transport protocol:\s*(.+)'
|
||
}
|
||
|
||
for key, pattern in patterns.items():
|
||
match = re.search(pattern, stdout)
|
||
if match:
|
||
info[key] = match.group(1).strip()
|
||
|
||
# 提取容量字节数
|
||
size_match = re.search(r'User Capacity:\s*[\d,]+\s*bytes\s*\[(\d+)\]', stdout)
|
||
if size_match:
|
||
info["size_bytes"] = safe_int(size_match.group(1))
|
||
|
||
# 是否为 SSD
|
||
if 'Solid State Device' in stdout or 'Rotation Rate: Solid State Device' in stdout:
|
||
info["is_ssd"] = True
|
||
elif 'Rotation Rate' in stdout and 'Solid State' not in stdout:
|
||
info["is_ssd"] = False
|
||
|
||
except:
|
||
pass
|
||
|
||
# 备用：从 /sys 获取大小
|
||
if "size_bytes" not in info or info["size_bytes"] == 0:
|
||
try:
|
||
dev_name = os.path.basename(device_path)
|
||
with open(f'/sys/block/{dev_name}/size', 'r') as f:
|
||
sectors = safe_int(f.read().strip())
|
||
info["size_bytes"] = sectors * 512
|
||
info["size_human"] = format_bytes(info["size_bytes"])
|
||
except:
|
||
pass
|
||
|
||
return info
|
||
|
||
|
||
def get_smart_data(device_path: str) -> Dict[str, Any]:
|
||
"""获取 SMART 数据。"""
|
||
result = {
|
||
"supported": False,
|
||
"enabled": False,
|
||
"overall": "UNKNOWN",
|
||
"attributes": {},
|
||
"self_tests": []
|
||
}
|
||
|
||
if not check_command_exists('smartctl'):
|
||
result["error"] = "smartctl 未安装"
|
||
return result
|
||
|
||
try:
|
||
# 检查 SMART 支持
|
||
_, stdout, _ = execute_command(
|
||
['smartctl', '-i', device_path],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
|
||
if 'SMART support is: Available' in stdout:
|
||
result["supported"] = True
|
||
if 'SMART support is: Enabled' in stdout:
|
||
result["enabled"] = True
|
||
|
||
# 获取所有 SMART 数据
|
||
_, stdout, _ = execute_command(
|
||
['smartctl', '-a', device_path],
|
||
check_returncode=False, timeout=15
|
||
)
|
||
|
||
# 解析整体健康状态
|
||
if 'PASSED' in stdout or 'OK' in stdout:
|
||
result["overall"] = "PASSED"
|
||
elif 'FAILED' in stdout:
|
||
result["overall"] = "FAILED"
|
||
|
||
# 解析 SMART 属性表 (ATA 设备)
|
||
if 'ID#' in stdout and 'ATTRIBUTE_NAME' in stdout:
|
||
lines = stdout.split('\n')
|
||
in_attributes = False
|
||
|
||
for line in lines:
|
||
if 'ID#' in line and 'ATTRIBUTE_NAME' in line:
|
||
in_attributes = True
|
||
continue
|
||
|
||
if in_attributes:
|
||
if not line.strip() or line.startswith('SMART'):
|
||
break
|
||
|
||
# 解析属性行
|
||
# 格式: ID# ATTRIBUTE_NAME FLAG VALUE WORST THRESH TYPE UPDATED WHEN_FAILED RAW_VALUE
|
||
parts = line.split()
|
||
if len(parts) >= 10:
|
||
attr_id = parts[0]
|
||
attr_name = parts[1]
|
||
attr_key = f"{attr_id} {attr_name}"
|
||
|
||
result["attributes"][attr_key] = {
|
||
"flag": parts[2],
|
||
"value": safe_int(parts[3]),
|
||
"worst": safe_int(parts[4]),
|
||
"thresh": safe_int(parts[5]),
|
||
"type": parts[6],
|
||
"updated": parts[7],
|
||
"when_failed": parts[8] if parts[8] != '-' else None,
|
||
"raw_value": ' '.join(parts[9:])
|
||
}
|
||
|
||
# 解析自检日志
|
||
if 'SMART Self-test log' in stdout:
|
||
self_test_section = False
|
||
for line in stdout.split('\n'):
|
||
if 'SMART Self-test log' in line:
|
||
self_test_section = True
|
||
continue
|
||
if self_test_section and line.strip() and not line.startswith('SMART'):
|
||
if '#' in line:
|
||
result["self_tests"].append(line.strip())
|
||
|
||
# 解析错误日志
|
||
if 'SMART Error Log' in stdout:
|
||
error_match = re.search(r'Error (\d+)\s+occurred at', stdout)
|
||
if error_match:
|
||
result["error_count"] = safe_int(error_match.group(1))
|
||
|
||
except Exception as e:
|
||
result["error"] = str(e)
|
||
|
||
return result
|
||
|
||
|
||
def get_nvme_data(device_path: str) -> Dict[str, Any]:
|
||
"""获取 NVMe 设备特有数据。"""
|
||
result = {
|
||
"health": "UNKNOWN",
|
||
"temperature": None,
|
||
"available_spare": None,
|
||
"percentage_used": None,
|
||
"data_units_read": None,
|
||
"data_units_written": None,
|
||
"host_reads": None,
|
||
"host_writes": None
|
||
}
|
||
|
||
if not check_command_exists('nvme'):
|
||
return result
|
||
|
||
try:
|
||
# 获取 SMART 日志
|
||
_, stdout, _ = execute_command(
|
||
['nvme', 'smart-log', device_path],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
|
||
# 解析关键指标
|
||
temp_match = re.search(r'temperature\s*:\s*(\d+)', stdout)
|
||
if temp_match:
|
||
result["temperature"] = safe_int(temp_match.group(1)) - 273 # 转换为摄氏度
|
||
|
||
spare_match = re.search(r'available spare\s*:\s*(\d+)%', stdout)
|
||
if spare_match:
|
||
result["available_spare"] = safe_int(spare_match.group(1))
|
||
|
||
used_match = re.search(r'percentage used\s*:\s*(\d+)%', stdout)
|
||
if used_match:
|
||
result["percentage_used"] = safe_int(used_match.group(1))
|
||
|
||
# 评估健康状态
|
||
if result["percentage_used"] is not None:
|
||
if result["percentage_used"] < 90:
|
||
result["health"] = "PASSED"
|
||
else:
|
||
result["health"] = "WARNING"
|
||
|
||
if result["available_spare"] is not None and result["available_spare"] < 10:
|
||
result["health"] = "WARNING"
|
||
|
||
except:
|
||
pass
|
||
|
||
return result
|
||
|
||
|
||
def analyze_health(smart_data: Dict[str, Any]) -> str:
|
||
"""分析设备健康状态。"""
|
||
if not smart_data.get("supported"):
|
||
return "UNKNOWN"
|
||
|
||
if smart_data.get("overall") == "FAILED":
|
||
return "FAILED"
|
||
|
||
# 检查关键属性
|
||
attrs = smart_data.get("attributes", {})
|
||
|
||
critical_attrs = {
|
||
'5 Reallocated_Sector_Ct': 'reallocated_sectors',
|
||
'197 Current_Pending_Sector': 'pending_sectors',
|
||
'198 Offline_Uncorrectable': 'offline_uncorrectable',
|
||
'196 Reallocation_Event_Count': 'reallocation_events'
|
||
}
|
||
|
||
for attr_name, description in critical_attrs.items():
|
||
if attr_name in attrs:
|
||
raw_value = attrs[attr_name].get('raw_value', '0')
|
||
value = safe_int(raw_value.split()[0])
|
||
if value > 0:
|
||
return "WARNING"
|
||
|
||
# 检查温度
|
||
for temp_attr in ['194 Temperature_Celsius', '190 Airflow_Temperature_Cel']:
|
||
if temp_attr in attrs:
|
||
temp = attrs[temp_attr].get('value', 0)
|
||
if temp > 60: # 温度阈值
|
||
return "WARNING"
|
||
|
||
return "PASSED"
|
||
|
||
|
||
def check_raid_status() -> Dict[str, Any]:
|
||
"""检查 RAID 阵列状态。"""
|
||
result = {
|
||
"raid_available": False,
|
||
"controllers": [],
|
||
"arrays": []
|
||
}
|
||
|
||
# 检查软件 RAID (mdadm)
|
||
if check_command_exists('mdadm'):
|
||
try:
|
||
_, stdout, _ = execute_command(
|
||
['mdadm', '--detail', '--scan'],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
|
||
if stdout.strip():
|
||
result["software_raid"] = True
|
||
result["mdadm_config"] = stdout.strip()
|
||
|
||
# 获取详细信息
|
||
_, detail, _ = execute_command(
|
||
['cat', '/proc/mdstat'],
|
||
check_returncode=False, timeout=5
|
||
)
|
||
result["mdstat"] = detail
|
||
|
||
# 解析每个阵列
|
||
for line in detail.split('\n'):
|
||
if line.startswith('md'):
|
||
parts = line.split()
|
||
array_info = {
|
||
"name": parts[0],
|
||
"status": "active" if "active" in line else "inactive"
|
||
}
|
||
|
||
# 检查是否有降级
|
||
if '_' in line or 'recovery' in line:
|
||
array_info["degraded"] = True
|
||
result["status"] = "warning"
|
||
|
||
result["arrays"].append(array_info)
|
||
|
||
except:
|
||
pass
|
||
|
||
# 检查硬件 RAID (MegaCli/storcli)
|
||
if check_command_exists('storcli'):
|
||
try:
|
||
_, stdout, _ = execute_command(
|
||
['storcli', '/c0', 'show'],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
result["hardware_raid"] = True
|
||
result["controller_type"] = "LSI/Broadcom"
|
||
result["storcli_output"] = stdout[:500] # 保存部分输出
|
||
except:
|
||
pass
|
||
elif check_command_exists('MegaCli'):
|
||
try:
|
||
_, stdout, _ = execute_command(
|
||
['MegaCli', '-AdpAllInfo', '-aALL'],
|
||
check_returncode=False, timeout=10
|
||
)
|
||
result["hardware_raid"] = True
|
||
result["controller_type"] = "LSI"
|
||
result["megacli_output"] = stdout[:500]
|
||
except:
|
||
pass
|
||
|
||
return result
|
||
|
||
|
||
def get_io_statistics() -> Dict[str, Any]:
|
||
"""获取 I/O 统计信息。"""
|
||
result = {}
|
||
|
||
# 从 /proc/diskstats 获取
|
||
try:
|
||
with open('/proc/diskstats', 'r') as f:
|
||
for line in f:
|
||
parts = line.split()
|
||
if len(parts) >= 14:
|
||
device = parts[2]
|
||
# 只关注物理磁盘
|
||
if device.startswith(('sd', 'hd', 'nvme', 'vd')) and not device[-1].isdigit():
|
||
result[device] = {
|
||
"reads_completed": safe_int(parts[3]),
|
||
"reads_merged": safe_int(parts[4]),
|
||
"sectors_read": safe_int(parts[5]),
|
||
"time_reading_ms": safe_int(parts[6]),
|
||
"writes_completed": safe_int(parts[7]),
|
||
"writes_merged": safe_int(parts[8]),
|
||
"sectors_written": safe_int(parts[9]),
|
||
"time_writing_ms": safe_int(parts[10]),
|
||
"ios_in_progress": safe_int(parts[11]),
|
||
"time_doing_ios_ms": safe_int(parts[12]),
|
||
"weighted_time_ios_ms": safe_int(parts[13])
|
||
}
|
||
except:
|
||
pass
|
||
|
||
return result
|
||
|
||
|
||
@require_root
|
||
def run_io_test(device_path: str, test_size_mb: int = 100) -> Dict[str, Any]:
|
||
"""
|
||
运行简单的 I/O 性能测试。
|
||
|
||
Args:
|
||
device_path: 设备路径
|
||
test_size_mb: 测试大小（MB）
|
||
|
||
Returns:
|
||
Dict[str, Any]: 测试结果
|
||
"""
|
||
result = {
|
||
"passed": False,
|
||
"device": device_path,
|
||
"test_size_mb": test_size_mb,
|
||
"read_speed_mbps": None,
|
||
"write_speed_mbps": None,
|
||
"errors": []
|
||
}
|
||
|
||
# 使用 fio 进行测试
|
||
if check_command_exists('fio'):
|
||
try:
|
||
import tempfile
|
||
|
||
with tempfile.NamedTemporaryFile(mode='w', suffix='.fio', delete=False) as f:
|
||
fio_config = f"""
|
||
[global]
|
||
directory=/tmp
|
||
filename=serverguard_test
|
||
direct=1
|
||
size={test_size_mb}M
|
||
unlink=1
|
||
|
||
[seq_read]
|
||
stonewall
|
||
rw=read
|
||
bs=1M
|
||
|
||
[seq_write]
|
||
stonewall
|
||
rw=write
|
||
bs=1M
|
||
"""
|
||
f.write(fio_config)
|
||
fio_file = f.name
|
||
|
||
try:
|
||
_, stdout, stderr = execute_command(
|
||
['fio', fio_file, '--output-format=json'],
|
||
timeout=120,
|
||
check_returncode=False
|
||
)
|
||
|
||
data = json.loads(stdout)
|
||
|
||
for job in data.get('jobs', []):
|
||
job_name = job.get('jobname', '')
|
||
read_bw = job.get('read', {}).get('bw', 0) / 1024 # 转换为 MB/s
|
||
write_bw = job.get('write', {}).get('bw', 0) / 1024
|
||
|
||
if 'read' in job_name.lower() and read_bw > 0:
|
||
result["read_speed_mbps"] = round(read_bw, 2)
|
||
if 'write' in job_name.lower() and write_bw > 0:
|
||
result["write_speed_mbps"] = round(write_bw, 2)
|
||
|
||
result["passed"] = True
|
||
|
||
finally:
|
||
os.unlink(fio_file)
|
||
|
||
except Exception as e:
|
||
result["errors"].append(str(e))
|
||
else:
|
||
result["errors"].append("fio 未安装")
|
||
|
||
return result
|
||
|
||
|
||
if __name__ == '__main__':
|
||
import json
|
||
print(json.dumps(run_storage_check(), indent=2, ensure_ascii=False))
|