docs: 完善数据模型设计
- 重命名 log_file 为 test_scenario（测试场景） - 拆分维度表：tx_panel、tx_hardware、tx_software、rx_type - 采用星型模型设计，消除数据冗余 - 为所有表添加软删除字段 is_deleted - 更新关系说明，包含全局查询过滤器配置 - 更新 ER 图和查询示例 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
ssss
205
docs/architecture/数据处理流程.md
Normal file
205
docs/architecture/数据处理流程.md
Normal file
@@ -0,0 +1,205 @@
|
||||
# 数据处理流程
|
||||
|
||||
> 所属模块：[架构概览](./概览.md)
|
||||
|
||||
---
|
||||
|
||||
## 1. 完整解析流程
|
||||
|
||||
```mermaid
|
||||
sequenceDiagram
|
||||
participant CLI as CLI Entry
|
||||
participant PS as ParseService
|
||||
participant LR as LogFileReader
|
||||
participant QP as QfodParser
|
||||
participant PP as PlossParser
|
||||
participant EHS as ErrorHandlingService
|
||||
participant DB as Database
|
||||
|
||||
CLI->>LR: 扫描日志目录
|
||||
LR-->>CLI: 返回文件列表
|
||||
|
||||
loop 每个日志文件
|
||||
CLI->>PS: ParseFileAsync(filePath)
|
||||
|
||||
PS->>DB: INSERT log_file (status=processing)
|
||||
DB-->>PS: 返回 file_id
|
||||
|
||||
PS->>LR: 读取文件内容
|
||||
LR-->>PS: 返回行列表
|
||||
|
||||
loop 逐行遍历
|
||||
alt 是 Qfod 行
|
||||
PS->>QP: Parse(line)
|
||||
QP-->>PS: ParseResult
|
||||
else 是 Ploss 行
|
||||
PS->>PP: Parse(line)
|
||||
PP-->>PS: ParseResult
|
||||
end
|
||||
|
||||
alt 解析成功
|
||||
PS->>PS: 加入记录列表
|
||||
else 解析失败
|
||||
PS->>EHS: RecordError(line, error)
|
||||
end
|
||||
end
|
||||
|
||||
PS->>DB: BATCH INSERT records
|
||||
PS->>DB: BATCH INSERT errors
|
||||
PS->>DB: UPDATE log_file (counts, status=completed)
|
||||
|
||||
PS-->>CLI: 返回 ParseReport
|
||||
end
|
||||
|
||||
CLI-->>CLI: 输出解析报告
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 2. ParseService 实现
|
||||
|
||||
```csharp
|
||||
// Services/ParseService.cs
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using WCTDataMiner.Data;
|
||||
using WCTDataMiner.Models;
|
||||
using WCTDataMiner.Parsers;
|
||||
|
||||
namespace WCTDataMiner.Services;
|
||||
|
||||
public class ParseService
|
||||
{
|
||||
private readonly WctMinerDbContext _context;
|
||||
private readonly QfodParser _qfodParser;
|
||||
private readonly PlossParser _plossParser;
|
||||
private readonly ErrorHandlingService _errorService;
|
||||
private readonly ILogger<ParseService> _logger;
|
||||
private readonly int _batchSize;
|
||||
|
||||
public ParseService(
|
||||
WctMinerDbContext context,
|
||||
QfodParser qfodParser,
|
||||
PlossParser plossParser,
|
||||
ErrorHandlingService errorService,
|
||||
ILogger<ParseService> logger,
|
||||
IConfiguration config)
|
||||
{
|
||||
_context = context;
|
||||
_qfodParser = qfodParser;
|
||||
_plossParser = plossParser;
|
||||
_errorService = errorService;
|
||||
_logger = logger;
|
||||
_batchSize = config.GetValue("Parser:BatchSize", 1000);
|
||||
}
|
||||
|
||||
public async Task<ParseReport> ParseFileAsync(string filePath)
|
||||
{
|
||||
var fileInfo = new FileInfo(filePath);
|
||||
var logFile = await CreateLogFileRecordAsync(fileInfo);
|
||||
|
||||
var lines = await File.ReadAllLinesAsync(filePath);
|
||||
var qfodRecords = new List<QfodRecord>();
|
||||
var plossRecords = new List<PlossRecord>();
|
||||
|
||||
for (int i = 0; i < lines.Length; i++)
|
||||
{
|
||||
var line = lines[i];
|
||||
var lineNumber = i + 1;
|
||||
|
||||
if (_qfodParser.CanParse(line))
|
||||
{
|
||||
var result = _qfodParser.Parse(line, lineNumber, logFile.Id);
|
||||
if (result.IsSuccess)
|
||||
qfodRecords.Add(result.Record!);
|
||||
else
|
||||
_errorService.RecordError(logFile.Id, lineNumber, line, result.ErrorType!, result.ErrorMessage!);
|
||||
}
|
||||
else if (_plossParser.CanParse(line))
|
||||
{
|
||||
var result = _plossParser.Parse(line, lineNumber, logFile.Id);
|
||||
if (result.IsSuccess)
|
||||
plossRecords.Add(result.Record!);
|
||||
else
|
||||
_errorService.RecordError(logFile.Id, lineNumber, line, result.ErrorType!, result.ErrorMessage!);
|
||||
}
|
||||
}
|
||||
|
||||
// 批量保存
|
||||
await BatchInsertAsync(qfodRecords, plossRecords, _errorService.GetErrors());
|
||||
|
||||
// 更新统计
|
||||
await UpdateLogFileStatsAsync(logFile.Id, qfodRecords.Count, plossRecords.Count, _errorService.ErrorCount);
|
||||
|
||||
return new ParseReport(fileInfo.Name, qfodRecords.Count, plossRecords.Count, _errorService.ErrorCount);
|
||||
}
|
||||
|
||||
private async Task<LogFile> CreateLogFileRecordAsync(FileInfo fileInfo)
|
||||
{
|
||||
var logFile = new LogFile
|
||||
{
|
||||
FileName = fileInfo.Name,
|
||||
FilePath = fileInfo.FullName,
|
||||
FileSize = fileInfo.Length,
|
||||
Status = ParseStatus.Processing
|
||||
};
|
||||
|
||||
_context.LogFiles.Add(logFile);
|
||||
await _context.SaveChangesAsync();
|
||||
return logFile;
|
||||
}
|
||||
|
||||
private async Task BatchInsertAsync(
|
||||
List<QfodRecord> qfodRecords,
|
||||
List<PlossRecord> plossRecords,
|
||||
List<ParseError> errors)
|
||||
{
|
||||
await using var transaction = await _context.Database.BeginTransactionAsync();
|
||||
|
||||
try
|
||||
{
|
||||
// 批量插入 QfodRecord
|
||||
foreach (var batch in qfodRecords.Chunk(_batchSize))
|
||||
{
|
||||
_context.QfodRecords.AddRange(batch);
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// 批量插入 PlossRecord
|
||||
foreach (var batch in plossRecords.Chunk(_batchSize))
|
||||
{
|
||||
_context.PlossRecords.AddRange(batch);
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// 批量插入 ParseError
|
||||
foreach (var batch in errors.Chunk(_batchSize))
|
||||
{
|
||||
_context.ParseErrors.AddRange(batch);
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await transaction.CommitAsync();
|
||||
}
|
||||
catch
|
||||
{
|
||||
await transaction.RollbackAsync();
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateLogFileStatsAsync(Guid logFileId, int qfodCount, int plossCount, int errorCount)
|
||||
{
|
||||
var logFile = await _context.LogFiles.FindAsync(logFileId);
|
||||
if (logFile != null)
|
||||
{
|
||||
logFile.QfodCount = qfodCount;
|
||||
logFile.PlossCount = plossCount;
|
||||
logFile.ErrorCount = errorCount;
|
||||
logFile.Status = ParseStatus.Completed;
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public record ParseReport(string FileName, int QfodCount, int PlossCount, int ErrorCount);
|
||||
```
|
||||
Reference in New Issue
Block a user