feat: 实现 FOD 日志数据采集 CLI 应用
This commit is contained in:
ssss
@@ -10,47 +10,52 @@
|
||||
sequenceDiagram
|
||||
participant CLI as CLI Entry
|
||||
participant PS as ParseService
|
||||
participant LR as LogFileReader
|
||||
participant SS as ScenarioService
|
||||
participant FNP as FileNameParser
|
||||
participant QP as QfodParser
|
||||
participant PP as PlossParser
|
||||
participant EHS as ErrorHandlingService
|
||||
participant DB as Database
|
||||
participant Log as Serilog
|
||||
|
||||
CLI->>PS: 扫描日志目录
|
||||
PS-->>CLI: 返回文件列表
|
||||
|
||||
CLI->>LR: 扫描日志目录
|
||||
LR-->>CLI: 返回文件列表
|
||||
|
||||
loop 每个日志文件
|
||||
CLI->>PS: ParseFileAsync(filePath)
|
||||
|
||||
PS->>DB: INSERT log_file (status=processing)
|
||||
DB-->>PS: 返回 file_id
|
||||
|
||||
PS->>LR: 读取文件内容
|
||||
LR-->>PS: 返回行列表
|
||||
|
||||
|
||||
PS->>FNP: 解析文件名
|
||||
FNP-->>PS: 场景信息（面板、版本、RX类型等）
|
||||
|
||||
PS->>SS: GetOrCreateScenarioAsync(场景信息)
|
||||
SS->>DB: 查询/创建维度记录
|
||||
SS->>DB: 查询/创建场景记录
|
||||
SS-->>PS: 返回 TestScenario
|
||||
|
||||
PS->>PS: 读取文件内容
|
||||
|
||||
loop 逐行遍历
|
||||
alt 是 Qfod 行
|
||||
PS->>QP: Parse(line)
|
||||
PS->>QP: Parse(line, scenarioId)
|
||||
QP-->>PS: ParseResult
|
||||
else 是 Ploss 行
|
||||
PS->>PP: Parse(line)
|
||||
PS->>PP: Parse(line, scenarioId)
|
||||
PP-->>PS: ParseResult
|
||||
end
|
||||
|
||||
|
||||
alt 解析成功
|
||||
PS->>PS: 加入记录列表
|
||||
else 解析失败
|
||||
PS->>EHS: RecordError(line, error)
|
||||
PS->>Log: LogError(line, error)
|
||||
end
|
||||
end
|
||||
|
||||
PS->>DB: BATCH INSERT records
|
||||
PS->>DB: BATCH INSERT errors
|
||||
PS->>DB: UPDATE log_file (counts, status=completed)
|
||||
|
||||
|
||||
PS->>DB: BATCH INSERT QfodRecords
|
||||
PS->>DB: BATCH INSERT PlossRecords
|
||||
PS->>DB: UPDATE scenario (qfod_count, ploss_count)
|
||||
|
||||
PS-->>CLI: 返回 ParseReport
|
||||
end
|
||||
|
||||
|
||||
CLI-->>CLI: 输出解析报告
|
||||
```
|
||||
|
||||
@@ -61,6 +66,7 @@ sequenceDiagram
|
||||
```csharp
|
||||
// Services/ParseService.cs
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Serilog;
|
||||
using WCTDataMiner.Data;
|
||||
using WCTDataMiner.Models;
|
||||
using WCTDataMiner.Parsers;
|
||||
@@ -70,36 +76,47 @@ namespace WCTDataMiner.Services;
|
||||
public class ParseService
|
||||
{
|
||||
private readonly WctMinerDbContext _context;
|
||||
private readonly ScenarioService _scenarioService;
|
||||
private readonly QfodParser _qfodParser;
|
||||
private readonly PlossParser _plossParser;
|
||||
private readonly ErrorHandlingService _errorService;
|
||||
private readonly ILogger<ParseService> _logger;
|
||||
private readonly FileNameParser _fileNameParser;
|
||||
private readonly int _batchSize;
|
||||
|
||||
public ParseService(
|
||||
WctMinerDbContext context,
|
||||
ScenarioService scenarioService,
|
||||
QfodParser qfodParser,
|
||||
PlossParser plossParser,
|
||||
ErrorHandlingService errorService,
|
||||
ILogger<ParseService> logger,
|
||||
FileNameParser fileNameParser,
|
||||
IConfiguration config)
|
||||
{
|
||||
_context = context;
|
||||
_scenarioService = scenarioService;
|
||||
_qfodParser = qfodParser;
|
||||
_plossParser = plossParser;
|
||||
_errorService = errorService;
|
||||
_logger = logger;
|
||||
_fileNameParser = fileNameParser;
|
||||
_batchSize = config.GetValue("Parser:BatchSize", 1000);
|
||||
}
|
||||
|
||||
public async Task<ParseReport> ParseFileAsync(string filePath)
|
||||
{
|
||||
var fileInfo = new FileInfo(filePath);
|
||||
var logFile = await CreateLogFileRecordAsync(fileInfo);
|
||||
|
||||
|
||||
// 解析文件名获取场景信息
|
||||
var scenarioInfo = _fileNameParser.Parse(fileInfo.Name);
|
||||
if (scenarioInfo == null)
|
||||
{
|
||||
Log.Error("无法解析文件名: {FileName}", fileInfo.Name);
|
||||
return new ParseReport(fileInfo.Name, 0, 0, 1, "文件名格式不匹配");
|
||||
}
|
||||
|
||||
// 获取或创建测试场景
|
||||
var scenario = await _scenarioService.GetOrCreateScenarioAsync(scenarioInfo);
|
||||
|
||||
var lines = await File.ReadAllLinesAsync(filePath);
|
||||
var qfodRecords = new List<QfodRecord>();
|
||||
var plossRecords = new List<PlossRecord>();
|
||||
int errorCount = 0;
|
||||
|
||||
for (int i = 0; i < lines.Length; i++)
|
||||
{
|
||||
@@ -108,53 +125,45 @@ public class ParseService
|
||||
|
||||
if (_qfodParser.CanParse(line))
|
||||
{
|
||||
var result = _qfodParser.Parse(line, lineNumber, logFile.Id);
|
||||
var result = _qfodParser.Parse(line, scenario.Id);
|
||||
if (result.IsSuccess)
|
||||
qfodRecords.Add(result.Record!);
|
||||
else
|
||||
_errorService.RecordError(logFile.Id, lineNumber, line, result.ErrorType!, result.ErrorMessage!);
|
||||
{
|
||||
Log.Error("Qfod解析失败 [{Line}] {Error}: {Message}",
|
||||
lineNumber, result.ErrorType, result.ErrorMessage);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
else if (_plossParser.CanParse(line))
|
||||
{
|
||||
var result = _plossParser.Parse(line, lineNumber, logFile.Id);
|
||||
var result = _plossParser.Parse(line, scenario.Id);
|
||||
if (result.IsSuccess)
|
||||
plossRecords.Add(result.Record!);
|
||||
else
|
||||
_errorService.RecordError(logFile.Id, lineNumber, line, result.ErrorType!, result.ErrorMessage!);
|
||||
{
|
||||
Log.Error("Ploss解析失败 [{Line}] {Error}: {Message}",
|
||||
lineNumber, result.ErrorType, result.ErrorMessage);
|
||||
errorCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 批量保存
|
||||
await BatchInsertAsync(qfodRecords, plossRecords, _errorService.GetErrors());
|
||||
|
||||
// 更新统计
|
||||
await UpdateLogFileStatsAsync(logFile.Id, qfodRecords.Count, plossRecords.Count, _errorService.ErrorCount);
|
||||
await BatchInsertAsync(qfodRecords, plossRecords);
|
||||
|
||||
return new ParseReport(fileInfo.Name, qfodRecords.Count, plossRecords.Count, _errorService.ErrorCount);
|
||||
}
|
||||
// 更新场景统计
|
||||
await UpdateScenarioStatsAsync(scenario.Id, qfodRecords.Count, plossRecords.Count);
|
||||
|
||||
private async Task<LogFile> CreateLogFileRecordAsync(FileInfo fileInfo)
|
||||
{
|
||||
var logFile = new LogFile
|
||||
{
|
||||
FileName = fileInfo.Name,
|
||||
FilePath = fileInfo.FullName,
|
||||
FileSize = fileInfo.Length,
|
||||
Status = ParseStatus.Processing
|
||||
};
|
||||
|
||||
_context.LogFiles.Add(logFile);
|
||||
await _context.SaveChangesAsync();
|
||||
return logFile;
|
||||
return new ParseReport(fileInfo.Name, qfodRecords.Count, plossRecords.Count, errorCount, null);
|
||||
}
|
||||
|
||||
private async Task BatchInsertAsync(
|
||||
List<QfodRecord> qfodRecords,
|
||||
List<PlossRecord> plossRecords,
|
||||
List<ParseError> errors)
|
||||
List<PlossRecord> plossRecords)
|
||||
{
|
||||
await using var transaction = await _context.Database.BeginTransactionAsync();
|
||||
|
||||
|
||||
try
|
||||
{
|
||||
// 批量插入 QfodRecord
|
||||
@@ -171,13 +180,6 @@ public class ParseService
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
// 批量插入 ParseError
|
||||
foreach (var batch in errors.Chunk(_batchSize))
|
||||
{
|
||||
_context.ParseErrors.AddRange(batch);
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
|
||||
await transaction.CommitAsync();
|
||||
}
|
||||
catch
|
||||
@@ -187,19 +189,28 @@ public class ParseService
|
||||
}
|
||||
}
|
||||
|
||||
private async Task UpdateLogFileStatsAsync(Guid logFileId, int qfodCount, int plossCount, int errorCount)
|
||||
private async Task UpdateScenarioStatsAsync(Guid scenarioId, int qfodCount, int plossCount)
|
||||
{
|
||||
var logFile = await _context.LogFiles.FindAsync(logFileId);
|
||||
if (logFile != null)
|
||||
// 使用累加更新（+=），支持同一场景的多次解析
|
||||
// 若需幂等解析，应在 ParseFileAsync 开始时检查场景是否已处理
|
||||
var scenario = await _context.TestScenarios.FindAsync(scenarioId);
|
||||
if (scenario != null)
|
||||
{
|
||||
logFile.QfodCount = qfodCount;
|
||||
logFile.PlossCount = plossCount;
|
||||
logFile.ErrorCount = errorCount;
|
||||
logFile.Status = ParseStatus.Completed;
|
||||
scenario.QfodCount += qfodCount;
|
||||
scenario.PlossCount += plossCount;
|
||||
await _context.SaveChangesAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public record ParseReport(string FileName, int QfodCount, int PlossCount, int ErrorCount);
|
||||
public record ParseReport(string FileName, int QfodCount, int PlossCount, int ErrorCount, string? ErrorMessage);
|
||||
|
||||
public record ScenarioInfo(
|
||||
string TxPanel,
|
||||
string TxHardware,
|
||||
string TxSoftware,
|
||||
string RxType,
|
||||
string? TestPurpose,
|
||||
DateOnly TestDate,
|
||||
int TestSequence);
|
||||
```
|
||||
Reference in New Issue
Block a user