WCTDataMiner/docs/architecture/数据处理流程.md

# 数据处理流程

> 所属模块：[架构概览](./概览.md)

---

## 1. 完整解析流程

```mermaid
sequenceDiagram
    participant CLI as CLI Entry
    participant PS as ParseService
    participant LR as LogFileReader
    participant QP as QfodParser
    participant PP as PlossParser
    participant EHS as ErrorHandlingService
    participant DB as Database

    CLI->>LR: 扫描日志目录
    LR-->>CLI: 返回文件列表
    
    loop 每个日志文件
        CLI->>PS: ParseFileAsync(filePath)
        
        PS->>DB: INSERT log_file (status=processing)
        DB-->>PS: 返回 file_id
        
        PS->>LR: 读取文件内容
        LR-->>PS: 返回行列表
        
        loop 逐行遍历
            alt 是 Qfod 行
                PS->>QP: Parse(line)
                QP-->>PS: ParseResult
            else 是 Ploss 行
                PS->>PP: Parse(line)
                PP-->>PS: ParseResult
            end
            
            alt 解析成功
                PS->>PS: 加入记录列表
            else 解析失败
                PS->>EHS: RecordError(line, error)
            end
        end
        
        PS->>DB: BATCH INSERT records
        PS->>DB: BATCH INSERT errors
        PS->>DB: UPDATE log_file (counts, status=completed)
        
        PS-->>CLI: 返回 ParseReport
    end
    
    CLI-->>CLI: 输出解析报告
```

---

## 2. ParseService 实现

```csharp
// Services/ParseService.cs
using Microsoft.EntityFrameworkCore;
using WCTDataMiner.Data;
using WCTDataMiner.Models;
using WCTDataMiner.Parsers;

namespace WCTDataMiner.Services;

public class ParseService
{
    private readonly WctMinerDbContext _context;
    private readonly QfodParser _qfodParser;
    private readonly PlossParser _plossParser;
    private readonly ErrorHandlingService _errorService;
    private readonly ILogger<ParseService> _logger;
    private readonly int _batchSize;

    public ParseService(
        WctMinerDbContext context,
        QfodParser qfodParser,
        PlossParser plossParser,
        ErrorHandlingService errorService,
        ILogger<ParseService> logger,
        IConfiguration config)
    {
        _context = context;
        _qfodParser = qfodParser;
        _plossParser = plossParser;
        _errorService = errorService;
        _logger = logger;
        _batchSize = config.GetValue("Parser:BatchSize", 1000);
    }

    public async Task<ParseReport> ParseFileAsync(string filePath)
    {
        var fileInfo = new FileInfo(filePath);
        var logFile = await CreateLogFileRecordAsync(fileInfo);
        
        var lines = await File.ReadAllLinesAsync(filePath);
        var qfodRecords = new List<QfodRecord>();
        var plossRecords = new List<PlossRecord>();

        for (int i = 0; i < lines.Length; i++)
        {
            var line = lines[i];
            var lineNumber = i + 1;

            if (_qfodParser.CanParse(line))
            {
                var result = _qfodParser.Parse(line, lineNumber, logFile.Id);
                if (result.IsSuccess)
                    qfodRecords.Add(result.Record!);
                else
                    _errorService.RecordError(logFile.Id, lineNumber, line, result.ErrorType!, result.ErrorMessage!);
            }
            else if (_plossParser.CanParse(line))
            {
                var result = _plossParser.Parse(line, lineNumber, logFile.Id);
                if (result.IsSuccess)
                    plossRecords.Add(result.Record!);
                else
                    _errorService.RecordError(logFile.Id, lineNumber, line, result.ErrorType!, result.ErrorMessage!);
            }
        }

        // 批量保存
        await BatchInsertAsync(qfodRecords, plossRecords, _errorService.GetErrors());
        
        // 更新统计
        await UpdateLogFileStatsAsync(logFile.Id, qfodRecords.Count, plossRecords.Count, _errorService.ErrorCount);

        return new ParseReport(fileInfo.Name, qfodRecords.Count, plossRecords.Count, _errorService.ErrorCount);
    }

    private async Task<LogFile> CreateLogFileRecordAsync(FileInfo fileInfo)
    {
        var logFile = new LogFile
        {
            FileName = fileInfo.Name,
            FilePath = fileInfo.FullName,
            FileSize = fileInfo.Length,
            Status = ParseStatus.Processing
        };
        
        _context.LogFiles.Add(logFile);
        await _context.SaveChangesAsync();
        return logFile;
    }

    private async Task BatchInsertAsync(
        List<QfodRecord> qfodRecords,
        List<PlossRecord> plossRecords,
        List<ParseError> errors)
    {
        await using var transaction = await _context.Database.BeginTransactionAsync();
        
        try
        {
            // 批量插入 QfodRecord
            foreach (var batch in qfodRecords.Chunk(_batchSize))
            {
                _context.QfodRecords.AddRange(batch);
                await _context.SaveChangesAsync();
            }

            // 批量插入 PlossRecord
            foreach (var batch in plossRecords.Chunk(_batchSize))
            {
                _context.PlossRecords.AddRange(batch);
                await _context.SaveChangesAsync();
            }

            // 批量插入 ParseError
            foreach (var batch in errors.Chunk(_batchSize))
            {
                _context.ParseErrors.AddRange(batch);
                await _context.SaveChangesAsync();
            }

            await transaction.CommitAsync();
        }
        catch
        {
            await transaction.RollbackAsync();
            throw;
        }
    }

    private async Task UpdateLogFileStatsAsync(Guid logFileId, int qfodCount, int plossCount, int errorCount)
    {
        var logFile = await _context.LogFiles.FindAsync(logFileId);
        if (logFile != null)
        {
            logFile.QfodCount = qfodCount;
            logFile.PlossCount = plossCount;
            logFile.ErrorCount = errorCount;
            logFile.Status = ParseStatus.Completed;
            await _context.SaveChangesAsync();
        }
    }
}

public record ParseReport(string FileName, int QfodCount, int PlossCount, int ErrorCount);
```