fix(aihit-sync): Оптимизация GetDeltaAsync и атомарность сохранения

- GetDeltaAsync: фильтрация по relevantParentNames вместо выгрузки всех связей из БД - SyncAsync: атомарность через ChangeTracker + CommitAsync без явной транзакции
fix(aihitRelationshipsSyncer): критические исправления RelationshipsSyncService
2026-07-15 14:40:10 +10:00 · 2026-07-15 14:04:35 +10:00 · 2026-07-14 14:55:49 +10:00
1 changed files with 251 additions and 226 deletions
--- a/PARR.AIHITRelationshipsSyncer/Services/Implementations/RelationshipsSyncService.cs
+++ b/PARR.AIHITRelationshipsSyncer/Services/Implementations/RelationshipsSyncService.cs
@@ -1,236 +1,261 @@
-using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Logging;
 using PARR.AIHITRelationshipsSyncer.Models;
 using PARR.AIHITRelationshipsSyncer.Services.Interfaces;
 using PARR.AIHITRelationshipsSyncer.Settings;
 using PARR.Core.Repositories.Interfaces.Unit;
 using PARR.Domain.Entities.Unit;
 using PARR.Domain.Settings;
-namespace PARR.AIHITRelationshipsSyncer.Services.Implementations
+namespace PARR.AIHITRelationshipsSyncer.Services.Implementations;
 internal class RelationshipsSyncService : IRelationshipsSyncService
 {
-    internal class RelationshipsSyncService : IRelationshipsSyncService
+    private readonly ILogger<RelationshipsSyncService> _logger;
    private readonly IUnitRepository _unitRepository;
    private readonly SettingsFromDb _settingsFromDb;
    public RelationshipsSyncService(
        ILogger<RelationshipsSyncService> logger,
        IUnitRepository unitRepository,
        SettingsFromDb settingsFromDb)
    {
-        private readonly ILogger<RelationshipsSyncService> logger;
+        _logger = logger;
-        private readonly IUnitRepository unitService;
+        _unitRepository = unitRepository;
-        private readonly WorkerSettings workerSettings;
+        _settingsFromDb = settingsFromDb;
        public RelationshipsSyncService(
            ILogger<RelationshipsSyncService> logger,
            IUnitRepository unitService,
            WorkerSettings workerSettings
            )
        {
            this.logger = logger;
            this.unitService = unitService;
            this.workerSettings = workerSettings;
        }
        public async Task SyncAsync(List<AihitData> aihitdata)
        {
            var validPairs = ValidateInput(aihitdata);
            if (!validPairs.Any()) return;
            var sourceUnitNames = ExtractUnitNames(validPairs);
            var existingUnits = await GetOrCreateUnitsAsync(sourceUnitNames);
            var (toRemove, toAdd) = await GetDeltaAsync(validPairs);
            await RemoveRelationshipsAsync(toRemove);
            await AddRelationshipsAsync(toAdd, existingUnits);
            await FinalCommitAsync(toRemove.Count, toAdd.Count);
        }
        private List<(string Parent, string Child)> ValidateInput(List<AihitData> data)
        {
            if (data == null || data.Count == 0)
            {
                logger.LogWarning("Получены пустые или null данные из АИХ ИТ. Пропускаю синхронизацию");
                return null!;
            }
            var validPairs = data
                            .Where(t => !string.IsNullOrWhiteSpace(t.ParentName) && !string.IsNullOrWhiteSpace(t.ChildName))
                            .Where(t => t.ParentName != t.ChildName)
                            .Select(t => (
                                Parent: t.ParentName!.Trim().ToUpperInvariant(),
                                Child: t.ChildName!.Trim().ToUpperInvariant()
                            ))
                            .Distinct()
                            .ToList();
            if (validPairs.Count == 0)
            {
                logger.LogWarning("После нормализации не осталось валидных связей. Пропускаю синхронизацию");
                return new();
            }
            logger.LogInformation("Получено {Count} валидных связей", validPairs.Count);
            return validPairs;
        }
        private static List<string> ExtractUnitNames(List<(string Parent, string Child)> pairs)
        {
            return pairs
                    .SelectMany(x => new[] { x.Parent, x.Child })
                    .Distinct()
                    .ToList();
        }
        private async Task<Dictionary<string, Unit>> GetOrCreateUnitsAsync(List<string> names)
        {
            var existing = await unitService.Get()
                                    .AsNoTracking()
                                    .ToDictionaryAsync(u => u.Name, StringComparer.Ordinal);
            var missing = names.Except(existing.Keys).ToList();
            if (!missing.Any()) return existing;
            logger.LogInformation("Создание отсутствующих Unit:{Count}", missing.Count);
            foreach (var name in missing)
            {
                var unit = new Unit
                {
                    Id = Guid.NewGuid(),
                    Name = name,
                    DateCreated = DateTimeOffset.UtcNow
                };
                if (await unitService.CreateAsync(unit))
                {
                    logger.LogInformation("Создан Unit: {Name}", name);
                    existing[name] = unit;
                }
                else
                    logger.LogError("Не удалось создать Unit: {Name}", name);
            }
            await unitService.CommitAsync();
            return existing;
        }
        private async Task<(List<ExistingRelationship> ToRemove, List<(string, string)> ToAdd)> GetDeltaAsync(List<(string Parent, string Child)> sourcePairs)
        {
            var allExistings = await unitService.Get()
                                    .AsNoTracking()
                                    .Include(u => u.ChildUnits).ThenInclude(r => r.ChildUnit)
                                    .Where(u => u.ChildUnits.Any())
                                    .SelectMany(u => u.ChildUnits, (parent, relationship) => new ExistingRelationship
                                    {
                                        ParentName = parent.Name,
                                        ChildName = relationship.ChildUnit!.Name,
                                        Relationship = relationship
                                    }
                                    )
                                    .ToListAsync();
            var sourceSet = new HashSet<(string, string)>(sourcePairs);
            var existingSet = new HashSet<(string, string)>(allExistings.Select(x => (x.ParentName, x.ChildName)));
            var toRemove = allExistings.Where(r => !sourceSet.Contains((r.ParentName, r.ChildName))).ToList();
            var toAdd = sourcePairs.Where(p => !existingSet.Contains(p)).ToList();
            logger.LogInformation("Связей для удаления: {ToRemove}, для добавления: {ToAdd}", toRemove.Count, toAdd.Count);
            return (toRemove, toAdd);
        }
        private async Task RemoveRelationshipsAsync(List<ExistingRelationship> toRemove)
        {
            if (!toRemove.Any()) return;
            var parentNames = toRemove.Select(r => r.ParentName).Distinct().ToList();
            var parents = await unitService.Get()
                                .Include(u => u.ChildUnits)
                                .Where(u => parentNames.Contains(u.Name))
                                .ToDictionaryAsync(u => u.Name, StringComparer.Ordinal);
            foreach (var rel in toRemove)
            {
                if (!parents.TryGetValue(rel.ParentName, out var parent))
                {
                    logger.LogWarning("Родитель '{Parent}' не найден при попытке удаления связи - '{Child}'", rel.ParentName, rel.ChildName);
                    continue;
                }
                var relationshipToRemove = parent.ChildUnits
                                                .FirstOrDefault(r => r.ChildUnitId == rel.Relationship.ChildUnitId);
                if (relationshipToRemove != null)
                {
                    logger.LogInformation("Удаление связи: '{Parent}' - '{Child}'", rel.ParentName, rel.ChildName);
                    parent.ChildUnits.Remove(relationshipToRemove);
                }
                else
                {
                    logger.LogWarning("Связь '{Parent}'-'{Child}' не найдена в коллекции ChildUnits для удаления", rel.ParentName, rel.ChildName);
                }
            }
        }
        private async Task AddRelationshipsAsync(List<(string Parent, string Child)> toAdd, Dictionary<string, Unit> allUnits)
        {
            if (!toAdd.Any()) return;
            var parentNames = toAdd.Select(x => x.Parent).Distinct().ToList();
            var parents = await unitService.Get()
                                    .Include(u => u.ChildUnits)
                                    .Where(u => parentNames.Contains(u.Name))
                                    .ToDictionaryAsync(u => u.Name, StringComparer.Ordinal);
            foreach (var (Parent, Child) in toAdd)
            {
                if (!parents.TryGetValue(Parent, out var parent) ||
                    !allUnits.TryGetValue(Child, out var child))
                {
                    logger.LogWarning("Пропущена связь '{Parent}' - '{Child}': Unit не найден", Parent, Child);
                    continue;
                }
                if (parent!.ChildUnits.Any(r => r.ChildUnitId == child.Id))
                {
                    logger.LogWarning("Связь уже существует: '{Parent}' - '{Child}'", Parent, Child);
                    continue;
                }
                parent.ChildUnits.Add(new UnitInUnit
                {
                    ParentUnitId = parent.Id,
                    ChildUnitId = child.Id,
                    DateCreated = DateTimeOffset.UtcNow
                });
                logger.LogInformation("Добавлена связь: '{Parent}' - '{Child}'", Parent, Child);
            }
        }
        private async Task FinalCommitAsync(int removedCount, int addedCount)
        {
            if (removedCount > 0 || addedCount > 0)
            {
                if (await unitService.CommitAsync())
                {
                    logger.LogInformation("Синхронизация завершена. Удалено: {Removed}, добавлено: {Added}", removedCount, addedCount);
                }
                else
                    logger.LogError("Не удалось сохранить изменения в БД");
            }
            else
                logger.LogInformation("Изменений не обнаружено");
        }
        private sealed class ExistingRelationship
        {
            public string ParentName { get; init; }
            public string ChildName { get; init; }
            public UnitInUnit Relationship { get; init; }
        }
    }
 }
    public async Task SyncAsync(List<AihitData> aihitData)
    {
        var validPairs = ValidateInput(aihitData);
        if (validPairs.Count == 0)
            return;
        if (!await PassSafeguardCheckAsync(validPairs.Count))
            return;
        var unitNames = ExtractUnitNames(validPairs);
        var unitsByName = await GetOrCreateUnitsAsync(unitNames);
        var (toRemove, toAdd) = await GetDeltaAsync(validPairs);
        // Удаление и добавление накапливаются в одном ChangeTracker.
        // CommitAsync сохраняет всё атомарно через SaveChangesAsync.
        await RemoveRelationshipsAsync(toRemove);
        await AddRelationshipsAsync(toAdd, unitsByName);
        await CommitChangesAsync(toRemove.Count, toAdd.Count);
    }
    private List<(string ParentName, string ChildName)> ValidateInput(List<AihitData> data)
    {
        if (data == null || data.Count == 0)
        {
            _logger.LogWarning("Получены пустые данные из АИХ ИТ. Синхронизация пропущена");
            return new List<(string, string)>();
        }
        var validPairs = data
            .Where(item => !string.IsNullOrWhiteSpace(item.ParentName) && !string.IsNullOrWhiteSpace(item.ChildName))
            .Where(item => item.ParentName != item.ChildName)
            .Select(item => (
                ParentName: item.ParentName!.Trim().ToUpperInvariant(),
                ChildName: item.ChildName!.Trim().ToUpperInvariant()
            ))
            .Distinct()
            .ToList();
        if (validPairs.Count == 0)
        {
            _logger.LogWarning("После нормализации не осталось валидных связей. Синхронизация пропущена");
            return new List<(string, string)>();
        }
        _logger.LogInformation("Получено {Count} валидных связей", validPairs.Count);
        return validPairs;
    }
    private async Task<bool> PassSafeguardCheckAsync(int incomingCount)
    {
        var previousCount = await _unitRepository.Get()
            .SelectMany(u => u.ChildUnits)
            .CountAsync();
        if (previousCount == 0)
            return true;
        var currentPercentage = Math.Round((decimal)incomingCount / previousCount * 100, 1, MidpointRounding.AwayFromZero);
        if (currentPercentage < _settingsFromDb.MinRelationshipsThresholdPct)
        {
            _logger.LogWarning(
                "Синхронизация отменена: количество полученных связей ниже порогового значения. " +
                "Получено: {CurrentCount} ({CurrentPercentage:F1}%), порог: {Threshold}% от прошлого объема ({PreviousCount})",
                incomingCount, currentPercentage, _settingsFromDb.MinRelationshipsThresholdPct, previousCount);
            return false;
        }
        return true;
    }
    private static List<string> ExtractUnitNames(List<(string ParentName, string ChildName)> pairs)
    {
        return pairs
            .SelectMany(pair => new[] { pair.ParentName, pair.ChildName })
            .Distinct()
            .ToList();
    }
    private async Task<Dictionary<string, Unit>> GetOrCreateUnitsAsync(List<string> names)
    {
        var existingUnits = await _unitRepository.Get()
            .AsNoTracking()
            .ToDictionaryAsync(u => u.Name, StringComparer.Ordinal);
        var missingNames = names.Except(existingUnits.Keys).ToList();
        if (missingNames.Count == 0)
            return existingUnits;
        _logger.LogInformation("Создание отсутствующих юнитов: {Count}", missingNames.Count);
        foreach (var name in missingNames)
        {
            var unit = new Unit
            {
                Id = Guid.NewGuid(),
                Name = name,
                DateCreated = DateTimeOffset.UtcNow
            };
            if (!await _unitRepository.CreateAsync(unit))
            {
                _logger.LogError("Не удалось создать юнит: {Name}", name);
                continue;
            }
            existingUnits[name] = unit;
            _logger.LogDebug("Создан юнит: {Name}", name);
        }
        await _unitRepository.CommitAsync();
        return existingUnits;
    }
    private async Task<(List<ExistingRelationship> ToRemove, List<(string ParentName, string ChildName)> ToAdd)> GetDeltaAsync(
        List<(string ParentName, string ChildName)> sourcePairs)
    {
        var relevantParentNames = sourcePairs.Select(p => p.ParentName).Distinct().ToList();
        var existingForParents = await _unitRepository.Get()
            .AsNoTracking()
            .Include(u => u.ChildUnits).ThenInclude(r => r.ChildUnit)
            .Where(u => relevantParentNames.Contains(u.Name))
            .SelectMany(u => u.ChildUnits, (parent, relationship) => new ExistingRelationship
            {
                ParentName = parent.Name,
                ChildName = relationship.ChildUnit!.Name,
                Relationship = relationship
            })
            .ToListAsync();
        var sourceSet = new HashSet<(string, string)>(sourcePairs);
        var existingSet = new HashSet<(string, string)>(
            existingForParents.Select(x => (x.ParentName, x.ChildName)));
        var toRemove = existingForParents
            .Where(r => !sourceSet.Contains((r.ParentName, r.ChildName)))
            .ToList();
        var toAdd = sourcePairs
            .Where(p => !existingSet.Contains(p))
            .ToList();
        _logger.LogInformation("Дельта: удаление {ToRemove}, добавление {ToAdd}", toRemove.Count, toAdd.Count);
        return (toRemove, toAdd);
    }
    private async Task RemoveRelationshipsAsync(List<ExistingRelationship> toRemove)
    {
        if (toRemove.Count == 0)
            return;
        var parentNames = toRemove.Select(r => r.ParentName).Distinct().ToList();
        var parents = await _unitRepository.Get()
            .Include(u => u.ChildUnits)
            .Where(u => parentNames.Contains(u.Name))
            .ToDictionaryAsync(u => u.Name, StringComparer.Ordinal);
        foreach (var relationship in toRemove)
        {
            if (!parents.TryGetValue(relationship.ParentName, out var parent))
            {
                _logger.LogWarning("Родитель '{Parent}' не найден при удалении связи с '{Child}'",
                    relationship.ParentName, relationship.ChildName);
                continue;
            }
            var entityToRemove = parent.ChildUnits
                .FirstOrDefault(r => r.ChildUnitId == relationship.Relationship.ChildUnitId);
            if (entityToRemove != null)
                parent.ChildUnits.Remove(entityToRemove);
        }
    }
    private async Task AddRelationshipsAsync(
        List<(string ParentName, string ChildName)> toAdd,
        Dictionary<string, Unit> unitsByName)
    {
        if (toAdd.Count == 0)
            return;
        var parentNames = toAdd.Select(x => x.ParentName).Distinct().ToList();
        var parents = await _unitRepository.Get()
            .Include(u => u.ChildUnits)
            .Where(u => parentNames.Contains(u.Name))
            .ToDictionaryAsync(u => u.Name, StringComparer.Ordinal);
        foreach (var (parentName, childName) in toAdd)
        {
            if (!parents.TryGetValue(parentName, out var parent) ||
                !unitsByName.TryGetValue(childName, out var child))
            {
                _logger.LogWarning("Пропущена связь '{Parent}' - '{Child}': юнит не найден", parentName, childName);
                continue;
            }
            if (parent.ChildUnits.Any(r => r.ChildUnitId == child.Id))
                continue;
            parent.ChildUnits.Add(new UnitInUnit
            {
                ParentUnitId = parent.Id,
                ChildUnitId = child.Id,
                DateCreated = DateTimeOffset.UtcNow
            });
        }
    }
    private async Task CommitChangesAsync(int removedCount, int addedCount)
    {
        if (removedCount == 0 && addedCount == 0)
        {
            _logger.LogInformation("Изменений не обнаружено");
            return;
        }
        if (!await _unitRepository.CommitAsync())
            throw new InvalidOperationException("Не удалось сохранить изменения связей в БД");
        _logger.LogInformation("Синхронизация завершена. Удалено: {Removed}, добавлено: {Added}", removedCount, addedCount);
    }
    private sealed class ExistingRelationship
    {
        public string ParentName { get; init; } = null!;
        public string ChildName { get; init; } = null!;
        public UnitInUnit Relationship { get; init; } = null!;
    }
 }