diff --git a/PARR.AIHITMainLoader/AihitMainLoader.cs b/PARR.AIHITMainLoader/AihitMainLoader.cs index 2087ba8c..be1c42b1 100644 --- a/PARR.AIHITMainLoader/AihitMainLoader.cs +++ b/PARR.AIHITMainLoader/AihitMainLoader.cs @@ -54,9 +54,7 @@ namespace PARR.AIHITMainLoader logger.LogInformation("Запуск загрузки данных из АИХ ИТ."); using (var scope = serviceProvider.CreateScope()) { - var aihitService = scope.ServiceProvider.GetService(); - if (aihitService == null) - throw new Exception($"Не найден сервис: {nameof(IAihitService)}"); + var aihitService = scope.ServiceProvider.GetRequiredService(); var services = GetServices(aihitService); @@ -89,14 +87,21 @@ namespace PARR.AIHITMainLoader int count = preparedDataList.Count; int packageSize = loaderSettings.PackageSize; + + if (packageSize <= 0) + { + logger.LogError("Некорректный размер пакета: {PackageSize}. Ожидалось > 0.", packageSize); + return; + } + var parts = count == 0 ? 0 : (count + packageSize - 1) / packageSize; - for (var i = 0; i < (count + packageSize - 1) / packageSize; i++) + for (var i = 0; i < parts; i++) { int start = i * packageSize; int length = Math.Min(packageSize, count - start); var batch = preparedDataList.GetRange(start, length).ToArray(); - var sendResult = await mqService.SendAsync(mqSettings, batch.ToArray()); + var sendResult = await mqService.SendAsync(mqSettings, batch); if (sendResult.IsSuccess) { logger.LogInformation($"Данные переданы в RabbitMQ: {batch.Count()}"); @@ -106,7 +111,8 @@ namespace PARR.AIHITMainLoader logger.LogError($"Ошибка при передаче данных в Rabbit, не переданные сообщения: {string.Join(", ", sendResult.NotSendMessages!)}"); } } - } catch (Exception ex) + } + catch (Exception ex) { logger.LogError(ex, "Ошибка при выполнении загрузки данных из АИХ ИТ."); } @@ -115,13 +121,12 @@ namespace PARR.AIHITMainLoader private List?>> GetServices(IAihitService service) { - var serviceList = new List?>>(); - - serviceList.Add(service.GetCvkData); - serviceList.Add(service.GetPtkData); - serviceList.Add(service.GetRegionalEKData); - - return serviceList; + return new() + { + service.GetCvkData, + service.GetPtkData, + service.GetRegionalEKData + }; } } } diff --git a/PARR.AIHITMainSyncer/AihitMainSyncer.cs b/PARR.AIHITMainSyncer/AihitMainSyncer.cs index dc0531f1..54dbf239 100644 --- a/PARR.AIHITMainSyncer/AihitMainSyncer.cs +++ b/PARR.AIHITMainSyncer/AihitMainSyncer.cs @@ -33,10 +33,7 @@ namespace PARR.AIHITMainSyncer { using (var scope = serviceProvider.CreateScope()) { - var syncerService = scope.ServiceProvider.GetService(); - if (syncerService == null) - throw new Exception("Не смог получить серивс ISyncerService, scope.ServiceProvider.GetService()"); - + var syncerService = scope.ServiceProvider.GetRequiredService(); await syncerService.SyncAsync(msg); } diff --git a/PARR.AIHITMainSyncer/Services/SyncerService.cs b/PARR.AIHITMainSyncer/Services/SyncerService.cs index 5a6a4f90..57f3053a 100644 --- a/PARR.AIHITMainSyncer/Services/SyncerService.cs +++ b/PARR.AIHITMainSyncer/Services/SyncerService.cs @@ -3,7 +3,6 @@ using Microsoft.Extensions.Logging; using Newtonsoft.Json.Linq; using PARR.BLL.Domain.Mq; using PARR.BLL.Services.Interfaces; -using PARR.DAL.Extensions; using PARR.DAL.Models.Unit; using PARR.DAL.Services.Interfaces.Unit; @@ -52,6 +51,9 @@ namespace PARR.AIHITMainSyncer.Services if (!objFromQuery.Properties.Any()) return; + foreach (var kvp in objFromQuery.Properties.ToList()) + objFromQuery.Properties[kvp.Key.Trim()] = kvp.Value?.Trim(); + var listAihitData = new List>(); var propertisWithSimpleValues = objFromQuery.Properties @@ -69,45 +71,54 @@ namespace PARR.AIHITMainSyncer.Services var unit = await GetUnit(objFromQuery.Name); - var unitValuesToRemove = unit.UnitValues.Where(t => !listAihitData.Any(a => - IsStringEqual(t.Field!.AihitName, a.Key) - ) || - !listAihitData.Any(a => - IsStringEqual(t.Field!.AihitName, a.Key) && IsStringEqual(t.Value!.Value, a.Value) - )).ToList(); + var incomingPairs = listAihitData + .Select(kv => new { kv.Key, kv.Value }) + .ToList(); + + var unitValuesToRemove = unit.UnitValues + .Where(uv => !incomingPairs.Any(ip => + IsStringEqual(uv.Field!.AihitName, ip.Key) && + IsStringEqual(uv.Value!.Value, ip.Value))) + .ToList(); + unitValuesToRemove.ForEach(uv => unit.UnitValues.Remove(uv)); - var unitValueExisting = unit.UnitValues.Where(t => listAihitData.Any(a => - IsStringEqual(t.Field!.AihitName, a.Key) && - IsStringEqual(t.Value!.Value, a.Value) - )).ToList(); - - var unitValueMising = listAihitData.Where(t => + var unitValueMissing = listAihitData.Where(t => !unit.UnitValues.Any(a => IsStringEqual(t.Key, a.Field!.AihitName) && IsStringEqual(t.Value, a.Value!.Value) )).ToList(); - if (unitValueMising.Any()) + if (unitValueMissing.Any()) { - var fieldsInChanges = unitValueMising.Select(s => s.Key).Distinct().ToList(); - var valuesInChanges = unitValueMising.Select(s => s.Value).Distinct().ToList(); + var fieldsInChanges = unitValueMissing.Select(s => s.Key).Distinct().ToList(); + var valuesInChanges = unitValueMissing.Select(s => s.Value).Distinct().ToList(); await SyncFieldsAsync(fieldsInChanges!, fieldsFromDB); await SyncValuesAsync(valuesInChanges); - //fieldsInChanges = fieldsInChanges.Select(s => s.ToLower()).ToList()!; - //valuesInChanges = valuesInChanges.Select(s => (s == null) ? null : Sanitize(s.ToLower())).ToList()!; + var normalizedFieldNames = fieldsInChanges + .Select(n => n?.Trim()) + .Where(n => n != null) + .ToHashSet(StringComparer.OrdinalIgnoreCase); - var fieldsNameWithId = await unitFieldService.Get().AsNoTracking().Where(t => fieldsInChanges.Any(c => - c == t.AihitName//.AihitName.ToLower() - )).ToListAsync(); - var valuesWithId = await unitFieldValueService.Get().AsNoTracking().Where(t => valuesInChanges.Any(c => - c == t.Value//((t.Value == null) ? null : t.Value.ToLower()) - )).ToListAsync(); + var fieldsNameWithId = await unitFieldService.Get().AsNoTracking() + .Where(f => normalizedFieldNames.Contains(f.AihitName)) + .ToListAsync(); - foreach (var change in unitValueMising) + var normalizedValues = valuesInChanges + .Where(v => v != null) + .Select(v => v!.Trim()) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var valuesWithId = await unitFieldValueService.Get().AsNoTracking() + .Where(v => + (v.Value == null && valuesInChanges.Contains(null)) || + (v.Value != null && normalizedValues.Contains(v.Value))) + .ToListAsync(); + + foreach (var change in unitValueMissing) ApplyChange(unit, fieldsNameWithId, valuesWithId, change); if (!await unitService.CommitAsync()) @@ -161,50 +172,72 @@ namespace PARR.AIHITMainSyncer.Services private async Task SyncValuesAsync(List values) { + bool hasNull = values.Any(v => v is null); // все поля из АИХИТ - var valuesFromAihit = values.Distinct(); + var normalizedNoneNullValues = values + .Where(v => v != null) + .Select(v => v!.Trim()) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToList(); - // список values для поиска в БД без null - var valuesFromAihitWithoutNull = valuesFromAihit.Where(x => x != null).ToList(); + // Получаем существующие значения из БД + var existingValuesInDb = await unitFieldValueService.Get() + .Where(v => normalizedNoneNullValues.Contains(v.Value!) || v.Value == null) + .Select(v => v.Value) + .ToListAsync(); - var valuesInDb = await unitFieldValueService.Get().Where(t => - (t.Value != null && - valuesFromAihitWithoutNull.Any(x => x == t.Value) - ) - || (t.Value == null) - ).Select(t => t.Value) - .ToListAsync(); + if (!hasNull) + existingValuesInDb.RemoveAll(v => v is null); - if (!valuesFromAihit.Any(x => x == null)) - valuesInDb.Remove(null); + var newNonNullValues = normalizedNoneNullValues + .Where(v => !existingValuesInDb.Any(ev => + string.Equals(ev, v, StringComparison.OrdinalIgnoreCase))) + .ToList(); - var newValues = valuesFromAihit.Where(t => !valuesInDb.Any(x => IsStringEqual(x, t))).ToList(); + var newValuesToInsert = new List(); + newValuesToInsert.AddRange(newNonNullValues); - if (newValues.Any()) - { - foreach (var item in newValues) + if (hasNull && !existingValuesInDb.Contains(null)) + newValuesToInsert.Add(null); + + if (newValuesToInsert.Count == 0) + return; + + // Создаём сущности — сохраняем оригинальный регистр из nonNullValues! + // (для null — просто null) + var newFieldValues = newValuesToInsert.Select(v => + new UnitFieldValue { - var value = new UnitFieldValue - { - Id = Guid.NewGuid(), - DateCreated = DateTimeOffset.UtcNow, - Value = item - }; + Id = Guid.NewGuid(), + Value = v // ← v — уже trim()-нутый non-null, или null + }) + .ToList(); - if (!await unitFieldValueService.CreateAsync(value)) - logger.LogError($"Не удалось создать запись в таблице FieldValues: {item}, {value.ToJson()}"); - else - logger.LogInformation($"Создана запись а таблице FieldValues: {item}, {value.ToJson()}"); + if (!await unitFieldValueService.AddRangeAsync(newFieldValues) || !await unitFieldValueService.CommitAsync()) + { + logger.LogError( + "Не удалось сохранить {Count} новых значений FieldValues", + newValuesToInsert.Count); + } + else + { + if (newValuesToInsert.Count <= 10) + { + var preview = string.Join(", ", newValuesToInsert.Select(v => v ?? "")); + logger.LogInformation("Добавлено {Count} значений FieldValues: [{Values}]", newValuesToInsert.Count, preview); + } + else + { + logger.LogInformation("Добавлено {Count} значений FieldValues (первые 5: {Preview})", + newValuesToInsert.Count, + string.Join(", ", newValuesToInsert.Take(5).Select(v => v ?? ""))); } - if (!await unitFieldValueService.CommitAsync()) - logger.LogError($"Не удалось применить изменения по добавлению новых Fields в базе данных"); } } - //private async Task SyncFieldsAsync(List fieldsFromAihit, List fieldsFromDB) + private async Task SyncFieldsAsync(List fieldsFromAihit, List fieldsFromDB) { - //var newFields = fieldsFromAihit.Where(t => !fieldsFromDB.Any(f => f.Name == t)); var newFields = fieldsFromAihit.Where(t => !fieldsFromDB.Any(f => f.AihitName == t)); if (newFields.Any()) @@ -231,11 +264,10 @@ namespace PARR.AIHITMainSyncer.Services private bool IsStringEqual(string? value1, string? value2) { - //var _value1 = value1?.ToLower().Trim(); - //var _value2 = value2?.ToLower().Trim(); - var _value1 = value1; - var _value2 = value2; - return _value1 == _value2; + return string.Equals( + value1?.Trim(), + value2?.Trim(), + StringComparison.OrdinalIgnoreCase); } @@ -252,7 +284,7 @@ namespace PARR.AIHITMainSyncer.Services private async Task CreateUnitAsync(string name) { - var unit = new Unit { Name = name.Trim() }; + var unit = new Unit { Name = name.Trim().ToUpperInvariant() }; if (!await unitService.CreateAsync(unit) || !await unitService.CommitAsync()) logger.LogError($"Не удалось создать Unit {unit.Name}"); @@ -265,8 +297,9 @@ namespace PARR.AIHITMainSyncer.Services public async Task GetUnitByName(string name) { + var normalized = name.Trim().ToUpperInvariant(); return await GetUnitWithFieldsAndValues() - .FirstOrDefaultAsync(u => u.Name.ToLower() == name.ToLower()); + .FirstOrDefaultAsync(u => u.Name == normalized); } diff --git a/PARR.AIHITSyncerWorker/Worker.cs b/PARR.AIHITSyncerWorker/Worker.cs index d119b0ea..d12479c9 100644 --- a/PARR.AIHITSyncerWorker/Worker.cs +++ b/PARR.AIHITSyncerWorker/Worker.cs @@ -17,12 +17,13 @@ namespace PARR.AIHITSyncerWorker protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await aihitSyncer.StartAsync(); + await Task.Delay(Timeout.Infinite, stoppingToken); } public override Task StopAsync(CancellationToken cancellationToken) { - aihitSyncer.StopAsync().Wait(); + aihitSyncer.StopAsync(); return base.StopAsync(cancellationToken); } diff --git a/PARR.AIHITSyncerWorker/appsettings.Development.json b/PARR.AIHITSyncerWorker/appsettings.Development.json index 3b67da3f..04114c4d 100644 --- a/PARR.AIHITSyncerWorker/appsettings.Development.json +++ b/PARR.AIHITSyncerWorker/appsettings.Development.json @@ -1,7 +1,7 @@ { "Serilog": { "MinimumLevel": { - "Default": "Information", + "Default": "Debug", "Override": { //"Microsoft": "Information", "Microsoft.Hosting.Lifetime": "Information"