feat(esppScheduleSync): создан проект. Добавлена основная логика синхронизации. Docker

This commit is contained in:
Mikhail Trubnikov
2023-11-22 16:34:16 +10:00
parent dc90275f6f
commit b2f150890c
27 changed files with 457 additions and 4 deletions

View File

@@ -3,6 +3,7 @@ variables:
PROD_NAME_API: "parr/parr-api"
PROD_NAME_AIHIT: "parr/parr-aihit-syncher"
PROD_NAME_ESPP_TEMPLATE: "parr/parr-espp-template-sync"
PROD_NAME_ESPP_SCHEDULE: "parr/parr-espp-schedule-sync"
PROD_NAME_GENERATOR_TEMPLATES: "parr/parr-generator-templates"
PROD_NAME_ESPP_ORDER_LOADER: "parr/parr-espp-order-loader"
PROD_NAME_ESPP_ORDER_MANAGER: "parr/parr-espp-order-manager"
@@ -358,3 +359,51 @@ prod_master_deploy:
- RUNNER: shell-api-swarm-01
tags:
- ${RUNNER}
### ESPP SCHEDULE SYNC PROD ###
prod_espp_schedule_build:
stage: build
only:
- /^es[0-9]+\.[0-9]+\.[0-9]+$/
except:
- branches
services:
- name: docker:20.10.21-dind
command: [
"--insecure-registry=10.99.253.167:8090",
"--registry-mirror=http://10.99.253.167:8090",
"--insecure-registry=harbor.dvgd.rzd",
"--tls=false"
]
variables:
DOCKER_HOST: tcp://docker:2375
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
script:
- APP_VERSION=$(echo $CI_COMMIT_TAG | tr -d es)
- IMAGE_VERSION=$(echo $CI_COMMIT_TAG | sed 's/es/v/g')
- docker build -t $REPO/$PROD_NAME_ESPP_SCHEDULE:$IMAGE_VERSION -t $REPO/$PROD_NAME_ESPP_SCHEDULE:latest -t $PROD_NAME_ESPP_SCHEDULE:$IMAGE_VERSION -t $PROD_NAME_ESPP_SCHEDULE:latest --build-arg app_version=$APP_VERSION -f PARR.EsppScheduleSyncWorker/Dockerfile .
- docker login -u $HARBOR_PUSH_USER -p $HARBOR_PUSH_PASS $REPO
- docker push --all-tags $REPO/$PROD_NAME_ESPP_SCHEDULE
tags:
- docker
prod_espp_schedule_deploy:
stage: deploy
environment:
name: parr-espp-schedule-sync
only:
- /^es[0-9]+\.[0-9]+\.[0-9]+$/
except:
- branches
script:
- IMAGE_VERSION=$(echo $CI_COMMIT_TAG | sed 's/es/v/g')
- docker login -u $HARBOR_PULL_USER -p $HARBOR_PULL_PASS $REPO
- tag=$IMAGE_VERSION docker stack deploy -c docker-compose.espp-schedule-sync.yml parr-espp-schedule-sync --with-registry-auth
parallel:
matrix:
- RUNNER: shell-api-swarm-01
tags:
- ${RUNNER}

View File

@@ -55,6 +55,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "PARR.Constants", "PARR.Cons
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PARR.EsppSync", "PARR.EsppSync\PARR.EsppSync.csproj", "{71121EF7-D4C1-43A4-9243-EF5C4C82030F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PARR.EsppScheduleSync", "PARR.EsppScheduleSync\PARR.EsppScheduleSync.csproj", "{1E7F0EFE-784E-42E1-96BA-1FF743998619}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PARR.EsppScheduleSyncWorker", "PARR.EsppScheduleSyncWorker\PARR.EsppScheduleSyncWorker.csproj", "{C597A1D7-1BC5-493B-BC29-03EC830FB090}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@@ -147,6 +151,14 @@ Global
{71121EF7-D4C1-43A4-9243-EF5C4C82030F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{71121EF7-D4C1-43A4-9243-EF5C4C82030F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{71121EF7-D4C1-43A4-9243-EF5C4C82030F}.Release|Any CPU.Build.0 = Release|Any CPU
{1E7F0EFE-784E-42E1-96BA-1FF743998619}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1E7F0EFE-784E-42E1-96BA-1FF743998619}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1E7F0EFE-784E-42E1-96BA-1FF743998619}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1E7F0EFE-784E-42E1-96BA-1FF743998619}.Release|Any CPU.Build.0 = Release|Any CPU
{C597A1D7-1BC5-493B-BC29-03EC830FB090}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{C597A1D7-1BC5-493B-BC29-03EC830FB090}.Debug|Any CPU.Build.0 = Debug|Any CPU
{C597A1D7-1BC5-493B-BC29-03EC830FB090}.Release|Any CPU.ActiveCfg = Release|Any CPU
{C597A1D7-1BC5-493B-BC29-03EC830FB090}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE

View File

@@ -10,6 +10,7 @@ COPY ["NuGet.config", "."]
COPY ["PARR.API/PARR.API.csproj", "PARR.API/"]
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
RUN dotnet restore "PARR.API/PARR.API.csproj"
COPY . .
WORKDIR "/src/PARR.API"

View File

@@ -11,6 +11,7 @@ COPY ["PARR.EsppOrderLoader/PARR.EsppOrderLoader.csproj", "PARR.EsppOrderLoader/
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.EsppApi/PARR.EsppApi.csproj", "PARR.EsppApi/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
RUN dotnet restore "PARR.EsppOrderLoaderWorker/PARR.EsppOrderLoaderWorker.csproj"
COPY . .
WORKDIR "/src/PARR.EsppOrderLoaderWorker"

View File

@@ -11,6 +11,7 @@ COPY ["PARR.EsppOrderManager/PARR.EsppOrderManager.csproj", "PARR.EsppOrderManag
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.EsppApi/PARR.EsppApi.csproj", "PARR.EsppApi/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
RUN dotnet restore "PARR.EsppOrderManagerWorker/PARR.EsppOrderManagerWorker.csproj"
COPY . .
WORKDIR "/src/PARR.EsppOrderManagerWorker"

View File

@@ -0,0 +1,14 @@
using PARR.Constants;
using PARR.EsppSync;
namespace PARR.EsppScheduleSync.Domain
{
internal class EsppObjectSchedule : IEsppObject
{
public required string TemplateName { get; set; }
public RobotsEnum Robot => RobotsEnum.ScheduleOrder;
//todo:
}
}

View File

@@ -0,0 +1,37 @@
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using PARR.BLL;
using PARR.DAL;
using PARR.EsppSync;
using PARR.EsppScheduleSync.Domain;
using PARR.EsppScheduleSync.Settings;
namespace PARR.EsppScheduleSync
{
public static class EsppScheduleSyncInstaller
{
public static void InstallEsppScheduleSyncServices(this IServiceCollection services, IConfiguration configuration)
{
services.InstallBllServices(configuration);
services.InstallEsppSyncServices<EsppObjectSchedule>(configuration);
var globalSettings = new GlobalSettings();
configuration.GetSection(nameof(GlobalSettings)).Bind(globalSettings);
services.AddSingleton(globalSettings);
services.AddTransient<IScheduleSyncher, ScheduleSyncher>();
}
public static IConfigurationBuilder AddEsppScheduleConfigurations(this IConfigurationBuilder builder, IServiceCollection services)
{
builder.AddDalConfigurations(services);
return builder;
}
public static void AddEsppScheduleSettings(this IServiceCollection services, IConfiguration configuration)
{
services.AddDallSettings(configuration);
}
}
}

View File

@@ -0,0 +1,8 @@
namespace PARR.EsppScheduleSync
{
public interface IScheduleSyncher
{
void Start();
void Stop();
}
}

View File

@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\PARR.BLL\PARR.BLL.csproj" />
<ProjectReference Include="..\PARR.DAL\PARR.DAL.csproj" />
<ProjectReference Include="..\PARR.EsppSync\PARR.EsppSync.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,91 @@
using Microsoft.Extensions.Logging;
using PARR.BLL.Services.Interfaces;
using PARR.DAL.Contracts;
using PARR.DAL.Models;
using PARR.EsppScheduleSync.Domain;
using PARR.EsppScheduleSync.Settings;
using PARR.EsppSync;
namespace PARR.EsppScheduleSync
{
internal class ScheduleSyncher : IScheduleSyncher
{
private readonly ILogger<ScheduleSyncher> logger;
private readonly GlobalSettings globalSettings;
private readonly IMqService mqService;
private readonly ISyncService<EsppObjectSchedule> syncService;
private readonly SettingsFromDb settingsFromDb;
public ScheduleSyncher(
ILogger<ScheduleSyncher> logger,
GlobalSettings globalSettings,
IMqService mqService,
ISyncService<EsppObjectSchedule> syncService,
SettingsFromDb settingsFromDb
)
{
this.logger = logger;
this.globalSettings = globalSettings;
this.mqService = mqService;
this.syncService = syncService;
this.settingsFromDb = settingsFromDb;
if (globalSettings.MqSettings == null)
{
logger.LogError("Нет секции настроек хранилища. MqSettings, EsppTemplates");
throw new Exception("Нет секции настроек хранилища. MqSettings, EsppTemplates");
}
}
public void Start()
{
var isConnected = mqService.InitConsumer(globalSettings!.MqSettings!, SyncScheduleAsync);
if (!isConnected)
throw new Exception("Ошибка при подключении к RabbitMq");
logger.LogInformation($"Запущена проверка очереди {globalSettings.MqSettings!.QueueName}.");
}
public void Stop()
{
mqService.Dispose();
logger.LogInformation($"=== === === Соединение с очередью {globalSettings.MqSettings!.QueueName} закрыто === === ===");
}
private async Task SyncScheduleAsync(string str)
{
await syncService.SyncEsppObjectAsync(str, ParseStrToEsppObject, ConvertDbObjToEsppObj);
}
/// <summary>
/// Преобразование модели БД в модель для сравнения
/// </summary>
/// <param name="template"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
private EsppObjectSchedule ConvertDbObjToEsppObj(Template template)
{
//todo:
throw new NotImplementedException();
}
/// <summary>
/// Парсинг из строки в модель для сравнения
/// </summary>
/// <param name="str"></param>
/// <returns></returns>
/// <exception cref="NotImplementedException"></exception>
private EsppObjectSchedule? ParseStrToEsppObject(string str)
{
//todo:
throw new NotImplementedException();
}
}
}

View File

@@ -0,0 +1,18 @@
using PARR.BLL.Contracts.Interfaces;
namespace PARR.EsppScheduleSync.Settings
{
internal class GlobalSettings
{
public MqSettings? MqSettings { get; set; }
public string ParsingSeparator { get; set; } = "<|>";
}
internal class MqSettings : IMqSettings
{
public string HostName { get; set; } = string.Empty;
public string QueueName { get; set; } = string.Empty;
public string User { get; set; } = string.Empty;
public string Password { get; set; } = string.Empty;
}
}

View File

@@ -0,0 +1,28 @@
#See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
FROM 10.99.253.167:8090/dotnet/runtime:7.0 AS base
WORKDIR /app
FROM 10.99.253.167:8090/dotnet/sdk:7.0 AS build
WORKDIR /src
COPY ["NuGet.config", "."]
COPY ["PARR.EsppScheduleSyncWorker/PARR.EsppScheduleSyncWorker.csproj", "PARR.EsppScheduleSyncWorker/"]
COPY ["PARR.EsppScheduleSync/PARR.EsppScheduleSync.csproj", "PARR.EsppScheduleSync/"]
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
COPY ["PARR.EsppSync/PARR.EsppSync.csproj", "PARR.EsppSync/"]
RUN dotnet restore "PARR.EsppScheduleSyncWorker/PARR.EsppScheduleSyncWorker.csproj"
COPY . .
WORKDIR "/src/PARR.EsppScheduleSyncWorker"
RUN dotnet build "PARR.EsppScheduleSyncWorker.csproj" -c Release -o /app/build
FROM build AS publish
ARG app_version=0.0.0-default
RUN dotnet publish "PARR.EsppScheduleSyncWorker.csproj" -c Release -o /app/publish /p:UseAppHost=false /p:Version=$app_version
FROM base AS final
WORKDIR /app
COPY --from=publish /app/publish .
ENTRYPOINT ["dotnet", "PARR.EsppScheduleSyncWorker.dll"]

View File

@@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk.Worker">
<PropertyGroup>
<TargetFramework>net7.0</TargetFramework>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<UserSecretsId>dotnet-PARR.EsppScheduleSyncWorker-30fab400-2a4f-412e-ad4d-ec19f45d7c50</UserSecretsId>
<DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Elastic.CommonSchema.Serilog" Version="8.6.1" />
<PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
<PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.19.4" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="7.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="7.0.1" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\PARR.EsppScheduleSync\PARR.EsppScheduleSync.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,32 @@
using Elastic.CommonSchema.Serilog;
using PARR.EsppScheduleSync;
using PARR.EsppScheduleSyncWorker;
using Serilog;
var builder = Host.CreateApplicationBuilder();
builder.Services.AddLogging(config =>
{
config.ClearProviders();
var logger = new LoggerConfiguration();
if (builder.Environment.IsProduction())
logger.WriteTo.Console(new EcsTextFormatter());
else
logger.WriteTo.Console();
logger.ReadFrom.Configuration(builder.Configuration);
config.AddSerilog(logger.CreateLogger());
});
builder.Services.InstallEsppScheduleSyncServices(builder.Configuration);
builder.Configuration.AddEsppScheduleConfigurations(builder.Services);
builder.Services.AddEsppScheduleSettings(builder.Configuration);
builder.Services.AddHostedService<Worker>();
var host = builder.Build();
host.Run();

View File

@@ -0,0 +1,14 @@
{
"profiles": {
"PARR.EsppScheduleSyncWorker": {
"commandName": "Project",
"environmentVariables": {
"DOTNET_ENVIRONMENT": "Development"
},
"dotnetRunMessages": true
},
"Docker": {
"commandName": "Docker"
}
}
}

View File

@@ -0,0 +1,29 @@
using PARR.DAL.TransformServices;
using PARR.EsppScheduleSync;
namespace PARR.EsppScheduleSyncWorker
{
public class Worker : BackgroundService
{
private readonly ILogger<Worker> _logger;
private readonly IScheduleSyncher scheduleSyncher;
public Worker(ILogger<Worker> logger, IScheduleSyncher scheduleSyncher)
{
_logger = logger;
this.scheduleSyncher = scheduleSyncher;
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
scheduleSyncher.Start();
}
public override Task StopAsync(CancellationToken cancellationToken)
{
scheduleSyncher.Stop();
return base.StopAsync(cancellationToken);
}
}
}

View File

@@ -0,0 +1,13 @@
{
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"GlobalSettings": {
"MqSettings": {
"HostName": "10.99.253.216"
}
}
}

View File

@@ -0,0 +1,38 @@
{
"ConnectionStrings": {
"DefaultConnection": "Server=10.99.253.184;Database=parr;User Id=app_parr; Password=PosdfkhT&)%sdfligL&%5546;"
},
"Logging": {
"LogLevel": {
"Default": "Information",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"Serilog": {
"MinimumLevel": {
"Default": "Debug",
"Override": {
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Debug"
}
},
"WriteTo": [
{
"Name": "File",
"Args": {
"path": "log/log-.txt",
"rollingInterval": "Day"
}
}
]
},
"GlobalSettings": {
"MqSettings": {
"HostName": "parr-rabbitmq",
"QueueName": "parr-espp-schedulers",
"User": "espp_schedulers_reader",
"Password": "KjdhGLJDshgd&^%84S2"
},
"ParsingSeparator": "<|>"
}
}

View File

@@ -7,7 +7,7 @@ namespace PARR.EsppTemplateSync.Settings
public MqSettings? MqSettings { get; set; }
public StorageSettings? StorageSettings { get; set; }
public List<string>? IgnoreTemplateFields { get; set; }
//public List<string>? IgnoreTemplateFields { get; set; }
public string ParsingSeparator { get; set; } = "<|>";
}

View File

@@ -29,6 +29,7 @@ namespace PARR.EsppTemplateSync
this.mqService = mqService;
this.syncService = syncService;
this.settingsFromDb = settingsFromDb;
if (globalSettings.MqSettings == null)
{
logger.LogError("Нет секции настроек хранилища. MqSettings, EsppTemplates");

View File

@@ -10,6 +10,7 @@ COPY ["PARR.EsppTemplateSyncWorker/PARR.EsppTemplateSyncWorker.csproj", "PARR.Es
COPY ["PARR.EsppTemplateSync/PARR.EsppTemplateSync.csproj", "PARR.EsppTemplateSync/"]
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
RUN dotnet restore "PARR.EsppTemplateSyncWorker/PARR.EsppTemplateSyncWorker.csproj"
COPY . .
WORKDIR "/src/PARR.EsppTemplateSyncWorker"

View File

@@ -43,7 +43,6 @@
"StorageSettings": {
"CheckIntervalSeconds": 30
},
"IgnoreTemplateFields": [ "Id", "Status", "Name", "DateCreated", "DateModified", "StatusCode" ],
"ParsingSeparator": "<|>"
}
}

View File

@@ -10,6 +10,7 @@ COPY ["PARR.GeneratorTemplatesWorker/PARR.GeneratorTemplatesWorker.csproj", "PAR
COPY ["PARR.GeneratorTemplates/PARR.GeneratorTemplates.csproj", "PARR.GeneratorTemplates/"]
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
RUN dotnet restore "PARR.GeneratorTemplatesWorker/PARR.GeneratorTemplatesWorker.csproj"
COPY . .
WORKDIR "/src/PARR.GeneratorTemplatesWorker"

View File

@@ -10,6 +10,7 @@ COPY ["PARR.MasterWorker/PARR.MasterWorker.csproj", "PARR.MasterWorker/"]
COPY ["PARR.Master/PARR.Master.csproj", "PARR.Master/"]
COPY ["PARR.BLL/PARR.BLL.csproj", "PARR.BLL/"]
COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
COPY ["PARR.Constants/PARR.Constants.csproj", "PARR.Constants/"]
RUN dotnet restore "PARR.MasterWorker/PARR.MasterWorker.csproj"
COPY . .
WORKDIR "/src/PARR.MasterWorker"

View File

@@ -16,8 +16,10 @@
- PARR.EsppOrderLoaderWorker
- PARR.EsppOrderManager - управление нарядами в ЕСПП (в работу, выполнить, и т.п.)
- PARR.EsppOrderManagerWorker
- PARR.EsppScheduleSync - логика синхронизации расписаний ЕСПП с БД PARR
- PARR.EsppScheduleSyncWorker - worker для PARR.EsppScheduleSync
- PARR.EsppSync - общая логика синхронизации шаблонов и расписаний ЕСПП с БД PARR
- PARR.EsppTemplateSync - логика синхронизации шаблонов ESPP с БД PARR
- PARR.EsppTemplateSync - логика синхронизации шаблонов ЕСПП с БД PARR
- PARR.EsppTemplateSyncWorker - worker для PARR.EsppTemplateSync
- PARR.GeneratorTemplates - логика генерации шаблонов в ПАРР
- PARR.GeneratorTemplatesWorker - Worker для PARR.GeneratorTemplates

View File

@@ -0,0 +1,23 @@
version: '3.4'
#ESPP SCHEDULE SYNC
services:
parr-espp-schedule-sync:
image: harbor.dvgd.rzd/parr/parr-espp-schedule-sync:${tag:-latest}
environment:
- ASPNETCORE_ENVIRONMENT=Production
- TZ=Europe/Moscow
logging:
driver: fluentd
options:
fluentd-address: dvgd-mng-02.dvgd.oao.rzd:24224
tag: parr.espp-schedule-sync.serilog
deploy:
replicas: 4
networks:
- parr-network
networks:
parr-network:
driver: overlay
external: true