feat(taskReconciliation, workloadBuilder): Dockerfile, ci-cd

.gitlab-ci.yml

@@ -18,6 +18,9 @@ variables:
   PROD_JOB_AUTO_CONTROL: "parr/parr-job-auto-control"
   PROD_TEMPLATE_MATCHER: "parr/parr-template-matcher"
   PROD_TEMPLATE_UPDATER: "parr/parr-template-updater"
+  PROD_WORKLOAD_BUILDER: "parr/parr-workload-builder"
+  PROD_TASK_RECONCILIATION: "parr/parr-task-reconciliation"
+
 
 stages:
   - build
@@ -899,5 +902,102 @@ prod_template_updater_deploy:
   parallel:
     matrix:
       - RUNNER: shell-api-swarm-01
+  tags:
+    - ${RUNNER}
+
+
+
+### WORKLOAD BUILDER PROD ###
+prod_workload_builder_build:
+  stage: build
+  only:
+    - /^wb[0-9]+\.[0-9]+\.[0-9]+$/
+  except:
+    - branches
+  services:
+    - name: docker:20.10.21-dind
+      command: [
+        "--insecure-registry=10.99.253.167:8090",
+        "--registry-mirror=http://10.99.253.167:8090",
+        "--insecure-registry=harbor.dvgd.rzd",
+        "--tls=false"
+        ]
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  script:
+    - APP_VERSION=$(echo $CI_COMMIT_TAG | tr -d wb)
+    - IMAGE_VERSION=$(echo $CI_COMMIT_TAG | sed 's/wb/v/g')
+    - docker build -t $REPO/$PROD_WORKLOAD_BUILDER:$IMAGE_VERSION -t $REPO/$PROD_WORKLOAD_BUILDER:latest -t $PROD_WORKLOAD_BUILDER:$IMAGE_VERSION -t $PROD_WORKLOAD_BUILDER:latest --build-arg app_version=$APP_VERSION -f PARR.WorkloadBuilderWorker/Dockerfile .
+    - docker login -u $HARBOR_PUSH_USER -p $HARBOR_PUSH_PASS $REPO
+    - docker push --all-tags $REPO/$PROD_WORKLOAD_BUILDER
+  tags:
+    - docker
+
+
+prod_workload_builder_deploy:
+  stage: deploy
+  environment:
+    name: parr-workload-builder
+  only:
+    - /^wb[0-9]+\.[0-9]+\.[0-9]+$/
+  except:
+    - branches
+  script:
+    - IMAGE_VERSION=$(echo $CI_COMMIT_TAG | sed 's/wb/v/g')
+    - docker login -u $HARBOR_PULL_USER -p $HARBOR_PULL_PASS $REPO 
+    - tag=$IMAGE_VERSION docker stack deploy -c docker-compose.workload-builder.yml parr-workload-builder --with-registry-auth
+  parallel:
+    matrix:
+      - RUNNER: shell-api-swarm-01
+  tags:
+    - ${RUNNER}
+
+
+### TASK RECONCILIATION PROD ###
+prod_task_reconciliation_build:
+  stage: build
+  only:
+    - /^tr[0-9]+\.[0-9]+\.[0-9]+$/
+  except:
+    - branches
+  services:
+    - name: docker:20.10.21-dind
+      command: [
+        "--insecure-registry=10.99.253.167:8090",
+        "--registry-mirror=http://10.99.253.167:8090",
+        "--insecure-registry=harbor.dvgd.rzd",
+        "--tls=false"
+        ]
+  variables:
+    DOCKER_HOST: tcp://docker:2375
+    DOCKER_DRIVER: overlay2
+    DOCKER_TLS_CERTDIR: ""
+  script:
+    - APP_VERSION=$(echo $CI_COMMIT_TAG | tr -d tr)
+    - IMAGE_VERSION=$(echo $CI_COMMIT_TAG | sed 's/tr/v/g')
+    - docker build -t $REPO/$PROD_TASK_RECONCILIATION:$IMAGE_VERSION -t $REPO/$PROD_TASK_RECONCILIATION:latest -t $PROD_TASK_RECONCILIATION:$IMAGE_VERSION -t $PROD_TASK_RECONCILIATION:latest --build-arg app_version=$APP_VERSION -f PARR.TaskReconciliationWorker/Dockerfile .
+    - docker login -u $HARBOR_PUSH_USER -p $HARBOR_PUSH_PASS $REPO
+    - docker push --all-tags $REPO/$PROD_TASK_RECONCILIATION
+  tags:
+    - docker
+
+
+prod_task_reconciliation_deploy:
+  stage: deploy
+  environment:
+    name: parr-task-reconciliation
+  only:
+    - /^tr[0-9]+\.[0-9]+\.[0-9]+$/
+  except:
+    - branches
+  script:
+    - IMAGE_VERSION=$(echo $CI_COMMIT_TAG | sed 's/tr/v/g')
+    - docker login -u $HARBOR_PULL_USER -p $HARBOR_PULL_PASS $REPO 
+    - tag=$IMAGE_VERSION docker stack deploy -c docker-compose.task-reconciliation.yml parr-task-reconciliation  --with-registry-auth
+  parallel:
+    matrix:
+      - RUNNER: shell-api-swarm-01
   tags:
     - ${RUNNER}

PARR.Core/Services/Task/Handlers/BaseTaskHandler.cs

@@ -226,14 +226,16 @@ namespace PARR.Core.Services.Task.Handlers
             // В простой реализации — выбрасываем событие, которое ловит воркер
 
             // Вариант: сохранить флаг, а воркер после ProcessAsync проверит и отправит
-            // Это будет реализовано в BackgroundService
 
             //todo: !!! Вот это делаем дальше!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
 
-            // тоже самое в WorkloadCacheBuilderService
-
             //todo:!!!!!!!!!
-            await System.Threading.Tasks.Task.Delay(50);
+
+            // Вообще это можно не использовать, так как если рэббит упал а мы еще раз тут же отправляем с задержкой, зачем туда отправлять если рэббит лежит?
+
+            await System.Threading.Tasks.Task.CompletedTask;
+
+            // await System.Threading.Tasks.Task.Delay(50);
         }
 
 

PARR.Core/Services/Workload/Implementations/WorkloadCacheBuilderService.cs

@@ -57,20 +57,10 @@ namespace PARR.Core.Services.Workload.Implementations
 
                     var result = await handler.ProcessAsync(queueMessage.TaskId);
 
-                    //// Если ошибка и есть retry — отправляем в очередь с задержкой
+                    if (result.IsSuccess)
-                    //if (!result.IsSuccess && handler is BaseTaskHandler baseHandler)
+                        logger.LogInformation("Задача {TaskId} успешно обработана", queueMessage.TaskId);
-                    //{
+                    else
-                    //    // TODO:
+                        logger.LogWarning("Задача {TaskId} завершена с ошибкой. Статус: {Status}", queueMessage.TaskId, result.ErrorMessage);
-                    //    // Если ошибка и есть retry — отправляем в очередь с задержкой
-                    //    // (это упрощенная реализация, в идеале — через ITaskQueueService)
-
-                    //    // Здесь нужна логика retry с задержкой
-                    //    // Можно реализовать через отдельный сервис
-
-                    //    //todo: !!! Вот это делаем дальше!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ->
-                    //    // тоже самое в BaseTaskHandler
-
-                    //}
                 }
             });
 

PARR.TaskReconciliationWorker/Dockerfile

@@ -0,0 +1,33 @@
+# See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
+
+# This stage is used when running from VS in fast mode (Default for Debug configuration)
+FROM 10.99.253.167:8090/dotnet/runtime:7.0 AS base
+WORKDIR /app
+
+
+# This stage is used to build the service project
+FROM 10.99.253.167:8090/dotnet/sdk:7.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["NuGet.config", "."]
+COPY ["PARR.TaskReconciliationWorker/PARR.TaskReconciliationWorker.csproj", "PARR.TaskReconciliationWorker/"]
+COPY ["PARR.Core/PARR.Core.csproj", "PARR.Core/"]
+COPY ["PARR.Domain/PARR.Domain.csproj", "PARR.Domain/"]
+COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
+COPY ["PARR.Infrastructure/PARR.Infrastructure.csproj", "PARR.Infrastructure/"]
+RUN dotnet restore "./PARR.TaskReconciliationWorker/PARR.TaskReconciliationWorker.csproj"
+COPY . .
+WORKDIR "/src/PARR.TaskReconciliationWorker"
+RUN dotnet build "./PARR.TaskReconciliationWorker.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+# This stage is used to publish the service project to be copied to the final stage
+FROM build AS publish
+ARG app_version=0.0.0-default
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "./PARR.TaskReconciliationWorker.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false /p:Version=$app_version
+
+# This stage is used in production or when running from VS in regular mode (Default when not using the Debug configuration)
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "PARR.TaskReconciliationWorker.dll"]

PARR.TaskReconciliationWorker/PARR.TaskReconciliationWorker.csproj

@@ -5,11 +5,13 @@
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>dotnet-PARR.TaskReconciliationWorker-90386488-d496-4f4d-8dcd-7dcc50882876</UserSecretsId>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Elastic.CommonSchema.Serilog" Version="8.6.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.23.0" />
     <PackageReference Include="Serilog.Extensions.Hosting" Version="7.0.0" />
     <PackageReference Include="Serilog.Settings.Configuration" Version="7.0.1" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />

PARR.TaskReconciliationWorker/Properties/launchSettings.json

@@ -1,11 +1,14 @@
-{
+{
   "profiles": {
     "PARR.TaskReconciliationWorker": {
       "commandName": "Project",
-      "dotnetRunMessages": true,
       "environmentVariables": {
         "DOTNET_ENVIRONMENT": "Development"
-      }
+      },
+      "dotnetRunMessages": true
+    },
+    "Container (Dockerfile)": {
+      "commandName": "Docker"
     }
   }
-}
+}

PARR.WorkloadBuilderWorker/Dockerfile

@@ -0,0 +1,33 @@
+# See https://aka.ms/customizecontainer to learn how to customize your debug container and how Visual Studio uses this Dockerfile to build your images for faster debugging.
+
+# This stage is used when running from VS in fast mode (Default for Debug configuration)
+FROM 10.99.253.167:8090/dotnet/runtime:7.0 AS base
+WORKDIR /app
+
+
+# This stage is used to build the service project
+FROM 10.99.253.167:8090/dotnet/sdk:7.0 AS build
+ARG BUILD_CONFIGURATION=Release
+WORKDIR /src
+COPY ["NuGet.config", "."]
+COPY ["PARR.WorkloadBuilderWorker/PARR.WorkloadBuilderWorker.csproj", "PARR.WorkloadBuilderWorker/"]
+COPY ["PARR.Core/PARR.Core.csproj", "PARR.Core/"]
+COPY ["PARR.Domain/PARR.Domain.csproj", "PARR.Domain/"]
+COPY ["PARR.DAL/PARR.DAL.csproj", "PARR.DAL/"]
+COPY ["PARR.Infrastructure/PARR.Infrastructure.csproj", "PARR.Infrastructure/"]
+RUN dotnet restore "./PARR.WorkloadBuilderWorker/PARR.WorkloadBuilderWorker.csproj"
+COPY . .
+WORKDIR "/src/PARR.WorkloadBuilderWorker"
+RUN dotnet build "./PARR.WorkloadBuilderWorker.csproj" -c $BUILD_CONFIGURATION -o /app/build
+
+# This stage is used to publish the service project to be copied to the final stage
+FROM build AS publish
+ARG app_version=0.0.0-default
+ARG BUILD_CONFIGURATION=Release
+RUN dotnet publish "./PARR.WorkloadBuilderWorker.csproj" -c $BUILD_CONFIGURATION -o /app/publish /p:UseAppHost=false /p:Version=$app_version
+
+# This stage is used in production or when running from VS in regular mode (Default when not using the Debug configuration)
+FROM base AS final
+WORKDIR /app
+COPY --from=publish /app/publish .
+ENTRYPOINT ["dotnet", "PARR.WorkloadBuilderWorker.dll"]

PARR.WorkloadBuilderWorker/PARR.WorkloadBuilderWorker.csproj

@@ -5,11 +5,13 @@
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>dotnet-PARR.WorkloadBuilderWorker-5cefb704-873b-40e7-b8a0-0537ca29e5ed</UserSecretsId>
+    <DockerDefaultTargetOS>Linux</DockerDefaultTargetOS>
   </PropertyGroup>
 
   <ItemGroup>
     <PackageReference Include="Elastic.CommonSchema.Serilog" Version="8.6.1" />
     <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" />
+    <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.23.0" />
     <PackageReference Include="Serilog.Extensions.Hosting" Version="7.0.0" />
     <PackageReference Include="Serilog.Settings.Configuration" Version="7.0.1" />
     <PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />

PARR.WorkloadBuilderWorker/Program.cs

@@ -1,7 +1,7 @@
 using Elastic.CommonSchema.Serilog;
 using PARR.Core;
 using PARR.DAL;
-using PARR.DAL.Services.Interfaces.Job;
+using PARR.Domain.Enums;
 using PARR.Infrastructure;
 using PARR.WorkloadBuilderWorker;
 using PARR.WorkloadBuilderWorker.Settings;
@@ -25,19 +25,27 @@ builder.Services.AddLogging(config =>
     config.AddSerilog(logger.CreateLogger());
 });
 
+var workerSettings = new WorkerSettings();
+builder.Configuration.GetSection(nameof(WorkerSettings)).Bind(workerSettings);
+builder.Services.AddSingleton(workerSettings);
+
 #region PARR modules
 builder.Services.InstallDalServices(builder.Configuration);
 builder.Services.AddCoreServices(builder.Configuration);
 builder.Services.AddInfrastructureServices(builder.Configuration);
+builder.Services.AddTaskManagement((provider, taskType) =>
+{
+    // Тут обрабатываем только задачи типа Workload
+    if (taskType == TaskTypeEnum.Workload)
+        return workerSettings.MqSettings;
+
+    throw new ArgumentException($"Настройки для типа задания {taskType} не найдены в конфигурации.");
+});
 
 builder.Configuration.AddDalConfigurations(builder.Services);
 builder.Services.AddDallSettings(builder.Configuration);
 #endregion
 
-var workerSettings = new WorkerSettings();
-builder.Configuration.GetSection(nameof(WorkerSettings)).Bind(workerSettings);
-builder.Services.AddSingleton(workerSettings);
-
 
 builder.Services.AddHostedService<Worker>();
 

PARR.WorkloadBuilderWorker/Properties/launchSettings.json

@@ -1,11 +1,14 @@
-{
+{
   "profiles": {
     "PARR.WorkloadBuilderWorker": {
       "commandName": "Project",
-      "dotnetRunMessages": true,
       "environmentVariables": {
         "DOTNET_ENVIRONMENT": "Development"
-      }
+      },
+      "dotnetRunMessages": true
+    },
+    "Container (Dockerfile)": {
+      "commandName": "Docker"
     }
   }
-}
+}

docker-compose.task-reconciliation.yml

@@ -0,0 +1,27 @@
+version: '3.4'
+
+# task reconciliation
+services:
+  parr-task-reconciliation:
+    image: harbor.dvgd.rzd/parr/parr-task-reconciliation:${tag:-latest}
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Production
+      - TZ=Europe/Moscow
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: dvgd-efk-01.dvgd.oao.rzd:24224
+        fluentd-retry-wait: '10s'
+        fluentd-max-retries: '30'
+        fluentd-async: 'true'
+        fluentd-buffer-limit: '52428800'
+        tag: parr.task-reconciliation.serilog
+    deploy:
+      replicas: 1
+    networks:
+      - parr-network
+
+networks:
+  parr-network:
+    driver: overlay
+    external: true  

docker-compose.workload-builder.yml

@@ -0,0 +1,27 @@
+version: '3.4'
+
+# workload builder
+services:
+  parr-workload-builder:
+    image: harbor.dvgd.rzd/parr/parr-workload-builder:${tag:-latest}
+    environment:
+      - ASPNETCORE_ENVIRONMENT=Production
+      - TZ=Europe/Moscow
+    logging:
+      driver: fluentd
+      options:
+        fluentd-address: dvgd-efk-01.dvgd.oao.rzd:24224
+        fluentd-retry-wait: '10s'
+        fluentd-max-retries: '30'
+        fluentd-async: 'true'
+        fluentd-buffer-limit: '52428800'
+        tag: parr.workload-builder.serilog
+    deploy:
+      replicas: 1
+    networks:
+      - parr-network
+
+networks:
+  parr-network:
+    driver: overlay
+    external: true  

stack.yml

@@ -1,75 +0,0 @@
-version: "3.4"
-
-# !!!not relevant!!!
-
-# services:
-#   parr-espp-template-sync:
-#     image: harbor.dvgd.rzd/parr/parr-espp-template-sync:${tag:-latest}
-#     container_name: parr-espp-template-sync
-#     restart: always
-#     environment:
-#       - ASPNETCORE_ENVIRONMENT=Production
-#       - TZ=Europe/Moscow
-#     #volumes:
-#       #- /var/log/parr-espp-template-sync:/app/log
-#       #- parr_storage_dev:/app/Data
-#     deploy:
-#       replicas: 2
-
-#   rabbitmq:
-#     image: rabbitmq:3.12.4-management
-#     container_name: rabbitmq
-#     hostname: parr-rabbitmq
-#     environment:
-#       - RABBITMQ_DEFAULT_USER=rmuser
-#       - RABBITMQ_DEFAULT_PASS=rmpassword
-#       - RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS=-rabbit log_levels [{connection,error},{default,error}] disk_free_limit 2147483648
-#     volumes:
-#       - parr_rabbit:/var/lib/rabbitmq
-#     ports:
-#       - 15672:15672
-#       - 5672:5672
-#     deploy:
-#       replicas: 1
-
-
-# volumes:
-#   parr_storage_dev:
-#     name: parr_storage_dev
-#     driver: local
-#     driver_opts:
-#       type: cifs
-#       o: "addr=dvgd-appfs-01,username=parruserdev,password=P@ssDevParr202MM"
-#       device: //dvgd-appfs-01/parr_storage_dev
-#   parr_rabbit:
-#     name: parr_rabbit
-#     driver_opts:
-#       type: "nfs"
-#       o: "addr=dvgd-appfs-01.dvgd.oao.rzd,soft,rw,nfsvers=4"
-#       device: ":/nfs/dvgd-swarm-01/parr_rabbit/data"
-
-
-  # # --- NGINX ---
-  # nginx-2:
-  #     image: nginx:latest
-  #     ports:
-  #       - '8089:80'
-  #     deploy:
-  #       replicas: 5
-  #       update_config:
-  #         parallelism: 2
-  #         order: start-first
-  #         failure_action: rollback
-  #         delay: 5s
-  #       rollback_config:
-  #         parallelism: 0
-  #         order: stop-first
-  #       restart_policy:
-  #         condition: any
-  #         delay: 5s
-  #         max_attempts: 3
-  #         window: 120s
-  #     healthcheck:
-  #       test: ["CMD", "service", "nginx", "status"]
-  #     networks:
-  #       - nginx-2