diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 4bd748c..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,71 +0,0 @@ -version: '3.8' -services: - collector: - build: - context: . - dockerfile: src/DataAggregator.Collector/Dockerfile - ports: - - "5000:80" - environment: - - CollectorType=OpenCN - - ASPNETCORE_ENVIRONMENT=Development - depends_on: - - influxdb - - registration - - registration: - build: - context: . - dockerfile: src/DataAggregator.Registration/Dockerfile - ports: - - "5001:80" - environment: - - ASPNETCORE_ENVIRONMENT=Development - - PGHOST=postgres - - PGPORT=5432 - - PGDATABASE=dataaggregator - - PGUSER=admin - - PGPASSWORD=password123 - depends_on: - - postgres - - postgres: - image: postgres:15 - environment: - POSTGRES_DB: dataaggregator - POSTGRES_USER: admin - POSTGRES_PASSWORD: password123 - ports: - - "5432:5432" - volumes: - - postgres_data:/var/lib/postgresql/data - - influxdb: - image: influxdb:3-core - ports: - - "8181:8181" - environment: - INFLUXDB3_AUTH_TOKEN: your-token - command: - - influxdb3 - - serve - - --node-id=node0 - - --object-store=file - - --data-dir=/var/lib/influxdb3 - volumes: - - influx_data:/var/lib/influxdb3 - - influxdb-ui: - image: influxdata/influxdb3-ui:1.0.0 - container_name: influxdb3-explorer - ports: - - "8888:80" - environment: - INFLUXDB_URL: http://influxdb:8181 - command: ["--mode=admin"] - depends_on: - - influxdb - -volumes: - postgres_data: - influx_data: diff --git a/docker/Dockerfile.Collector b/docker/Dockerfile.Collector new file mode 100644 index 0000000..3e8f84d --- /dev/null +++ b/docker/Dockerfile.Collector @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build +WORKDIR /src + +COPY NuGet.config /NuGet.config +COPY lib /lib + +COPY src/DataAggregator.Collector/*.csproj ./DataAggregator.Collector/ +COPY src/DataAggregator.Shared/*.csproj ./DataAggregator.Shared/ +COPY src/DataAggregator.Collector.OpenCNCapnProtoConnector/*.csproj ./DataAggregator.Collector.OpenCNCapnProtoConnector/ + +RUN dotnet restore ./DataAggregator.Collector/DataAggregator.Collector.App.csproj --configfile /NuGet.config + +COPY src/ ./ +WORKDIR /src/DataAggregator.Collector +RUN dotnet publish -c Release -o /app/publish --configfile /NuGet.config + +FROM mcr.microsoft.com/dotnet/aspnet:9.0 +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /app/publish . +ENTRYPOINT ["dotnet", "DataAggregator.Collector.App.dll"] diff --git a/docker/Dockerfile.Processor b/docker/Dockerfile.Processor new file mode 100644 index 0000000..5bf8349 --- /dev/null +++ b/docker/Dockerfile.Processor @@ -0,0 +1,17 @@ +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build +WORKDIR /src + +COPY src/DataAggregator.Processor/*.csproj ./DataAggregator.Processor/ +COPY src/DataAggregator.Shared/*.csproj ./DataAggregator.Shared/ + +RUN dotnet restore ./DataAggregator.Processor/DataAggregator.Processor.csproj + +COPY src/ ./ +WORKDIR /src/DataAggregator.Processor +RUN dotnet publish -c Release -o /app/publish + +FROM mcr.microsoft.com/dotnet/aspnet:9.0 +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /app/publish . +ENTRYPOINT ["dotnet", "DataAggregator.Processor.dll"] diff --git a/docker/Dockerfile.Registration b/docker/Dockerfile.Registration new file mode 100644 index 0000000..5ae6a54 --- /dev/null +++ b/docker/Dockerfile.Registration @@ -0,0 +1,17 @@ +FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build +WORKDIR /src + +COPY src/DataAggregator.Registration/*.csproj ./DataAggregator.Registration/ +COPY src/DataAggregator.Shared/*.csproj ./DataAggregator.Shared/ + +RUN dotnet restore ./DataAggregator.Registration/DataAggregator.Registration.csproj + +COPY src/ ./ +WORKDIR /src/DataAggregator.Registration +RUN dotnet publish -c Release -o /app/publish + +FROM mcr.microsoft.com/dotnet/aspnet:9.0 +RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /app/publish . +ENTRYPOINT ["dotnet", "DataAggregator.Registration.dll"] diff --git a/env/docker-compose.yml b/env/docker-compose.yml new file mode 100644 index 0000000..bb3dede --- /dev/null +++ b/env/docker-compose.yml @@ -0,0 +1,54 @@ +services: + postgres: + image: postgres:15 + environment: + POSTGRES_DB: dataaggregator + POSTGRES_USER: admin + POSTGRES_PASSWORD: password123 + ports: + - "5432:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + + influxdb: + image: influxdb:3-core + ports: + - "8181:8181" + environment: + INFLUXDB3_AUTH_TOKEN: YOUR-TOKEN + command: + - influxdb3 + - serve + - --node-id=node0 + - --object-store=file + - --data-dir=/var/lib/influxdb3 + volumes: + - influx_data:/var/lib/influxdb3 + + influxdb-ui: + image: influxdata/influxdb3-ui:1.0.0 + container_name: influxdb3-explorer + ports: + - "8888:80" + environment: + INFLUXDB_URL: http://influxdb:8181 + command: [ "--mode=admin" ] + depends_on: + - influxdb + + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - "3001:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=admin + volumes: + - grafana_data:/var/lib/grafana + - ./volumes/grafana/data:/data/ + +volumes: + postgres_data: + influx_data: + grafana_data: diff --git a/src/DataAggregator.Collector/DataAggregator.Collector.http b/src/DataAggregator.Collector/DataAggregator.Collector.http deleted file mode 100644 index ea6bd0b..0000000 --- a/src/DataAggregator.Collector/DataAggregator.Collector.http +++ /dev/null @@ -1,16 +0,0 @@ -@DataAggregator.Collector_HostAddress = http://localhost:5091 - -GET {{DataAggregator.Collector_HostAddress}}/weatherforecast/ -Accept: application/json - -### - -@apiBaseUrl = http://localhost:5000 - -### Health Check -GET {{apiBaseUrl}}/api/healthcheck -Accept: application/json - -### Direct Health Endpoint -GET {{apiBaseUrl}}/health -Accept: application/json diff --git a/src/DataAggregator.Collector/Dockerfile b/src/DataAggregator.Collector/Dockerfile deleted file mode 100644 index e748846..0000000 --- a/src/DataAggregator.Collector/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base -WORKDIR /app -EXPOSE 80 - -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build -WORKDIR /src -COPY ["src/DataAggregator.Collector/DataAggregator.Collector.csproj", "src/DataAggregator.Collector/"] -COPY ["src/DataAggregator.Shared/DataAggregator.Shared.csproj", "src/DataAggregator.Shared/"] -RUN dotnet restore "src/DataAggregator.Collector/DataAggregator.Collector.csproj" -COPY . . -WORKDIR "/src/src/DataAggregator.Collector" -RUN dotnet build "DataAggregator.Collector.csproj" -c Release -o /app/build - -FROM build AS publish -RUN dotnet publish "DataAggregator.Collector.csproj" -c Release -o /app/publish - -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "DataAggregator.Collector.dll"] diff --git a/src/DataAggregator.Collector/Program.cs b/src/DataAggregator.Collector/Program.cs index b569324..5a8aaa1 100644 --- a/src/DataAggregator.Collector/Program.cs +++ b/src/DataAggregator.Collector/Program.cs @@ -5,6 +5,7 @@ using DataAggregator.Collector.Shared.DataStorage.Influx; using DataAggregator.Collector.Shared.LocalStorage; using DataAggregator.Collector.Shared.Registration; +using Microsoft.AspNetCore.HttpOverrides; using Microsoft.Extensions.Options; using Serilog; @@ -13,8 +14,6 @@ // Configure Serilog from appsettings.json Log.Logger = new LoggerConfiguration() .ReadFrom.Configuration(builder.Configuration) - .WriteTo.Console() - .Enrich.FromLogContext() .CreateLogger(); builder.Host.UseSerilog(); @@ -24,6 +23,16 @@ builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(c => c.SwaggerDoc("v1", new() { Title = "DataAggregator Collector API", Version = "v1" })); +if (!builder.Environment.IsDevelopment()) +{ + builder.Services.Configure(options => + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + options.KnownNetworks.Clear(); + options.KnownProxies.Clear(); + }); +} + // Get collector type from configuration string? collectorType = builder.Configuration["CollectorType"]; @@ -37,11 +46,12 @@ SetupConfiguration(builder); // Configure HTTP clients -builder.Services.AddHttpClient(); +string registrationBaseUrl = builder.Configuration["Collector:RegistrationService:BaseUrl"] ?? "http://localhost:5000"; +string registrationEndpoint = builder.Configuration["Collector:RegistrationService:Endpoint"] ?? "api/DeviceRegistration/register"; + builder.Services.AddHttpClient("RegistrationClient", client => { - string registrationEndpoint = builder.Configuration["RegistrationService:Endpoint"] ?? "http://localhost:5001"; - client.BaseAddress = new Uri(registrationEndpoint); + client.BaseAddress = new Uri(registrationBaseUrl); client.DefaultRequestHeaders.Add("Accept", "application/json"); }); @@ -60,14 +70,6 @@ IHttpClientFactory httpClientFactory = sp.GetRequiredService(); HttpClient httpClient = httpClientFactory.CreateClient("RegistrationClient"); - string? registrationEndpoint = builder.Configuration["Collector:RegistrationService:Endpoint"]; - - if (string.IsNullOrEmpty(registrationEndpoint)) - { - Log.Warning("Registration service endpoint not configured, using default: http://localhost:5001/api/DeviceRegistration/register"); - registrationEndpoint = "http://localhost:5001/api/DeviceRegistration/register"; - } - return new RegistrationService(httpClient, registrationEndpoint); }); @@ -117,8 +119,11 @@ }); } -app.UseHttpsRedirection(); -app.UseAuthorization(); +if (!app.Environment.IsDevelopment()) +{ + app.UseForwardedHeaders(); +} + app.MapControllers(); app.MapHealthChecks("/health"); diff --git a/src/DataAggregator.Collector/appsettings.json b/src/DataAggregator.Collector/appsettings.json index 83198c7..3dbac9b 100644 --- a/src/DataAggregator.Collector/appsettings.json +++ b/src/DataAggregator.Collector/appsettings.json @@ -1,20 +1,28 @@ { - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, + "AllowedHosts": "*", + "Serilog": { "MinimumLevel": { "Default": "Information", "Override": { - "Microsoft": "Warning", + "Microsoft.AspNetCore": "Warning", "System": "Warning" } + }, + "WriteTo": [ + { + "Name": "Console", + "Args": { + "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {ServiceName}: {Message:lj}{NewLine}{Exception}" + } + } + ], + "Enrich": [ "FromLogContext" ], + "Properties": { + "ServiceName": "Collector" } }, - "AllowedHosts": "*", + "CollectorType": "OpenCN", "Collector": { "DeviceName": "Micro5", @@ -27,7 +35,8 @@ "Port": 7002 }, "RegistrationService": { - "Endpoint": "http://localhost:5137/api/DeviceRegistration/register" + "BaseUrl": "http://localhost:5137", + "Endpoint": "api/DeviceRegistration/register" }, "BufferSettings": { "MaxBufferSize": 10000 diff --git a/src/DataAggregator.Processor/Dockerfile b/src/DataAggregator.Processor/Dockerfile deleted file mode 100644 index 061cb5e..0000000 --- a/src/DataAggregator.Processor/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM mcr.microsoft.com/dotnet/aspnet:9.0 AS base -WORKDIR /app -EXPOSE 80 -EXPOSE 443 - -FROM mcr.microsoft.com/dotnet/sdk:9.0 AS build -WORKDIR /src -COPY ["src/DataAggregator.Processor/DataAggregator.Processor.csproj", "src/DataAggregator.Processor/"] -COPY ["src/DataAggregator.Shared/DataAggregator.Shared.csproj", "src/DataAggregator.Shared/"] -COPY ["src/DataAggregator.Collector.Shared/DataAggregator.Collector.Shared.csproj", "src/DataAggregator.Collector.Shared/"] -RUN dotnet restore "src/DataAggregator.Processor/DataAggregator.Processor.csproj" -COPY . . -WORKDIR "/src/src/DataAggregator.Processor" -RUN dotnet build "DataAggregator.Processor.csproj" -c Release -o /app/build - -FROM build AS publish -RUN dotnet publish "DataAggregator.Processor.csproj" -c Release -o /app/publish /p:UseAppHost=false - -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . - -# Create models directory for ONNX models -RUN mkdir -p /app/models - -# Set environment variables -ENV ASPNETCORE_URLS=http://+:80 -ENV ASPNETCORE_ENVIRONMENT=Production - -ENTRYPOINT ["dotnet", "DataAggregator.Processor.dll"] \ No newline at end of file diff --git a/src/DataAggregator.Processor/Program.cs b/src/DataAggregator.Processor/Program.cs index 6827537..b8f3648 100644 --- a/src/DataAggregator.Processor/Program.cs +++ b/src/DataAggregator.Processor/Program.cs @@ -5,6 +5,7 @@ using DataAggregator.Processor.Services.Prediction; using DataAggregator.Processor.Services.Processing.Factory; using DataAggregator.Processor.Services.Registration; +using Microsoft.AspNetCore.HttpOverrides; using Serilog; WebApplicationBuilder builder = WebApplication.CreateBuilder(args); @@ -12,8 +13,6 @@ // Configure Serilog from appsettings.json Log.Logger = new LoggerConfiguration() .ReadFrom.Configuration(builder.Configuration) - .WriteTo.Console() - .Enrich.FromLogContext() .CreateLogger(); builder.Host.UseSerilog(); @@ -23,6 +22,16 @@ builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(c => c.SwaggerDoc("v1", new() { Title = "DataAggregator Processor API", Version = "v1" })); +if (!builder.Environment.IsDevelopment()) +{ + builder.Services.Configure(options => + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + options.KnownNetworks.Clear(); + options.KnownProxies.Clear(); + }); +} + // Register health checks builder.Services.AddHealthChecks(); @@ -68,8 +77,11 @@ }); } -app.UseHttpsRedirection(); -app.UseAuthorization(); +if (!app.Environment.IsDevelopment()) +{ + app.UseForwardedHeaders(); +} + app.MapControllers(); app.MapHealthChecks("/health"); diff --git a/src/DataAggregator.Processor/appsettings.json b/src/DataAggregator.Processor/appsettings.json index d08c2b4..797729f 100644 --- a/src/DataAggregator.Processor/appsettings.json +++ b/src/DataAggregator.Processor/appsettings.json @@ -1,23 +1,28 @@ { - "Logging": { - "LogLevel": { - "Default": "Information", - "Microsoft.AspNetCore": "Warning" - } - }, + "AllowedHosts": "*", "Serilog": { "MinimumLevel": { "Default": "Information", "Override": { - "Microsoft": "Warning", + "Microsoft.AspNetCore": "Warning", "System": "Warning" } + }, + "WriteTo": [ + { + "Name": "Console", + "Args": { + "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {ServiceName}: {Message:lj}{NewLine}{Exception}" + } + } + ], + "Enrich": [ "FromLogContext" ], + "Properties": { + "ServiceName": "Processor" } }, - "AllowedHosts": "*", - "PredictionService": { "RegistrationServiceUrl": "http://localhost:5137", "Machines": [ diff --git a/src/DataAggregator.Registration/DataAggregator.Registration.http b/src/DataAggregator.Registration/DataAggregator.Registration.http deleted file mode 100644 index 4e41139..0000000 --- a/src/DataAggregator.Registration/DataAggregator.Registration.http +++ /dev/null @@ -1,6 +0,0 @@ -@DataAggregator.Registration_HostAddress = http://localhost:5137 - -GET {{DataAggregator.Registration_HostAddress}}/weatherforecast/ -Accept: application/json - -### diff --git a/src/DataAggregator.Registration/Dockerfile b/src/DataAggregator.Registration/Dockerfile deleted file mode 100644 index 383d17a..0000000 --- a/src/DataAggregator.Registration/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM mcr.microsoft.com/dotnet/aspnet:8.0 AS base -WORKDIR /app -EXPOSE 80 - -FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build -WORKDIR /src -COPY ["src/DataAggregator.Registration/DataAggregator.Registration.csproj", "src/DataAggregator.Registration/"] -COPY ["src/DataAggregator.Shared/DataAggregator.Shared.csproj", "src/DataAggregator.Shared/"] -RUN dotnet restore "src/DataAggregator.Registration/DataAggregator.Registration.csproj" -COPY . . -WORKDIR "/src/src/DataAggregator.Registration" -RUN dotnet build "DataAggregator.Registration.csproj" -c Release -o /app/build - -FROM build AS publish -RUN dotnet publish "DataAggregator.Registration.csproj" -c Release -o /app/publish - -FROM base AS final -WORKDIR /app -COPY --from=publish /app/publish . -ENTRYPOINT ["dotnet", "DataAggregator.Registration.dll"] diff --git a/src/DataAggregator.Registration/Program.cs b/src/DataAggregator.Registration/Program.cs index 28687d7..ea9f352 100644 --- a/src/DataAggregator.Registration/Program.cs +++ b/src/DataAggregator.Registration/Program.cs @@ -4,6 +4,7 @@ using DataAggregator.Registration.DeviceManagement.Services; using DataAggregator.Registration.InfluxService.Configuration; using DataAggregator.Registration.InfluxService.Services; +using Microsoft.AspNetCore.HttpOverrides; using Microsoft.EntityFrameworkCore; using Serilog; @@ -11,9 +12,7 @@ // Configure Serilog from appsettings.json Log.Logger = new LoggerConfiguration() - .ReadFrom.Configuration(builder.Configuration) // Read configuration from appsettings.json - .WriteTo.Console() // Write logs to the console - .Enrich.FromLogContext() // Add context to logs + .ReadFrom.Configuration(builder.Configuration) .CreateLogger(); builder.Host.UseSerilog(); // Use Serilog as the logging provider @@ -23,20 +22,20 @@ builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(c => c.SwaggerDoc("v1", new() { Title = "DataAggregator API", Version = "v1" })); -// Configure Entity Framework Core with PostgreSQL -string? pgHost = builder.Configuration["PGHOST"]; -string? pgPort = builder.Configuration["PGPORT"]; -string? pgDb = builder.Configuration["PGDATABASE"]; -string? pgUser = builder.Configuration["PGUSER"]; -string? pgPassword = builder.Configuration["PGPASSWORD"]; - -if (string.IsNullOrEmpty(pgHost) || string.IsNullOrEmpty(pgDb) || string.IsNullOrEmpty(pgUser) || string.IsNullOrEmpty(pgPassword)) +// Configure forwarded headers for reverse proxy scenarios +if (!builder.Environment.IsDevelopment()) { - Log.Fatal("PostgreSQL environment variables are not properly configured."); - throw new InvalidOperationException("PostgreSQL environment variables are not properly configured."); + builder.Services.Configure(options => + { + options.ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto; + options.KnownNetworks.Clear(); + options.KnownProxies.Clear(); + }); } -string connectionString = $"Host={pgHost};Port={pgPort};Database={pgDb};Username={pgUser};Password={pgPassword}"; +// Configure Entity Framework Core with PostgreSQL +string connectionString = builder.Configuration.GetConnectionString("DefaultConnection") + ?? throw new InvalidOperationException("Connection string 'DefaultConnection' not found in configuration."); builder.Services.AddDbContext(options => options.UseNpgsql(connectionString)); @@ -86,14 +85,12 @@ c.RoutePrefix = string.Empty; }); } -else + +if (!app.Environment.IsDevelopment()) { - app.UseHsts(); + app.UseForwardedHeaders(); // Use forwarded headers in production } -app.UseHttpsRedirection(); -app.UseAuthorization(); - app.MapControllers(); app.MapHealthChecks("/health"); diff --git a/src/DataAggregator.Registration/appsettings.json b/src/DataAggregator.Registration/appsettings.json index fd0b67b..4ff9b31 100644 --- a/src/DataAggregator.Registration/appsettings.json +++ b/src/DataAggregator.Registration/appsettings.json @@ -1,17 +1,37 @@ { - "Logging": { - "LogLevel": { + "AllowedHosts": "*", + + "Serilog": { + "MinimumLevel": { "Default": "Information", - "Microsoft.AspNetCore": "Warning" + "Override": { + "Microsoft.AspNetCore": "Warning", + "System": "Warning" + } + }, + "WriteTo": [ + { + "Name": "Console", + "Args": { + "outputTemplate": "[{Timestamp:HH:mm:ss} {Level:u3}] {ServiceName}: {Message:lj}{NewLine}{Exception}" + } + } + ], + "Enrich": [ "FromLogContext" ], + "Properties": { + "ServiceName": "Registration" } }, - "AllowedHosts": "*", + + "ConnectionStrings": { + "DefaultConnection": "Server=localhost;Port=5432;Database=dataaggregator;User Id=admin;Password=password123;" + }, "Influx": { "Endpoints": [ { "Name": "DefaultEndpoint", - "Token": "apiv3_ebQj3GBz6CuRk3Xo6XXXP2daOdE790MvqrIAYf4SJLC1Br46gxHq7vM5YZpouYr9-17bYQbDqGFus6MrIe3ClQ", + "Token": "YOUR_TOKEN", "Endpoint": "http://localhost:8181" } ]