feat: export PostgreSQL training dataset as JSON
This commit is contained in:
@@ -0,0 +1,29 @@
|
|||||||
|
using System.Text.Json;
|
||||||
|
using QuantEngine.Core.Interfaces;
|
||||||
|
|
||||||
|
namespace QuantEngine.Application.Services;
|
||||||
|
|
||||||
|
public sealed class LearningDatasetService
|
||||||
|
{
|
||||||
|
private readonly ILearningDatasetReader _reader;
|
||||||
|
|
||||||
|
public LearningDatasetService(ILearningDatasetReader reader) => _reader = reader;
|
||||||
|
|
||||||
|
public async Task<string> ExportJsonAsync(string outputPath, int limit = 1000)
|
||||||
|
{
|
||||||
|
var rows = await _reader.ReadTrainingExamplesAsync(limit);
|
||||||
|
var payload = new
|
||||||
|
{
|
||||||
|
formula_id = "ENGINE_HISTORY_TRAINING_DATASET_V1",
|
||||||
|
gate = rows.Count > 0 ? "PASS" : "DATA_MISSING",
|
||||||
|
generated_at = DateTimeOffset.UtcNow,
|
||||||
|
sample_count = rows.Count,
|
||||||
|
source = "engine_history.training_example_v1",
|
||||||
|
rows
|
||||||
|
};
|
||||||
|
var path = Path.GetFullPath(outputPath);
|
||||||
|
Directory.CreateDirectory(Path.GetDirectoryName(path)!);
|
||||||
|
await File.WriteAllTextAsync(path, JsonSerializer.Serialize(payload, new JsonSerializerOptions { WriteIndented = true }));
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
namespace QuantEngine.Core.Interfaces;
|
||||||
|
|
||||||
|
public interface ILearningDatasetReader
|
||||||
|
{
|
||||||
|
Task<IReadOnlyList<IDictionary<string, object?>>> ReadTrainingExamplesAsync(int limit = 1000);
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
using Dapper;
|
||||||
|
using QuantEngine.Core.Interfaces;
|
||||||
|
using QuantEngine.Infrastructure.Data;
|
||||||
|
|
||||||
|
namespace QuantEngine.Infrastructure.Repositories;
|
||||||
|
|
||||||
|
public sealed class LearningDatasetReader : ILearningDatasetReader
|
||||||
|
{
|
||||||
|
private readonly IDbConnectionFactory _connectionFactory;
|
||||||
|
|
||||||
|
public LearningDatasetReader(IDbConnectionFactory connectionFactory) => _connectionFactory = connectionFactory;
|
||||||
|
|
||||||
|
public async Task<IReadOnlyList<IDictionary<string, object?>>> ReadTrainingExamplesAsync(int limit = 1000)
|
||||||
|
{
|
||||||
|
if (limit is < 1 or > 10000)
|
||||||
|
throw new ArgumentOutOfRangeException(nameof(limit));
|
||||||
|
|
||||||
|
using var connection = _connectionFactory.CreateConnection();
|
||||||
|
var rows = await connection.QueryAsync(
|
||||||
|
"SELECT * FROM engine_history.training_example_v1 ORDER BY decided_at DESC LIMIT @Limit",
|
||||||
|
new { Limit = limit });
|
||||||
|
return rows.Select(row => (IDictionary<string, object?>)row).ToList();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -97,7 +97,9 @@ try
|
|||||||
builder.Services.AddScoped<IWorkspaceRepository, WorkspaceRepository>();
|
builder.Services.AddScoped<IWorkspaceRepository, WorkspaceRepository>();
|
||||||
builder.Services.AddScoped<IPostgresqlHistoryStore, PostgresqlHistoryStore>();
|
builder.Services.AddScoped<IPostgresqlHistoryStore, PostgresqlHistoryStore>();
|
||||||
builder.Services.AddScoped<INormalizedLearningStore, NormalizedLearningStore>();
|
builder.Services.AddScoped<INormalizedLearningStore, NormalizedLearningStore>();
|
||||||
|
builder.Services.AddScoped<ILearningDatasetReader, LearningDatasetReader>();
|
||||||
builder.Services.AddScoped<DecisionLearningService>();
|
builder.Services.AddScoped<DecisionLearningService>();
|
||||||
|
builder.Services.AddScoped<LearningDatasetService>();
|
||||||
builder.Services.AddScoped<IPostgresqlHistorySnapshotReader, PostgresqlHistorySnapshotReader>();
|
builder.Services.AddScoped<IPostgresqlHistorySnapshotReader, PostgresqlHistorySnapshotReader>();
|
||||||
builder.Services.AddScoped<HistoryIngestionService>();
|
builder.Services.AddScoped<HistoryIngestionService>();
|
||||||
builder.Services.AddScoped<ICollectionRepository, CollectionRepository>();
|
builder.Services.AddScoped<ICollectionRepository, CollectionRepository>();
|
||||||
|
|||||||
Reference in New Issue
Block a user