42 lines
3.6 KiB
C#
42 lines
3.6 KiB
C#
using Dapper;
|
|
using Kbx.Shared.Experiments.Generated;
|
|
using Npgsql;
|
|
namespace Kbx.Shared.Experiments;
|
|
public sealed class KbxExperimentOverviewQuery(NpgsqlDataSource dataSource)
|
|
{
|
|
public async Task<KbxExperimentOverviewResponse> ExecuteAsync(Guid tenantId,CancellationToken ct)
|
|
{
|
|
await using var connection=await dataSource.OpenConnectionAsync(ct);var items=new List<KbxExperimentOverviewRow>();
|
|
foreach(var def in KbxExperimentCatalog.Experiments.Values)
|
|
{
|
|
var runtime=await connection.QuerySingleOrDefaultAsync<RuntimeRow>(new CommandDefinition("select state State,rollout_percent RolloutPercent,kill_switch KillSwitch,updated_at UpdatedAt from kbx.experiment_runtime where tenant_id=@TenantId and experiment_id=@Id",new{TenantId=tenantId,Id=def.Id},cancellationToken:ct));
|
|
var since=runtime?.UpdatedAt ?? DateTimeOffset.UtcNow.AddDays(-30);
|
|
var metrics=(await connection.QueryAsync<MetricRow>(new CommandDefinition("""
|
|
select experiment_variant Variant,
|
|
count(*) filter(where event_name='experiment.exposed') Exposures,
|
|
count(*) filter(where event_name='task.complete') TaskCompletions,
|
|
count(*) filter(where event_name='interaction.execute') Interactions,
|
|
count(*) filter(where event_name='command.execute') Commands,
|
|
count(*) filter(where event_name='validation.failed') ValidationFailures,
|
|
percentile_cont(0.95) within group(order by duration_ms) filter(where event_name='task.complete' and duration_ms is not null) TaskP95
|
|
from kbx.ux_events where tenant_id=@TenantId and experiment_id=@Id and occurred_at>=@Since
|
|
group by experiment_variant;
|
|
""",new{TenantId=tenantId,Id=def.Id,Since=since},cancellationToken:ct))).Select(x=>new KbxExperimentVariantMetric(x.Variant,x.Exposures,x.TaskCompletions,x.TaskCompletions==0?null:Math.Round((double)x.Interactions/x.TaskCompletions,2),x.TaskP95,x.Commands==0?null:Math.Round((double)x.ValidationFailures*100d/x.Commands,2))).ToArray();
|
|
var state=runtime?.State??def.State;var decision=Decide(def,state,runtime?.KillSwitch??false,metrics);
|
|
items.Add(new(def.Id,def.ScreenId,state,runtime?.RolloutPercent??def.RolloutPercent,runtime?.KillSwitch??false,decision,metrics,runtime?.UpdatedAt));
|
|
}
|
|
return new(items);
|
|
}
|
|
private static string Decide(KbxExperimentDefinition def,string state,bool killed,IReadOnlyList<KbxExperimentVariantMetric> rows)
|
|
{
|
|
if(killed||state=="rolled-back")return "rolled-back";var control=rows.FirstOrDefault(x=>x.Variant=="control"),treatment=rows.FirstOrDefault(x=>x.Variant!="control");
|
|
if(control is null||treatment is null||control.Exposures<def.MinExposurePerVariant||treatment.Exposures<def.MinExposurePerVariant)return "inconclusive";
|
|
if(control.TaskCompletionP95Ms is >0 && treatment.TaskCompletionP95Ms>control.TaskCompletionP95Ms*1.20)return "guardrail-breach";
|
|
if(control.ValidationFailureRate is not null&&treatment.ValidationFailureRate>control.ValidationFailureRate+2)return "guardrail-breach";
|
|
if(control.InteractionsPerTask is >0&&treatment.InteractionsPerTask<=control.InteractionsPerTask*(1-def.MinimumImprovementPercent/100d))return "candidate";
|
|
return "neutral";
|
|
}
|
|
private sealed record RuntimeRow(string State,int RolloutPercent,bool KillSwitch,DateTimeOffset UpdatedAt);
|
|
private sealed record MetricRow(string Variant,long Exposures,long TaskCompletions,long Interactions,long Commands,long ValidationFailures,double? TaskP95);
|
|
}
|