V13-FE-011: finalize search list layout slice
This commit is contained in:
+18
@@ -0,0 +1,18 @@
|
||||
// generated from contracts/experiments/kbx.experiments.json; do not edit.
|
||||
namespace Kbx.Shared.Experiments.Generated;
|
||||
|
||||
public sealed record KbxFeatureFlagDefinition(string Id,string ScreenId,string Surface,string DefaultVariant,bool KillSwitch);
|
||||
public sealed record KbxVariantWeight(string Key,int Weight);
|
||||
public sealed record KbxExperimentDefinition(string Id,string FlagId,string ScreenId,string State,int RolloutPercent,int MinExposurePerVariant,IReadOnlyList<KbxVariantWeight> Variants,string PrimaryMetric,string PrimaryDirection,double MinimumImprovementPercent);
|
||||
public static class KbxExperimentCatalog
|
||||
{
|
||||
public const string SourceSha256 = "2510a2927bab297858e09e75c8af9bb766d91b92cc0daf99433f6377d399fbf9";
|
||||
public static readonly IReadOnlyDictionary<string,KbxFeatureFlagDefinition> FeatureFlags = new Dictionary<string,KbxFeatureFlagDefinition>(StringComparer.Ordinal)
|
||||
{
|
||||
["oms.order-list.exception-summary-v2"] = new("oms.order-list.exception-summary-v2", "OMS-ORD-001", "information-emphasis", "control", true)
|
||||
};
|
||||
public static readonly IReadOnlyDictionary<string,KbxExperimentDefinition> Experiments = new Dictionary<string,KbxExperimentDefinition>(StringComparer.Ordinal)
|
||||
{
|
||||
["exp.oms.order-list.exception-summary-v2"] = new("exp.oms.order-list.exception-summary-v2", "oms.order-list.exception-summary-v2", "OMS-ORD-001", "draft", 0, 200, new[] { new KbxVariantWeight("control", 50), new KbxVariantWeight("exception-summary", 50) }, "semantic_interactions_per_task", "lower", 10)
|
||||
};
|
||||
}
|
||||
+20
@@ -0,0 +1,20 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using Kbx.Shared.Experiments.Generated;
|
||||
namespace Kbx.Shared.Experiments;
|
||||
public static class KbxExperimentAssignmentPolicy
|
||||
{
|
||||
public static bool IsEnrolled(Guid tenantId,Guid userId,string experimentId,int rolloutPercent) =>
|
||||
rolloutPercent>0 && Bucket(tenantId,userId,experimentId,"enroll") < rolloutPercent*100;
|
||||
public static string ChooseVariant(Guid tenantId,Guid userId,KbxExperimentDefinition definition)
|
||||
{
|
||||
var point=Bucket(tenantId,userId,definition.Id,"variant")%100,cumulative=0;
|
||||
foreach(var item in definition.Variants){cumulative+=item.Weight;if(point<cumulative)return item.Key;}
|
||||
return "control";
|
||||
}
|
||||
private static int Bucket(Guid tenantId,Guid userId,string experimentId,string purpose)
|
||||
{
|
||||
var bytes=SHA256.HashData(Encoding.UTF8.GetBytes($"{tenantId:N}|{userId:N}|{experimentId}|{purpose}|kbx-v1"));
|
||||
return (int)(BitConverter.ToUInt32(bytes,0)%10000);
|
||||
}
|
||||
}
|
||||
+45
@@ -0,0 +1,45 @@
|
||||
using Dapper;
|
||||
using Kbx.Shared.Experiments.Generated;
|
||||
using Npgsql;
|
||||
|
||||
namespace Kbx.Shared.Experiments;
|
||||
|
||||
public sealed class KbxExperimentAssignmentService(NpgsqlDataSource dataSource)
|
||||
{
|
||||
public async Task<KbxExperimentAssignmentsResponse> EvaluateAsync(Guid tenantId,Guid userId,string screenId,CancellationToken ct)
|
||||
{
|
||||
var definitions=KbxExperimentCatalog.Experiments.Values.Where(x=>x.ScreenId==screenId).ToArray();
|
||||
var items=new List<KbxExperimentAssignmentDto>();
|
||||
await using var connection=await dataSource.OpenConnectionAsync(ct);
|
||||
foreach(var definition in definitions)
|
||||
{
|
||||
var runtime=await connection.QuerySingleOrDefaultAsync<RuntimeRow>(new CommandDefinition("""
|
||||
select state State,rollout_percent RolloutPercent,kill_switch KillSwitch
|
||||
from kbx.experiment_runtime where tenant_id=@TenantId and experiment_id=@ExperimentId;
|
||||
""",new{TenantId=tenantId,ExperimentId=definition.Id},cancellationToken:ct));
|
||||
var state=runtime?.State ?? definition.State;
|
||||
var rollout=runtime?.RolloutPercent ?? definition.RolloutPercent;
|
||||
var killed=runtime?.KillSwitch ?? false;
|
||||
// Regular users only receive actively enrolled assignments. Draft/paused/rolled-back definitions stay internal.
|
||||
if(killed||state!="running"||rollout<=0)continue;
|
||||
// Rollout percentage remains authoritative even for previously assigned users.
|
||||
// Assignment is preserved for reproducibility and becomes active again if rollout is raised later.
|
||||
if(!KbxExperimentAssignmentPolicy.IsEnrolled(tenantId,userId,definition.Id,rollout))continue;
|
||||
var existing=await connection.QuerySingleOrDefaultAsync<string>(new CommandDefinition("""
|
||||
select variant from kbx.experiment_assignments where tenant_id=@TenantId and experiment_id=@ExperimentId and user_id=@UserId;
|
||||
""",new{TenantId=tenantId,ExperimentId=definition.Id,UserId=userId},cancellationToken:ct));
|
||||
if(existing is not null){items.Add(new(definition.Id,definition.FlagId,screenId,true,existing,state));continue;}
|
||||
var variant=KbxExperimentAssignmentPolicy.ChooseVariant(tenantId,userId,definition);
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.experiment_assignments(tenant_id,experiment_id,user_id,variant)
|
||||
values(@TenantId,@ExperimentId,@UserId,@Variant) on conflict do nothing;
|
||||
""",new{TenantId=tenantId,ExperimentId=definition.Id,UserId=userId,Variant=variant},cancellationToken:ct));
|
||||
existing=await connection.QuerySingleAsync<string>(new CommandDefinition("""
|
||||
select variant from kbx.experiment_assignments where tenant_id=@TenantId and experiment_id=@ExperimentId and user_id=@UserId;
|
||||
""",new{TenantId=tenantId,ExperimentId=definition.Id,UserId=userId},cancellationToken:ct));
|
||||
items.Add(new(definition.Id,definition.FlagId,screenId,true,existing,state));
|
||||
}
|
||||
return new(items);
|
||||
}
|
||||
private sealed record RuntimeRow(string State,int RolloutPercent,bool KillSwitch);
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace Kbx.Shared.Experiments;
|
||||
public sealed record KbxExperimentAssignmentDto(string ExperimentId,string FlagId,string ScreenId,bool Enrolled,string Variant,string State);
|
||||
public sealed record KbxExperimentAssignmentsResponse(IReadOnlyList<KbxExperimentAssignmentDto> Assignments);
|
||||
public sealed record KbxExperimentVariantMetric(string Variant,long Exposures,long TaskCompletions,double? InteractionsPerTask,double? TaskCompletionP95Ms,double? ValidationFailureRate);
|
||||
public sealed record KbxExperimentOverviewRow(string ExperimentId,string ScreenId,string State,int RolloutPercent,bool KillSwitch,string Decision,IReadOnlyList<KbxExperimentVariantMetric> Variants,DateTimeOffset? UpdatedAt);
|
||||
public sealed record KbxExperimentOverviewResponse(IReadOnlyList<KbxExperimentOverviewRow> Items);
|
||||
public sealed record KbxExperimentRolloutRequest(int RolloutPercent,string State,string Reason);
|
||||
public sealed record KbxExperimentRollbackRequest(string Reason);
|
||||
+11
@@ -0,0 +1,11 @@
|
||||
namespace Kbx.Shared.Experiments;
|
||||
// Schedule hourly at most. This is an operational safety net, not a statistical significance engine.
|
||||
public sealed class KbxExperimentGuardrailJob(KbxExperimentOverviewQuery query,KbxExperimentRuntimeStore store)
|
||||
{
|
||||
public async Task ExecuteAsync(Guid tenantId,CancellationToken ct)
|
||||
{
|
||||
var overview=await query.ExecuteAsync(tenantId,ct);
|
||||
foreach(var item in overview.Items.Where(x=>x.State=="running"&&x.Decision=="guardrail-breach"))
|
||||
await store.RollbackAsync(tenantId,Guid.Empty,item.ExperimentId,"automatic guardrail rollback",null,ct);
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
using Dapper;
|
||||
using Kbx.Shared.Experiments.Generated;
|
||||
using Npgsql;
|
||||
namespace Kbx.Shared.Experiments;
|
||||
public sealed class KbxExperimentOverviewQuery(NpgsqlDataSource dataSource)
|
||||
{
|
||||
public async Task<KbxExperimentOverviewResponse> ExecuteAsync(Guid tenantId,CancellationToken ct)
|
||||
{
|
||||
await using var connection=await dataSource.OpenConnectionAsync(ct);var items=new List<KbxExperimentOverviewRow>();
|
||||
foreach(var def in KbxExperimentCatalog.Experiments.Values)
|
||||
{
|
||||
var runtime=await connection.QuerySingleOrDefaultAsync<RuntimeRow>(new CommandDefinition("select state State,rollout_percent RolloutPercent,kill_switch KillSwitch,updated_at UpdatedAt from kbx.experiment_runtime where tenant_id=@TenantId and experiment_id=@Id",new{TenantId=tenantId,Id=def.Id},cancellationToken:ct));
|
||||
var since=runtime?.UpdatedAt ?? DateTimeOffset.UtcNow.AddDays(-30);
|
||||
var metrics=(await connection.QueryAsync<MetricRow>(new CommandDefinition("""
|
||||
select experiment_variant Variant,
|
||||
count(*) filter(where event_name='experiment.exposed') Exposures,
|
||||
count(*) filter(where event_name='task.complete') TaskCompletions,
|
||||
count(*) filter(where event_name='interaction.execute') Interactions,
|
||||
count(*) filter(where event_name='command.execute') Commands,
|
||||
count(*) filter(where event_name='validation.failed') ValidationFailures,
|
||||
percentile_cont(0.95) within group(order by duration_ms) filter(where event_name='task.complete' and duration_ms is not null) TaskP95
|
||||
from kbx.ux_events where tenant_id=@TenantId and experiment_id=@Id and occurred_at>=@Since
|
||||
group by experiment_variant;
|
||||
""",new{TenantId=tenantId,Id=def.Id,Since=since},cancellationToken:ct))).Select(x=>new KbxExperimentVariantMetric(x.Variant,x.Exposures,x.TaskCompletions,x.TaskCompletions==0?null:Math.Round((double)x.Interactions/x.TaskCompletions,2),x.TaskP95,x.Commands==0?null:Math.Round((double)x.ValidationFailures*100d/x.Commands,2))).ToArray();
|
||||
var state=runtime?.State??def.State;var decision=Decide(def,state,runtime?.KillSwitch??false,metrics);
|
||||
items.Add(new(def.Id,def.ScreenId,state,runtime?.RolloutPercent??def.RolloutPercent,runtime?.KillSwitch??false,decision,metrics,runtime?.UpdatedAt));
|
||||
}
|
||||
return new(items);
|
||||
}
|
||||
private static string Decide(KbxExperimentDefinition def,string state,bool killed,IReadOnlyList<KbxExperimentVariantMetric> rows)
|
||||
{
|
||||
if(killed||state=="rolled-back")return "rolled-back";var control=rows.FirstOrDefault(x=>x.Variant=="control"),treatment=rows.FirstOrDefault(x=>x.Variant!="control");
|
||||
if(control is null||treatment is null||control.Exposures<def.MinExposurePerVariant||treatment.Exposures<def.MinExposurePerVariant)return "inconclusive";
|
||||
if(control.TaskCompletionP95Ms is >0 && treatment.TaskCompletionP95Ms>control.TaskCompletionP95Ms*1.20)return "guardrail-breach";
|
||||
if(control.ValidationFailureRate is not null&&treatment.ValidationFailureRate>control.ValidationFailureRate+2)return "guardrail-breach";
|
||||
if(control.InteractionsPerTask is >0&&treatment.InteractionsPerTask<=control.InteractionsPerTask*(1-def.MinimumImprovementPercent/100d))return "candidate";
|
||||
return "neutral";
|
||||
}
|
||||
private sealed record RuntimeRow(string State,int RolloutPercent,bool KillSwitch,DateTimeOffset UpdatedAt);
|
||||
private sealed record MetricRow(string Variant,long Exposures,long TaskCompletions,long Interactions,long Commands,long ValidationFailures,double? TaskP95);
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
using Kbx.Shared.Experiments.Generated;
|
||||
using Kbx.Shared.Runtime;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using Microsoft.AspNetCore.SignalR;
|
||||
namespace Kbx.Shared.Experiments;
|
||||
public sealed record KbxExperimentChangedEvent(string ExperimentId,string ScreenId,DateTimeOffset ChangedAt);
|
||||
[Authorize]
|
||||
public sealed class KbxExperimentHub : Hub
|
||||
{
|
||||
public override async Task OnConnectedAsync()
|
||||
{
|
||||
var tenantId=RuntimeIdentity.TenantId(Context.User!);
|
||||
if(tenantId!=Guid.Empty)await Groups.AddToGroupAsync(Context.ConnectionId,$"experiment-tenant:{tenantId:N}");
|
||||
await base.OnConnectedAsync();
|
||||
}
|
||||
}
|
||||
public sealed class KbxExperimentRolloutPublisher(IHubContext<KbxExperimentHub> hub)
|
||||
{
|
||||
public Task PublishAsync(Guid tenantId,string experimentId,CancellationToken ct)
|
||||
{
|
||||
var screenId=KbxExperimentCatalog.Experiments.TryGetValue(experimentId,out var definition)?definition.ScreenId:string.Empty;
|
||||
return hub.Clients.Group($"experiment-tenant:{tenantId:N}").SendAsync("ExperimentChanged",new KbxExperimentChangedEvent(experimentId,screenId,DateTimeOffset.UtcNow),ct);
|
||||
}
|
||||
}
|
||||
public static class KbxExperimentRegistration
|
||||
{
|
||||
public static IServiceCollection AddKbxExperiments(this IServiceCollection services)
|
||||
{
|
||||
services.AddSignalR();services.AddSingleton<KbxExperimentAssignmentService>();services.AddSingleton<KbxExperimentOverviewQuery>();services.AddSingleton<KbxExperimentRuntimeStore>();services.AddSingleton<KbxExperimentRolloutPublisher>();services.AddTransient<KbxExperimentGuardrailJob>();return services;
|
||||
}
|
||||
public static IEndpointRouteBuilder MapKbxExperiments(this IEndpointRouteBuilder endpoints){endpoints.MapHub<KbxExperimentHub>("/hubs/kbx-experiments");return endpoints;}
|
||||
}
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
using System.Text.Json;
|
||||
using Dapper;
|
||||
using Kbx.Shared.Experiments.Generated;
|
||||
using Npgsql;
|
||||
namespace Kbx.Shared.Experiments;
|
||||
public sealed class KbxExperimentRuntimeStore(NpgsqlDataSource dataSource,KbxExperimentRolloutPublisher publisher)
|
||||
{
|
||||
public async Task UpdateRolloutAsync(Guid tenantId,Guid actorId,string experimentId,int percent,string state,string reason,string? correlationId,CancellationToken ct)
|
||||
{
|
||||
RequireKnownExperiment(experimentId);if(percent is <0 or >100)throw new ArgumentOutOfRangeException(nameof(percent));if(state is not ("running" or "paused"))throw new ArgumentException("state must be running or paused");if(string.IsNullOrWhiteSpace(reason))throw new ArgumentException("reason is required");
|
||||
await ChangeAsync(tenantId,actorId,experimentId,"rollout",reason,correlationId,ct,async(connection,tx,before)=>{
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.experiment_runtime(tenant_id,experiment_id,state,rollout_percent,kill_switch,updated_by)
|
||||
values(@TenantId,@ExperimentId,@State,@Percent,false,@Actor)
|
||||
on conflict(tenant_id,experiment_id) do update set state=excluded.state,rollout_percent=excluded.rollout_percent,kill_switch=false,version=kbx.experiment_runtime.version+1,updated_by=excluded.updated_by,updated_at=now();
|
||||
""",new{TenantId=tenantId,ExperimentId=experimentId,State=state,Percent=percent,Actor=actorId},tx,cancellationToken:ct));
|
||||
});
|
||||
await publisher.PublishAsync(tenantId,experimentId,ct);
|
||||
}
|
||||
public async Task RollbackAsync(Guid tenantId,Guid actorId,string experimentId,string reason,string? correlationId,CancellationToken ct)
|
||||
{
|
||||
RequireKnownExperiment(experimentId);if(string.IsNullOrWhiteSpace(reason))throw new ArgumentException("reason is required");
|
||||
await ChangeAsync(tenantId,actorId,experimentId,"rollback",reason,correlationId,ct,async(connection,tx,before)=>{
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.experiment_runtime(tenant_id,experiment_id,state,rollout_percent,kill_switch,updated_by)
|
||||
values(@TenantId,@ExperimentId,'rolled-back',0,true,@Actor)
|
||||
on conflict(tenant_id,experiment_id) do update set state='rolled-back',rollout_percent=0,kill_switch=true,version=kbx.experiment_runtime.version+1,updated_by=excluded.updated_by,updated_at=now();
|
||||
""",new{TenantId=tenantId,ExperimentId=experimentId,Actor=actorId},tx,cancellationToken:ct));
|
||||
});
|
||||
await publisher.PublishAsync(tenantId,experimentId,ct);
|
||||
}
|
||||
private static void RequireKnownExperiment(string experimentId)
|
||||
{
|
||||
if(!KbxExperimentCatalog.Experiments.ContainsKey(experimentId))throw new ArgumentException($"Unknown experiment: {experimentId}");
|
||||
}
|
||||
private async Task ChangeAsync(Guid tenantId,Guid actorId,string experimentId,string action,string reason,string? correlationId,CancellationToken ct,Func<NpgsqlConnection,NpgsqlTransaction,string?,Task> mutate)
|
||||
{
|
||||
await using var connection=await dataSource.OpenConnectionAsync(ct);await using var tx=await connection.BeginTransactionAsync(ct);
|
||||
var before=await connection.QuerySingleOrDefaultAsync<string>(new CommandDefinition("select row_to_json(x)::text from (select state,rollout_percent,kill_switch,version from kbx.experiment_runtime where tenant_id=@TenantId and experiment_id=@ExperimentId for update) x",new{TenantId=tenantId,ExperimentId=experimentId},tx,cancellationToken:ct));
|
||||
await mutate(connection,tx,before);
|
||||
var after=await connection.QuerySingleOrDefaultAsync<string>(new CommandDefinition("select row_to_json(x)::text from (select state,rollout_percent,kill_switch,version from kbx.experiment_runtime where tenant_id=@TenantId and experiment_id=@ExperimentId) x",new{TenantId=tenantId,ExperimentId=experimentId},tx,cancellationToken:ct));
|
||||
await connection.ExecuteAsync(new CommandDefinition("""
|
||||
insert into kbx.experiment_audit(id,tenant_id,experiment_id,action,before_state,after_state,actor_id,reason,correlation_id)
|
||||
values(@Id,@TenantId,@ExperimentId,@Action,cast(@Before as jsonb),cast(@After as jsonb),@Actor,@Reason,@CorrelationId);
|
||||
""",new{Id=Guid.NewGuid(),TenantId=tenantId,ExperimentId=experimentId,Action=action,Before=before,After=after,Actor=actorId,Reason=reason,CorrelationId=correlationId},tx,cancellationToken:ct));
|
||||
await tx.CommitAsync(ct);
|
||||
}
|
||||
}
|
||||
+3
@@ -0,0 +1,3 @@
|
||||
using Kbx.Shared.Experiments.Generated;using Xunit;
|
||||
namespace Kbx.Shared.Experiments.Tests;
|
||||
public sealed class KbxExperimentContractTests{[Fact]public void ReferenceExperiment_IsDisabledByDefault(){var x=KbxExperimentCatalog.Experiments["exp.oms.order-list.exception-summary-v2"];Assert.Equal("draft",x.State);Assert.Equal(0,x.RolloutPercent);Assert.Equal(100,x.Variants.Sum(v=>v.Weight));Assert.Contains(x.Variants,v=>v.Key=="control");}}
|
||||
Reference in New Issue
Block a user