+3 -1

Iceshrimp.Backend/Core/Extensions/ApplicationBuilderExtensions.cs

reviewed

··· 1 1 using Iceshrimp.Backend.Core.Configuration; 2 2 + using Iceshrimp.Backend.Core.Helpers; 2 3 using Iceshrimp.Backend.Core.Services; 3 4 using Npgsql; 4 5 using OpenTelemetry.Logs; ··· 17 18 18 19 builder.Services.AddOpenTelemetry() 19 20 .WithTracing(tracing => tracing 20 20 - .AddSource(TraceService.Source, "Iceshrimp.NET/LdSignature", "Iceshrimp.NET/LdHelpers", "Iceshrimp.NET/HttpSignature") 21 21 + .AddSource(Telemetry.Source) 21 22 .AddAspNetCoreInstrumentation(opts => 22 23 { 23 24 opts.RecordException = true; ··· 33 34 .AddNpgsql() 34 35 .AddOtlpExporter()) 35 36 .WithMetrics(metrics => metrics 37 37 + .AddMeter(Telemetry.Source) 36 38 .AddAspNetCoreInstrumentation() 37 39 .AddHttpClientInstrumentation() 38 40 .AddNpgsqlInstrumentation()

+4 -8

Iceshrimp.Backend/Core/Federation/ActivityStreams/LdHelpers.cs

reviewed

··· 1 1 using System.Collections.Concurrent; 2 2 - using System.Diagnostics; 3 2 using Iceshrimp.Backend.Core.Configuration; 4 3 using Iceshrimp.Backend.Core.Database.Tables; 5 4 using Iceshrimp.Backend.Core.Federation.ActivityStreams.Types; ··· 15 14 16 15 public static class LdHelpers 17 16 { 18 18 - private static readonly ActivitySource ActivitySource = new("Iceshrimp.NET/LdHelpers", VersionHelpers.VersionInfo.Value.Version); 19 19 - 20 17 private static readonly Dictionary<string, RemoteDocument> PreloadedContexts = new() 21 18 { 22 19 ["https://purl.archive.org/socialweb/webfinger"] = GetPreloadedContext("wf.json"), ··· 72 69 73 70 private static RemoteDocument CustomLoader(Uri uri, JsonLdLoaderOptions jsonLdLoaderOptions) 74 71 { 75 75 - // ReSharper disable once ExplicitCallerInfoArgument 76 76 - using var activity = ActivitySource.StartActivity("Load Context"); 72 72 + using var activity = Telemetry.ActivitySource.StartActivity("Load Context"); 77 73 activity?.AddTag("uri", uri); 78 74 79 75 var key = uri.AbsolutePath == "/schemas/litepub-0.1.jsonld" ? "litepub-0.1" : uri.ToString(); ··· 144 140 145 141 public static JObject Compact(JToken? json) 146 142 { 147 147 - using var _ = ActivitySource.StartActivity(); 143 143 + using var _ = Telemetry.ActivitySource.StartActivity(); 148 144 return JsonLdProcessor.Compact(json, FederationContext, Options); 149 145 } 150 146 151 147 public static JArray Expand(JToken? json) 152 148 { 153 153 - using var _ = ActivitySource.StartActivity(); 149 149 + using var _ = Telemetry.ActivitySource.StartActivity(); 154 150 return JsonLdProcessor.Expand(json, Options); 155 151 } 156 152 ··· 163 159 { 164 160 return JsonLdProcessor.Canonicalize([json]); 165 161 } 166 166 - } 162 162 + }

+6 -5

Iceshrimp.Backend/Core/Federation/Cryptography/HttpSignature.cs

reviewed

··· 4 4 using System.Security.Cryptography; 5 5 using System.Text; 6 6 using Iceshrimp.Backend.Core.Extensions; 7 7 + using Iceshrimp.Backend.Core.Helpers; 7 8 using Iceshrimp.Backend.Core.Middleware; 8 9 using Iceshrimp.Shared.Helpers; 9 10 using Microsoft.Extensions.Primitives; ··· 12 13 13 14 public static class HttpSignature 14 15 { 15 15 - private static readonly TimeSpan MaxClockSkew = TimeSpan.FromMinutes(5); 16 16 - private static readonly ActivitySource ActivitySource = new("Iceshrimp.NET/HttpSignature", VersionHelpers.VersionInfo.Value.Version); 16 16 + public static readonly TimeSpan MaxClockSkew = TimeSpan.FromMinutes(5); 17 17 18 18 public static async Task<bool> VerifyAsync( 19 19 HttpRequest request, HttpSignatureHeader signature, ··· 24 24 throw new GracefulException(HttpStatusCode.Forbidden, "Request is missing required headers"); 25 25 26 26 // verify signature with "correct" query string behavior 27 27 + 27 28 // ReSharper disable once ExplicitCallerInfoArgument 28 28 - using var activity = ActivitySource.StartActivity("HTTP Signature Verify"); 29 29 + using var activity = Telemetry.ActivitySource.StartActivity("HTTP Signature Verify"); 29 30 activity?.AddTag("key", signature.KeyId); 30 31 31 32 var signingString = ··· 110 111 ) 111 112 { 112 113 // ReSharper disable once ExplicitCallerInfoArgument 113 113 - using var activity = ActivitySource.StartActivity("HTTP Sign"); 114 114 + using var activity = Telemetry.ActivitySource.StartActivity("HTTP Sign"); 114 115 activity?.AddTag("key", keyId); 115 116 116 117 ArgumentNullException.ThrowIfNull(request.RequestUri); ··· 213 214 public readonly string? Opaque = opaque; 214 215 public readonly byte[] Signature = signature; 215 216 } 216 216 - } 217 217 + }

+3 -5

Iceshrimp.Backend/Core/Federation/Cryptography/LdSignature.cs

reviewed

··· 18 18 19 19 public static class LdSignature 20 20 { 21 21 - private static readonly ActivitySource ActivitySource = new("Iceshrimp.NET/LdSignature", VersionHelpers.VersionInfo.Value.Version); 22 22 - 23 21 public static Task<bool> VerifyAsync(JArray activity, JArray rawActivity, string key, string? keyId = null) 24 22 { 25 23 if (activity.ToArray() is not [JObject obj]) ··· 32 30 public static async Task<bool> VerifyAsync(JObject activity, JObject rawActivity, string key, string? keyId = null) 33 31 { 34 32 // ReSharper disable once ExplicitCallerInfoArgument 35 35 - using var traceActivity = ActivitySource.StartActivity("LD Signature Verify"); 33 33 + using var traceActivity = Telemetry.ActivitySource.StartActivity("LD Signature Verify"); 36 34 traceActivity?.AddTag("key", keyId); 37 35 38 36 var options = activity[$"{Constants.W3IdSecurityNs}#signature"]; ··· 64 62 public static async Task<JObject> SignAsync(JObject activity, string key, string? creator) 65 63 { 66 64 // ReSharper disable once ExplicitCallerInfoArgument 67 67 - using var _ = ActivitySource.StartActivity("LD Sign"); 65 65 + using var _ = Telemetry.ActivitySource.StartActivity("LD Sign"); 68 66 69 67 var options = new SignatureOptions 70 68 { ··· 142 140 [JC(typeof(VC))] 143 141 public DateTime? Created { get; set; } 144 142 } 145 145 - } 143 143 + }

+25

Iceshrimp.Backend/Core/Helpers/Telemetry.cs

reviewed

··· 1 1 + using System.Diagnostics; 2 2 + using System.Diagnostics.Metrics; 3 3 + using Iceshrimp.Shared.Helpers; 4 4 + 5 5 + namespace Iceshrimp.Backend.Core.Helpers; 6 6 + 7 7 + /// <summary> 8 8 + /// https://opentelemetry.io/docs/languages/dotnet/instrumentation/#setting-up-an-activitysource 9 9 + /// 10 10 + /// It is recommended to use a custom type to hold references for ActivitySource. 11 11 + /// This avoids possible type collisions with other components in the DI container. 12 12 + /// </summary> 13 13 + public static class Telemetry 14 14 + { 15 15 + public const string Source = "Iceshrimp.NET"; 16 16 + 17 17 + public static ActivitySource ActivitySource { get; } = new(Source, VersionHelpers.VersionInfo.Value.Version); 18 18 + public static Meter Meter { get; } = new(Source, VersionHelpers.VersionInfo.Value.Version); 19 19 + 20 20 + public static void Dispose() 21 21 + { 22 22 + ActivitySource.Dispose(); 23 23 + Meter.Dispose(); 24 24 + } 25 25 + }

+30 -12

Iceshrimp.Backend/Core/Services/QueueService.cs

reviewed

··· 1 1 using System.Diagnostics; 2 2 using System.Diagnostics.CodeAnalysis; 3 3 + using System.Diagnostics.Metrics; 3 4 using System.Text.Json; 4 5 using Iceshrimp.Backend.Core.Configuration; 5 6 using Iceshrimp.Backend.Core.Database; ··· 33 34 public readonly BackfillUserQueue BackfillUserQueue = new(queueConcurrency.Value.BackfillUser); 34 35 35 36 public IEnumerable<string> QueueNames => _queues.Select(p => p.Name); 37 37 + 38 38 + protected internal static readonly Counter<long> DelayedJobCounter = 39 39 + Telemetry.Meter.CreateCounter<long>("queue.jobs.delayed", "jobs", "number of jobs ever delayed (includes the same job if delayed multiple times)"); 40 40 + 41 41 + protected internal static readonly Counter<long> FinishedJobCounter = 42 42 + Telemetry.Meter.CreateCounter<long>("queue.jobs.finished", "jobs", "number of jobs ever finished"); 36 43 37 44 protected override async Task ExecuteAsync(CancellationToken token) 38 45 { ··· 209 216 private readonly string _exceptionPrefix = 210 217 $" at {typeof(PostgresJobQueue<T>).Namespace}.{typeof(PostgresJobQueue<T>).Name}"; 211 218 219 219 + private readonly Gauge<long> _queuedJobGauge = 220 220 + Telemetry.Meter.CreateGauge<long>($"queue.jobs.queued.{name.Replace('-', '_')}", "jobs", "number of jobs currently queued"); 221 221 + 222 222 + private readonly Gauge<long> _activeJobGauge = 223 223 + Telemetry.Meter.CreateGauge<long>($"queue.jobs.active.{name.Replace('-', '_')}", "jobs", "number of jobs currently active"); 224 224 + 212 225 /* 213 226 * This is the main queue processor loop. The algorithm aims to only ever have as many workers running as needed, 214 227 * conserving resources. ··· 271 284 await using var scope = GetScope(); 272 285 await using var db = GetDbContext(scope); 273 286 274 274 - var queuedCount = await db.GetJobQueuedCountAsync(name, token); 287 287 + var queuedCount = await db.GetJobQueuedCountAsync(name, token); 288 288 + _queuedJobGauge.Record(queuedCount); 289 289 + _activeJobGauge.Record(_semaphore.ActiveCount); 290 290 + 275 291 var actualParallelism = Math.Min(_semaphore.CurrentCount, queuedCount); 276 276 - 277 292 if (actualParallelism == 0) 278 293 { 279 294 // Not doing this causes a TOC/TOU race condition, even if it'd likely only be a couple CPU cycles wide. ··· 432 447 433 448 private async Task ProcessJobAsync(IServiceScope processorScope, IServiceScope jobScope, CancellationToken token) 434 449 { 435 435 - var activitySource = processorScope.ServiceProvider.GetRequiredService<TraceService>().ActivitySource; 436 436 - using var activity = activitySource.StartActivity($"process {name}", ActivityKind.Consumer); 450 450 + using var activity = Telemetry.ActivitySource.StartActivity($"process {name}", ActivityKind.Consumer); 437 451 438 452 await using var db = GetDbContext(processorScope); 439 453 ··· 463 477 await db.SaveChangesAsync(token); 464 478 activity?.SetStatus(ActivityStatusCode.Error); 465 479 activity?.AddEvent(new ActivityEvent(job.ExceptionMessage)); 480 480 + QueueService.FinishedJobCounter.Add(1, new("messaging.destination.name", name), new("error", true)); 466 481 return; 467 482 } 468 483 ··· 479 494 job.StackTrace = e.StackTrace; 480 495 job.Exception = e.ToString(); 481 496 482 482 - activity?.SetStatus(ActivityStatusCode.Error) 483 483 - .AddException(e); 497 497 + activity?.AddException(e); 484 498 485 499 var queueName = data is BackgroundTaskJobData ? name + $" ({data.GetType().Name})" : name; 486 500 if (e is GracefulException { Details: not null } ce) ··· 504 518 if (job.Status is Job.JobStatus.Failed) 505 519 { 506 520 job.FinishedAt = DateTime.UtcNow; 521 521 + 522 522 + activity?.SetStatus(ActivityStatusCode.Error); 523 523 + QueueService.FinishedJobCounter.Add(1, new("messaging.destination.name", name), new("error", true)); 507 524 } 508 525 else if (job.Status is Job.JobStatus.Delayed) 509 526 { ··· 519 536 _logger.LogTrace("Job in queue {queue} was delayed to {newTime} after {duration} ms, has been queued since {origTime}", 520 537 name, job.DelayedUntil.Value.ToLocalTime().ToStringIso8601Like(), job.Duration, 521 538 job.QueuedAt.ToLocalTime().ToStringIso8601Like()); 539 539 + QueueService.DelayedJobCounter.Add(1, [new("messaging.destination.name", name)]); 522 540 db.ChangeTracker.Clear(); 523 541 db.Update(job); 524 542 await db.SaveChangesAsync(token); ··· 530 548 { 531 549 job.Status = Job.JobStatus.Completed; 532 550 job.FinishedAt = DateTime.UtcNow; 551 551 + 552 552 + QueueService.FinishedJobCounter.Add(1, new("messaging.destination.name", name), new("error", false)); 533 553 534 554 if (job.QueueDuration < 10_000) 535 555 { ··· 550 570 551 571 public async Task EnqueueAsync(T jobData, string? mutex = null) 552 572 { 573 573 + using var activity = Telemetry.ActivitySource.StartActivity($"schedule {name}", ActivityKind.Producer); 574 574 + 553 575 await using var scope = GetScope(); 554 576 await using var db = GetDbContext(scope); 555 555 - 556 556 - var activitySource = scope.ServiceProvider.GetRequiredService<TraceService>().ActivitySource; 557 557 - using var activity = activitySource.StartActivity($"schedule {name}", ActivityKind.Producer); 558 577 559 578 var id = Ulid.NewUlid().ToGuid(); 560 579 ··· 580 599 581 600 public async Task ScheduleAsync(T jobData, DateTime triggerAt, string? mutex = null) 582 601 { 602 602 + using var activity = Telemetry.ActivitySource.StartActivity($"schedule {name}", ActivityKind.Producer); 603 603 + 583 604 await using var scope = GetScope(); 584 605 await using var db = GetDbContext(scope); 585 585 - 586 586 - var activitySource = scope.ServiceProvider.GetRequiredService<TraceService>().ActivitySource; 587 587 - using var activity = activitySource.StartActivity($"schedule {name}", ActivityKind.Producer); 588 606 589 607 var id = Ulid.NewUlid().ToGuid(); 590 608

-23

Iceshrimp.Backend/Core/Services/TraceService.cs

reviewed

··· 1 1 - using System.Diagnostics; 2 2 - using Iceshrimp.Backend.Core.Extensions; 3 3 - using Iceshrimp.Shared.Helpers; 4 4 - 5 5 - namespace Iceshrimp.Backend.Core.Services; 6 6 - 7 7 - /// <summary> 8 8 - /// https://opentelemetry.io/docs/languages/dotnet/instrumentation/#setting-up-an-activitysource 9 9 - /// 10 10 - /// It is recommended to use a custom type to hold references for ActivitySource. 11 11 - /// This avoids possible type collisions with other components in the DI container. 12 12 - /// </summary> 13 13 - public class TraceService : IDisposable, ISingletonService 14 14 - { 15 15 - public const string Source = "Iceshrimp.NET"; 16 16 - 17 17 - public ActivitySource ActivitySource { get; } = new(Source, VersionHelpers.VersionInfo.Value.Version); 18 18 - 19 19 - public void Dispose() 20 20 - { 21 21 - ActivitySource.Dispose(); 22 22 - } 23 23 - }

+12 -3

Iceshrimp.Backend/Startup.cs

reviewed

··· 2 2 using System.Net; 3 3 using Iceshrimp.Backend.Core.Extensions; 4 4 using Iceshrimp.Backend.Core.Helpers; 5 5 + using Iceshrimp.Backend.Core.Services; 5 6 using Iceshrimp.Backend.Pages.Shared; 6 7 using Iceshrimp.Backend.SignalR; 7 8 using Iceshrimp.Backend.SignalR.Authentication; ··· 97 98 98 99 var elapsed = (DateTime.Now - Process.GetCurrentProcess().StartTime).GetTotalMilliseconds(); 99 100 app.Logger.LogInformation("Startup complete after {ms} ms.", elapsed); 100 100 - await app.StartAsync(); 101 101 - app.SetKestrelUnixSocketPermissions(); 102 102 - await app.WaitForShutdownAsync(); 101 101 + 102 102 + try 103 103 + { 104 104 + await app.StartAsync(); 105 105 + app.SetKestrelUnixSocketPermissions(); 106 106 + await app.WaitForShutdownAsync(); 107 107 + } 108 108 + finally 109 109 + { 110 110 + Telemetry.Dispose(); 111 111 + }