From c44274b7c2146caccebe8aaee793136e2ef671c9 Mon Sep 17 00:00:00 2001 From: "Eric J. Smith" Date: Sun, 10 May 2026 22:35:03 -0500 Subject: [PATCH 1/3] Add project sample data generation --- src/Exceptionless.Core/Bootstrapper.cs | 1 + .../GenerateSampleEventsWorkItemHandler.cs | 61 ++++++++++++++++- .../ResetProjectDataWorkItemHandler.cs | 50 ++++++++++++++ .../WorkItems/GenerateSampleEventsWorkItem.cs | 2 + .../WorkItems/ResetProjectDataWorkItem.cs | 7 ++ .../Utility/SampleDataService.cs | 14 ++++ .../app/project/configure-controller.js | 28 ++++++++ .../app/project/configure.tpl.html | 13 ++++ .../components/project/project-service.js | 7 +- .../ClientApp.angular/lang/en-us.json | 4 ++ .../ClientApp.angular/lang/zh-cn.json | 4 ++ .../src/lib/features/projects/api.svelte.ts | 30 +++++++- .../[projectId]/configure/+page.svelte | 39 +++++++++++ .../Controllers/ProjectController.cs | 26 ++++++- .../Controllers/Data/openapi.json | 68 +++++++++++++++++++ .../Controllers/ProjectControllerTests.cs | 63 ++++++++++++++--- tests/http/projects.http | 8 +++ 17 files changed, 409 insertions(+), 16 deletions(-) create mode 100644 src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs create mode 100644 src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index ee1db07f1e..5bd772047f 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -107,6 +107,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); + handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); handlers.Register(s.GetRequiredService); diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs index cb89024b5f..5f311f2bdc 100644 --- a/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/GenerateSampleEventsWorkItemHandler.cs @@ -35,7 +35,12 @@ public GenerateSampleEventsWorkItemHandler( public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) { - return _lockProvider.TryAcquireAsync(nameof(GenerateSampleEventsWorkItemHandler), TimeSpan.FromMinutes(30), cancellationToken); + var generateSampleEventsWorkItem = (GenerateSampleEventsWorkItem)workItem; + string cacheKey = String.IsNullOrEmpty(generateSampleEventsWorkItem.ProjectId) + ? nameof(GenerateSampleEventsWorkItemHandler) + : $"{nameof(GenerateSampleEventsWorkItemHandler)}:{generateSampleEventsWorkItem.ProjectId}"; + + return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(30), cancellationToken); } public override async Task HandleItemAsync(WorkItemContext context) @@ -49,7 +54,14 @@ public override async Task HandleItemAsync(WorkItemContext context) var generator = new RandomEventGenerator(_timeProvider); var utcNow = _timeProvider.GetUtcNow().UtcDateTime; - var minDate = utcNow.AddDays(-daysBack); + int acceptedDaysBack = Math.Min(daysBack, 3); + var minDate = utcNow.AddDays(-acceptedDaysBack); + + if (!String.IsNullOrEmpty(workItem.OrganizationId) || !String.IsNullOrEmpty(workItem.ProjectId)) + { + await GenerateProjectSampleEventsAsync(context, generator, workItem, eventCount, minDate, utcNow); + return; + } var projectResults = await _projectRepository.GetByOrganizationIdAsync(SampleDataService.TEST_ORG_ID); var projectList = projectResults.Documents.ToList(); @@ -69,7 +81,7 @@ public override async Task HandleItemAsync(WorkItemContext context) int eventsPerProject = eventCount / projectList.Count; int remainder = eventCount % projectList.Count; int totalProcessed = 0; - const int batchSize = 50; + const int batchSize = 100; for (int p = 0; p < projectList.Count; p++) { @@ -98,4 +110,47 @@ public override async Task HandleItemAsync(WorkItemContext context) await context.ReportProgressAsync(100, $"Generated {totalProcessed} sample events across {projectList.Count} projects"); Log.LogInformation("Generated {TotalEvents} sample events across {ProjectCount} projects", totalProcessed, projectList.Count); } + + private async Task GenerateProjectSampleEventsAsync(WorkItemContext context, RandomEventGenerator generator, GenerateSampleEventsWorkItem workItem, int eventCount, DateTime minDate, DateTime utcNow) + { + if (String.IsNullOrEmpty(workItem.OrganizationId) || String.IsNullOrEmpty(workItem.ProjectId)) + { + Log.LogWarning("Unable to generate project sample events because organization id or project id was not specified"); + return; + } + + var organization = await _organizationRepository.GetByIdAsync(workItem.OrganizationId); + if (organization is null) + { + Log.LogWarning("Organization {OrganizationId} not found when generating sample events", workItem.OrganizationId); + return; + } + + var project = await _projectRepository.GetByIdAsync(workItem.ProjectId); + if (project is null || project.OrganizationId != organization.Id) + { + Log.LogWarning("Project {ProjectId} not found in organization {OrganizationId} when generating sample events", workItem.ProjectId, workItem.OrganizationId); + return; + } + + int totalProcessed = 0; + const int batchSize = 100; + var events = generator.Generate(organization.Id, project.Id, eventCount, minDate, utcNow); + + for (int i = 0; i < events.Count; i += batchSize) + { + if (context.CancellationToken.IsCancellationRequested) + break; + + var batch = events.Skip(i).Take(batchSize).ToList(); + await _eventPipeline.RunAsync(batch, organization, project); + totalProcessed += batch.Count; + + int percentage = (int)Math.Min(99, totalProcessed * 100.0 / eventCount); + await context.ReportProgressAsync(percentage, $"Processed {totalProcessed}/{eventCount} events"); + } + + await context.ReportProgressAsync(100, $"Generated {totalProcessed} sample events for project {project.Id}"); + Log.LogInformation("Generated {TotalEvents} sample events for project {ProjectId}", totalProcessed, project.Id); + } } diff --git a/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs new file mode 100644 index 0000000000..30ea5fe4ab --- /dev/null +++ b/src/Exceptionless.Core/Jobs/WorkItemHandlers/ResetProjectDataWorkItemHandler.cs @@ -0,0 +1,50 @@ +using Exceptionless.Core.Models.WorkItems; +using Exceptionless.Core.Repositories; +using Foundatio.Caching; +using Foundatio.Jobs; +using Foundatio.Lock; +using Microsoft.Extensions.Logging; + +namespace Exceptionless.Core.Jobs.WorkItemHandlers; + +public class ResetProjectDataWorkItemHandler : WorkItemHandlerBase +{ + private readonly IEventRepository _eventRepository; + private readonly IStackRepository _stackRepository; + private readonly ICacheClient _cacheClient; + private readonly ILockProvider _lockProvider; + + public ResetProjectDataWorkItemHandler(IEventRepository eventRepository, IStackRepository stackRepository, ICacheClient cacheClient, ILockProvider lockProvider, ILoggerFactory loggerFactory) : base(loggerFactory) + { + _eventRepository = eventRepository; + _stackRepository = stackRepository; + _cacheClient = cacheClient; + _lockProvider = lockProvider; + } + + public override Task GetWorkItemLockAsync(object workItem, CancellationToken cancellationToken = default) + { + string cacheKey = $"{nameof(ResetProjectDataWorkItemHandler)}:{((ResetProjectDataWorkItem)workItem).ProjectId}"; + return _lockProvider.TryAcquireAsync(cacheKey, TimeSpan.FromMinutes(15), cancellationToken); + } + + public override async Task HandleItemAsync(WorkItemContext context) + { + var workItem = context.GetData()!; + + using (Log.BeginScope(new ExceptionlessState().Organization(workItem.OrganizationId).Project(workItem.ProjectId))) + { + Log.LogInformation("Received reset project data work item for project: {ProjectId}", workItem.ProjectId); + await context.ReportProgressAsync(0, "Starting project data reset..."); + + long removedEvents = await _eventRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId); + await context.ReportProgressAsync(50, $"Events removed: {removedEvents}"); + + long removedStacks = await _stackRepository.RemoveAllByProjectIdAsync(workItem.OrganizationId, workItem.ProjectId); + await _cacheClient.RemoveByPrefixAsync(String.Concat("stack-filter:", workItem.OrganizationId, ":", workItem.ProjectId)); + + await context.ReportProgressAsync(100, $"Events removed: {removedEvents}, stacks removed: {removedStacks}"); + Log.LogInformation("Reset project data for project {ProjectId}. Events removed: {RemovedEvents}, stacks removed: {RemovedStacks}", workItem.ProjectId, removedEvents, removedStacks); + } + } +} diff --git a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs index 06ec8c4dde..a25dbd94de 100644 --- a/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs +++ b/src/Exceptionless.Core/Models/WorkItems/GenerateSampleEventsWorkItem.cs @@ -2,6 +2,8 @@ namespace Exceptionless.Core.Models.WorkItems; public record GenerateSampleEventsWorkItem { + public string? OrganizationId { get; init; } + public string? ProjectId { get; init; } public int EventCount { get; init; } = 100; public int DaysBack { get; init; } = 7; } diff --git a/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs new file mode 100644 index 0000000000..4d4410c6c0 --- /dev/null +++ b/src/Exceptionless.Core/Models/WorkItems/ResetProjectDataWorkItem.cs @@ -0,0 +1,7 @@ +namespace Exceptionless.Core.Models.WorkItems; + +public record ResetProjectDataWorkItem +{ + public required string OrganizationId { get; init; } + public required string ProjectId { get; init; } +} diff --git a/src/Exceptionless.Core/Utility/SampleDataService.cs b/src/Exceptionless.Core/Utility/SampleDataService.cs index c778b21fe6..897a2e92cc 100644 --- a/src/Exceptionless.Core/Utility/SampleDataService.cs +++ b/src/Exceptionless.Core/Utility/SampleDataService.cs @@ -286,4 +286,18 @@ await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem }); _logger.LogInformation("Enqueued sample event generation: {EventCount} events over {DaysBack} days", eventCount, daysBack); } + + public async Task EnqueueSampleEventsAsync(string organizationId, string projectId, int eventCount = 100, int daysBack = 7) + { + string workItemId = await _workItemQueue.EnqueueAsync(new GenerateSampleEventsWorkItem + { + OrganizationId = organizationId, + ProjectId = projectId, + EventCount = eventCount, + DaysBack = daysBack + }); + + _logger.LogInformation("Enqueued sample event generation for project {ProjectId}: {EventCount} events over {DaysBack} days", projectId, eventCount, daysBack); + return workItemId; + } } diff --git a/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js b/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js index fbc2dde362..b10b60fe1b 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js +++ b/src/Exceptionless.Web/ClientApp.angular/app/project/configure-controller.js @@ -168,6 +168,32 @@ $state.go("app.project-frequent", { projectId: vm._projectId }); } + function generateSampleData() { + if (vm.isGeneratingSampleData) { + return; + } + + function onSuccess() { + notificationService.success( + translateService.T("Sample data generation has been queued. Events will appear shortly.") + ); + } + + function onFailure() { + notificationService.error( + translateService.T("An error occurred while generating sample data for your project.") + ); + } + + vm.isGeneratingSampleData = true; + return projectService + .generateSampleData(vm._projectId) + .then(onSuccess, onFailure) + .finally(function () { + vm.isGeneratingSampleData = false; + }); + } + this.$onInit = function $onInit() { vm._projectId = $stateParams.id; vm._canRedirect = $stateParams.redirect === "true"; @@ -177,9 +203,11 @@ vm.copyCommandLineCode = copyCommandLineCode; vm.copied = copied; vm.currentProjectType = {}; + vm.generateSampleData = generateSampleData; vm.isBashShell = isBashShell; vm.isCommandLine = isCommandLine; vm.isDotNet = isDotNet; + vm.isGeneratingSampleData = false; vm.isJavaScript = isJavaScript; vm.isNode = isNode; vm.navigateToDashboard = navigateToDashboard; diff --git a/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html b/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html index e6c6060819..c9fb2d6373 100644 --- a/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html +++ b/src/Exceptionless.Web/ClientApp.angular/app/project/configure.tpl.html @@ -182,6 +182,19 @@