From c829e8584fdd62abe682dc47d7b769a1d927c4a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AE=97=E6=97=AD?= <1619917346@qq.com> Date: Sun, 10 Aug 2025 23:28:52 +0800 Subject: [PATCH 1/6] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=90=8E=E5=8F=B0?= =?UTF-8?q?=E9=98=9F=E5=88=97+=E6=B6=88=E6=81=AF=E9=98=9F=E5=88=97?= =?UTF-8?q?=E7=9A=84=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../UniversalAdminSystem.Api/appsettings.json | 12 ++- .../Common/interfaces/IEventBus.cs | 6 ++ .../FileStorage/DTOs/FileProcessingJobDto.cs | 3 + .../FileStorage/Factory/DocParserFactory.cs | 7 -- .../FileStorage/Interfaces/IDocParser.cs | 2 +- .../Interfaces/IFileProcessingQueue.cs | 8 ++ .../FileStorage/Services/FileAppService.cs | 31 +++----- .../FileStorage/Aggregates/File.cs | 6 +- .../Configs/RabbitMqConfig.cs | 27 +++++++ .../AddInfrastructureService.cs | 42 +++++++++- .../FileStorage/Parsers/DocParserFactory.cs | 17 ++++ .../{ => Parsers}/MdParseService.cs | 6 +- .../FileStorage/Parsers/PdfParseService.cs | 30 ++++++++ .../FileStorage/Parsers/TxtParseService.cs | 11 +++ .../FileStorage/TxtParseService.cs | 0 .../UniversalAdminSystemDbContext.cs | 13 +++- .../Consumers/FileProcessingJobConsumer.cs | 77 +++++++++++++++++++ .../RabbitMQ/Jobs/FileProcessingJob.cs | 14 ++++ .../Publishers/FileProcessingJobPublisher.cs | 60 +++++++++++++++ .../RabbitMQ/Queues/EfFileProcessingQueue.cs | 40 ++++++++++ .../RabbitMQ/RabbitMqEventBus.cs | 30 ++++++++ ...UniversalAdminSystem.Infrastructure.csproj | 3 + 22 files changed, 400 insertions(+), 45 deletions(-) create mode 100644 backend/src/UniversalAdminSystem.Application/Common/interfaces/IEventBus.cs create mode 100644 backend/src/UniversalAdminSystem.Application/FileStorage/DTOs/FileProcessingJobDto.cs delete mode 100644 backend/src/UniversalAdminSystem.Application/FileStorage/Factory/DocParserFactory.cs create mode 100644 backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileProcessingQueue.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Configs/RabbitMqConfig.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/DocParserFactory.cs rename backend/src/UniversalAdminSystem.Infrastructure/FileStorage/{ => Parsers}/MdParseService.cs (86%) create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/PdfParseService.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/TxtParseService.cs delete mode 100644 backend/src/UniversalAdminSystem.Infrastructure/FileStorage/TxtParseService.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs diff --git a/backend/src/UniversalAdminSystem.Api/appsettings.json b/backend/src/UniversalAdminSystem.Api/appsettings.json index 5a0c1dd..8db1331 100644 --- a/backend/src/UniversalAdminSystem.Api/appsettings.json +++ b/backend/src/UniversalAdminSystem.Api/appsettings.json @@ -8,7 +8,8 @@ }, "AllowedHosts": "*", "ConnectionStrings": { - "pgSql": "server=127.0.0.1;port=5432;uid=postgres;password=031028@yue;database=universal_admin" + "pgSql": "server=127.0.0.1;port=5432;uid=postgres;password=031028@yue;database=universal_admin", + "RabbitMq": "amqp://guest:guest@rabbitmq:5672/" }, "Jwt": { "Key": "YourSuperSecretKey1232347509872093oiqewupori", @@ -19,5 +20,14 @@ "K2": { "BaseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1/", "ApiKey": "sk-a31163f6b59e44dcbdb87c668482ce96" + }, + "RabbitMq": { + "Host": "rabbitmq", + "Port": 5672, + "Username": "guest", + "Password": "guest", + "Exchange": "file-processing", + "Queue": "file-processing-queue", + "RoutingKey": "file-processing" } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/Common/interfaces/IEventBus.cs b/backend/src/UniversalAdminSystem.Application/Common/interfaces/IEventBus.cs new file mode 100644 index 0000000..c8646a9 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Application/Common/interfaces/IEventBus.cs @@ -0,0 +1,6 @@ +namespace UniversalAdminSystem.Application.Common.Interfaces; + +public interface IEventBus +{ + Task PublishAsync(string exchange, string routingKey, string payload, string? messageId = null, CancellationToken ct = default); +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/DTOs/FileProcessingJobDto.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/DTOs/FileProcessingJobDto.cs new file mode 100644 index 0000000..1781585 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/DTOs/FileProcessingJobDto.cs @@ -0,0 +1,3 @@ +namespace UniversalAdminSystem.Application.FileStorage.DTOs; + +public record FileProcessingJobDto(Guid FileId, string FilePath, string ContentType, long Size); \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Factory/DocParserFactory.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Factory/DocParserFactory.cs deleted file mode 100644 index 58bbea3..0000000 --- a/backend/src/UniversalAdminSystem.Application/FileStorage/Factory/DocParserFactory.cs +++ /dev/null @@ -1,7 +0,0 @@ -using UniversalAdminSystem.Domian.FileStorage.Interfaces; - -namespace UniversalAdminSystem.Domian.FileStorage.Factory; - -public static class DocParserFactory -{ -} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IDocParser.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IDocParser.cs index 067113c..dc2f7dd 100644 --- a/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IDocParser.cs +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IDocParser.cs @@ -3,5 +3,5 @@ namespace UniversalAdminSystem.Application.FileStorage.Interfaces; public interface IDocParser { // bool CanParse(); - string Parse(string path); + Task ParseAsync(string path); } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileProcessingQueue.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileProcessingQueue.cs new file mode 100644 index 0000000..1c0b5a6 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileProcessingQueue.cs @@ -0,0 +1,8 @@ +using UniversalAdminSystem.Application.FileStorage.DTOs; + +namespace UniversalAdminSystem.Application.FileStorage.Interfaces; + +public interface IFileProcessingQueue +{ + Task EnqueueAsync(FileProcessingJobDto jd, CancellationToken ct = default); +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs index 78eac96..2fad793 100644 --- a/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs @@ -15,14 +15,16 @@ public class FileAppService : IFileAppService private readonly IFileValidationService _fileValidator; private readonly IFileStorageService _localfileStorage; private readonly IUnitOfWork _unitOfWork; + private readonly IFileProcessingQueue _fileProcessingQueue; - public FileAppService(IFileRepository fileRepository, IFileValidationService fileValidator, IFileStorageService localfileStorage, IUnitOfWork unitOfWork) + public FileAppService(IFileRepository fileRepository, IFileValidationService fileValidator, IFileStorageService localfileStorage, IUnitOfWork unitOfWork, IFileProcessingQueue fileProcessingQueue) { _fileRepository = fileRepository; _fileValidator = fileValidator; _localfileStorage = localfileStorage; _unitOfWork = unitOfWork; + _fileProcessingQueue = fileProcessingQueue; } public async Task UploadAsync(IFormFile file) @@ -56,19 +58,12 @@ public class FileAppService : IFileAppService Console.WriteLine("添加数据到成功！提交事务！"); await _unitOfWork.CommitAsync(); - // 后续文档异步处理 - // ... - _ = Task.Run(async () => - { - try - { - await FileProcessingAsync(); - } - catch (Exception) - { - throw; - } - }); + // 文件处理任务入队 + await _fileProcessingQueue.EnqueueAsync(new FileProcessingJobDto( + fileEntity.Id.Value, + fileInfo.FullName, + file.ContentType, + file.Length), CancellationToken.None); return new FileUploadResultDto ( @@ -88,14 +83,6 @@ public class FileAppService : IFileAppService } } - private async Task FileProcessingAsync() - { - // 文本提取 - // 文本分片 - // 向量化 - // 向量入库 - } - public async Task> GetList() { var files = await _fileRepository.GetAllAsync(); diff --git a/backend/src/UniversalAdminSystem.Domian/FileStorage/Aggregates/File.cs b/backend/src/UniversalAdminSystem.Domian/FileStorage/Aggregates/File.cs index b05ab05..f5e30c3 100644 --- a/backend/src/UniversalAdminSystem.Domian/FileStorage/Aggregates/File.cs +++ b/backend/src/UniversalAdminSystem.Domian/FileStorage/Aggregates/File.cs @@ -1,7 +1,6 @@ using UniversalAdminSystem.Domian.Core; using UniversalAdminSystem.Domian.Core.ValueObjects; using UniversalAdminSystem.Domian.FileStorage.ValueObjects; -using UniversalAdminSystem.Domian.UserManagement.ValueObj; namespace UniversalAdminSystem.Domian.FileStorage.Aggregates; @@ -32,10 +31,7 @@ public class File : AggregateRoot FileAccessLevel accessLevel = FileAccessLevel.Private ) { - // if (!FileType.AllowedTypes.Contains(type.Value)) - // throw new ArgumentException("不支持的文件类型"); - if (size.Value > FileSize.MaxSize) - throw new ArgumentException($"文件大小不能超过{FileSize.MaxSize / 1024 / 1024}MB"); + if (size.Value > FileSize.MaxSize) throw new ArgumentException($"文件大小不能超过{FileSize.MaxSize / 1024 / 1024}MB"); return new File { Id = FileId.Create(), diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Configs/RabbitMqConfig.cs b/backend/src/UniversalAdminSystem.Infrastructure/Configs/RabbitMqConfig.cs new file mode 100644 index 0000000..edd279c --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Configs/RabbitMqConfig.cs @@ -0,0 +1,27 @@ +using System.Text.Json.Serialization; + +namespace UniversalAdminSystem.Infrastructure.Configs; + +public class RabbitMqConfig +{ + [property: JsonPropertyName("Host")] + public string Host { get; set; } = string.Empty; + + [property: JsonPropertyName("Port")] + public int Port { get; set; } + + [property: JsonPropertyName("Username")] + public string Username { get; set; } = string.Empty; + + [property: JsonPropertyName("Password")] + public string Password { get; set; } = string.Empty; + + [property: JsonPropertyName("Exchange")] + public string Exchange { get; set; } = string.Empty; + + [property: JsonPropertyName("Queue")] + public string Queue { get; set; } = string.Empty; + + [property: JsonPropertyName("RoutingKey")] + public string RoutingKey { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs b/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs index 0fdddc8..0d5aba2 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs @@ -23,7 +23,13 @@ using FluentValidation; using UniversalAdminSystem.Domian.UserConversations.IRepository; using UniversalAdminSystem.Domian.knowledge.IRepository; using UniversalAdminSystem.Application.FileStorage.Interfaces; -using UniversalAdminSystem.Infrastructure.Configs; +using UniversalAdminSystem.Application.Common.Interfaces; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Queues; +using UniversalAdminSystem.Infrastructure.RabbitMQ; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Publishers; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Consumers; +using Microsoft.AspNetCore.Connections; +using RabbitMQ.Client; namespace UniversalAdminSystem.Infrastructure.DependencyInject; @@ -110,7 +116,7 @@ public static class AddInfrastrutureService { // services.Configure(configuration.GetSection("AllowedFiles")); // 注册数据库上下文 - services.AddDbContext(x => x.UseNpgsql(configuration.GetConnectionString("pgSql"),x=>x.UseVector())); + services.AddDbContext(x => x.UseNpgsql(configuration.GetConnectionString("pgSql"), x => x.UseVector())); // 注册配置 services.AddAllConfig(configuration); @@ -128,7 +134,7 @@ public static class AddInfrastrutureService typeof(ISystemSettingRepository).Assembly, typeof(IConversationsRepository).Assembly, typeof(IDocumentChunkRepository).Assembly, - + }, new Assembly[] { @@ -172,6 +178,36 @@ public static class AddInfrastrutureService services.AddHostedService(); services.AddSingleton(); + // 注册文件处理相关服务 + services.AddSingleton(); + services.AddSingleton(sp => + { + var factory = new ConnectionFactory + { + Uri = new Uri(sp.GetRequiredService().GetConnectionString("RabbitMq") ?? throw new InvalidOperationException("Missing RabbitMq connection string")), + DispatchConsumersAsync = true + }; + return factory.CreateConnection(); + }); + services.AddTransient(sp => + { + var conn = sp.GetRequiredService(); + var ch = conn.CreateModel(); + ch.ConfirmSelect(); + var exchange = configuration["RabbitMq:Exchange"] ?? "file-processing"; + var queue = configuration["RabbitMq:Queue"] ?? "file-processing-queue"; + var routingKey = configuration["RabbitMq:RoutingKey"] ?? "file-processing"; + + ch.ExchangeDeclare(exchange, "topic", durable: true, autoDelete: false); + ch.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false, arguments: null); + ch.QueueBind(queue, exchange, routingKey); + return ch; + }); + + services.AddSingleton(); + services.AddHostedService(); + services.AddHostedService(); + return services; } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/DocParserFactory.cs b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/DocParserFactory.cs new file mode 100644 index 0000000..e35f464 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/DocParserFactory.cs @@ -0,0 +1,17 @@ +using UniversalAdminSystem.Application.FileStorage.Interfaces; + +namespace UniversalAdminSystem.Infrastructure.FileStorage.Parsers; + +public class DocParserFactory +{ + public IDocParser GetParser(string mimeType) + { + return mimeType.ToLower() switch + { + "application/pdf" => new PdfParseService(), + "text/markdown" => new MdParseService(), + "text/plain" => new TxtParseService(), + _ => throw new NotImplementedException($"没有为 MIME 类型 '{mimeType}' 实现解析器") + }; + } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/MdParseService.cs b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/MdParseService.cs similarity index 86% rename from backend/src/UniversalAdminSystem.Infrastructure/FileStorage/MdParseService.cs rename to backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/MdParseService.cs index a363362..89e9b89 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/MdParseService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/MdParseService.cs @@ -4,17 +4,17 @@ using HtmlAgilityPack; using Markdig; using UniversalAdminSystem.Application.FileStorage.Interfaces; -namespace UniversalAdminSystem.Infrastructure.FileStorage; +namespace UniversalAdminSystem.Infrastructure.FileStorage.Parsers; public sealed partial class MdParseService : IDocParser { private static readonly Regex _blankLines = MyRegex(); - public string Parse(string path) + public async Task ParseAsync(string path) { if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("文件路径不能为空", nameof(path)); using var reader = new StreamReader(path); - var mdContent = reader.ReadToEnd(); + var mdContent = await reader.ReadToEndAsync(); return Convert(mdContent); } diff --git a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/PdfParseService.cs b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/PdfParseService.cs new file mode 100644 index 0000000..8875eff --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/PdfParseService.cs @@ -0,0 +1,30 @@ +using System.Text; +using iText.Kernel.Pdf; +using iText.Kernel.Pdf.Canvas.Parser; +using UniversalAdminSystem.Application.FileStorage.Interfaces; + +namespace UniversalAdminSystem.Infrastructure.FileStorage.Parsers; + +public class PdfParseService : IDocParser +{ + public async Task ParseAsync(string path) + { + return await Task.Run(() => + { + var text = new StringBuilder(); + using (var pdfReader = new PdfReader(path)) + { + using var pdfDocument = new PdfDocument(pdfReader); + int numberOfPages = pdfDocument.GetNumberOfPages(); + for (int i = 1; i <= numberOfPages; i++) + { + var page = pdfDocument.GetPage(i); + var pageText = PdfTextExtractor.GetTextFromPage(page); + text.Append(pageText); + } + + } + return text.ToString(); + }); + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/TxtParseService.cs b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/TxtParseService.cs new file mode 100644 index 0000000..d240da3 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/Parsers/TxtParseService.cs @@ -0,0 +1,11 @@ +using UniversalAdminSystem.Application.FileStorage.Interfaces; + +namespace UniversalAdminSystem.Infrastructure.FileStorage.Parsers; + +public class TxtParseService : IDocParser +{ + public async Task ParseAsync(string path) + { + return await File.ReadAllTextAsync(path); + } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/TxtParseService.cs b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/TxtParseService.cs deleted file mode 100644 index e69de29..0000000 diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs index 0173534..e6e001b 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs @@ -16,6 +16,7 @@ using UniversalAdminSystem.Domian.knowledge.ValueObj; using NpgsqlTypes; using Pgvector; using UniversalAdminSystem.Domian.UserConversations.Aggregates; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; namespace UniversalAdminSystem.Infrastructure.Persistence.DbContexts; @@ -30,17 +31,16 @@ public class UniversalAdminSystemDbContext : DbContext public DbSet LogEntries { get; set; } public DbSet SystemSettings { get; set; } public DbSet Files { get; set; } - public DbSet Conversations { get; set; } - public DbSet Messages { get; set; } - public DbSet Chunks { get; set; } + public DbSet FileProcessingJobs { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { // 启用"vector"扩展 modelBuilder.HasPostgresExtension("vector"); + // 配置UserInfo实体 modelBuilder.Entity(entity => { @@ -289,6 +289,13 @@ public class UniversalAdminSystemDbContext : DbContext ); }); + modelBuilder.Entity(entity => + { + entity.HasKey(fpje => fpje.Id); + entity.HasIndex(fpje => new { fpje.Status, fpje.NextAttemptAt, fpje.CreatedAt }); + entity.Property(fpje => fpje.Status).HasMaxLength(32); + }); + // 忽略值对象 modelBuilder.Ignore(); } diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs new file mode 100644 index 0000000..e47d80a --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs @@ -0,0 +1,77 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using RabbitMQ.Client; +using RabbitMQ.Client.Events; +using UniversalAdminSystem.Infrastructure.FileStorage.Parsers; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; + +namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Consumers; + +public class FileProcessingJobConsumer : BackgroundService +{ + private readonly IModel _ch; + private readonly ILogger _logger; + + public FileProcessingJobConsumer(IModel ch, ILogger logger) + { + _ch = ch; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + try + { + var consumer = new EventingBasicConsumer(_ch); + consumer.Received += async (sender, e) => + { + var body = e.Body.ToArray(); + var message = Encoding.UTF8.GetString(body); + _logger.LogInformation("收到文件处理任务: {Message}", message); + try + { + await ProcessMessageAsync(message, stoppingToken); + _ch.BasicAck(e.DeliveryTag, false); + _logger.LogInformation("文件处理任务处理成功: {Message}", message); + } + catch (Exception ex) + { + _ch.BasicNack(e.DeliveryTag, false, true); + _logger.LogError(ex, "文件处理任务处理失败: {Message}", message); + } + }; + _ch.BasicConsume(queue: "file-processing-queue", autoAck: false, consumer: consumer); + await Task.Delay(Timeout.Infinite, stoppingToken); + } + catch (Exception ex) { _logger.LogError(ex, "文件处理任务消费失败"); } + } + + private async Task ProcessMessageAsync(string message, CancellationToken stoppingToken) + { + var job = JsonSerializer.Deserialize(message); + if (job == null) + { + _logger.LogError("文件处理任务反序列化失败: {Message}", message); + return; + } + // 文本提取 + _logger.LogInformation("开始处理文件: {FilePath}", job.FilePath); + var parser = new DocParserFactory().GetParser(job.ContentType); + var text = await parser.ParseAsync(job.FilePath); + _logger.LogInformation("文件处理完成: {FilePath}", job.FilePath); + + // 文本分片 + _logger.LogInformation("开始分片文件: {FilePath}", job.FilePath); + + // Embedding + _logger.LogInformation("开始Embedding文件: {FilePath}", job.FilePath); + + // Vector Store + _logger.LogInformation("开始Vector Store文件: {FilePath}", job.FilePath); + + // 文件处理完成 + _logger.LogInformation("文件处理完成: {FilePath}", job.FilePath); + } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs new file mode 100644 index 0000000..c41aaf7 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs @@ -0,0 +1,14 @@ +namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; + +public class FileProcessingJob +{ + public Guid Id { get; set; } = Guid.NewGuid(); + public Guid FileId { get; set; } + public string FilePath { get; set; } = default!; + public string ContentType { get; set; } = default!; + public long Size { get; set; } + public string Status { get; set; } = "Pending"; // Pending|Processing|Succeeded|Failed + public int RetryCount { get; set; } = 0; + public DateTime CreatedAt { get; set; } = DateTime.UtcNow; + public DateTime? NextAttemptAt { get; set; } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs new file mode 100644 index 0000000..645c421 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs @@ -0,0 +1,60 @@ +using System.Text.Json; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using UniversalAdminSystem.Application.Common.Interfaces; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; + +namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Publishers; + +public class FileProcessingJobPublisher : BackgroundService +{ + private readonly IServiceScopeFactory _sf; + private readonly IEventBus _bus; + private readonly ILogger _logger; + + public FileProcessingJobPublisher(IServiceScopeFactory sf, IEventBus bus, ILogger logger) + { _sf = sf; _bus = bus; _logger = logger; } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + try + { + using var scope = _sf.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var jobs = await db.FileProcessingJobs + .Where(x => x.Status == "Pending" && (x.NextAttemptAt == null || x.NextAttemptAt <= DateTime.UtcNow)) + .OrderBy(x => x.CreatedAt) + .Take(100) + .ToListAsync(stoppingToken); + + foreach (var j in jobs) j.Status = "Processing"; + await db.SaveChangesAsync(stoppingToken); + + foreach (var j in jobs) + { + try + { + var payload = JsonSerializer.Serialize(new { j.Id, j.FileId, j.FilePath, j.ContentType, j.Size }); + await _bus.PublishAsync("file-processing", "file-processing", payload, messageId: j.Id.ToString(), stoppingToken); + j.Status = "Succeeded"; + } + catch (Exception ex) + { + j.RetryCount += 1; + j.Status = "Pending"; + j.NextAttemptAt = DateTime.UtcNow.AddSeconds(Math.Min(300, Math.Pow(2, j.RetryCount))); + _logger.LogError(ex, "发布失败: {JobId}", j.Id); + } + } + await db.SaveChangesAsync(stoppingToken); + } + catch (Exception ex) { _logger.LogError(ex, "文件处理任务发布循环错误"); } + + await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken); + } + } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs new file mode 100644 index 0000000..2a0bbf0 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs @@ -0,0 +1,40 @@ +using Microsoft.Extensions.Logging; +using UniversalAdminSystem.Application.FileStorage.DTOs; +using UniversalAdminSystem.Application.FileStorage.Interfaces; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; + +namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Queues; + +public class EfFileProcessingQueue : IFileProcessingQueue +{ + private readonly UniversalAdminSystemDbContext _dbContext; + private readonly Logger _logger; + + public EfFileProcessingQueue(UniversalAdminSystemDbContext dbContext, Logger logger) + { + _dbContext = dbContext; + _logger = logger; + } + + public async Task EnqueueAsync(FileProcessingJobDto jd, CancellationToken ct = default) + { + try + { + _dbContext.FileProcessingJobs.Add(new FileProcessingJob + { + FileId = jd.FileId, + FilePath = jd.FilePath, + ContentType = jd.ContentType, + Size = jd.Size + }); + await _dbContext.SaveChangesAsync(ct); + _logger.LogInformation("文件处理任务已入队: {FileId}", jd.FileId); + } + catch (Exception ex) + { + _logger.LogError(ex, "文件处理任务入队失败: {FileId}", jd.FileId); + throw; + } + } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs new file mode 100644 index 0000000..2065305 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs @@ -0,0 +1,30 @@ +using Microsoft.Extensions.Configuration; +using UniversalAdminSystem.Application.Common.Interfaces; +using System.Text; +using RabbitMQ.Client; + +namespace UniversalAdminSystem.Infrastructure.RabbitMQ; + +public class RabbitMqEventBus : IEventBus, IDisposable +{ + private readonly IModel _ch; + + public RabbitMqEventBus(IModel ch, IConfiguration cfg) + { + _ch = ch; + _ch.ConfirmSelect(); + } + + public Task PublishAsync(string exchange, string routingKey, string payload, string? messageId = null, CancellationToken ct = default) + { + var props = _ch.CreateBasicProperties(); + props.Persistent = true; + props.MessageId = messageId; + props.ContentType = "application/json"; + _ch.BasicPublish(exchange, routingKey, mandatory: true, basicProperties: props, body: Encoding.UTF8.GetBytes(payload)); + _ch.WaitForConfirmsOrDie(TimeSpan.FromSeconds(5)); + return Task.CompletedTask; + } + + public void Dispose() { _ch?.Dispose(); } +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj b/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj index 3302db0..9a5269a 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj +++ b/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj @@ -8,10 +8,13 @@ + + + -- Gitee From a6990038c444e1a2c00f9b3438725dd967f05893 Mon Sep 17 00:00:00 2001 From: xiaoyaolanren2 <18033910316@163.com> Date: Tue, 12 Aug 2025 17:05:26 +0800 Subject: [PATCH 2/6] =?UTF-8?q?=E5=AE=8C=E6=88=90=E8=8E=B7=E5=8F=96?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E6=B6=88=E6=81=AFapi=E7=AB=AF=E7=82=B9?= =?UTF-8?q?=EF=BC=8C=E5=9C=A8=E6=95=B0=E6=8D=AE=E5=BA=93=E4=B8=8A=E4=B8=8B?= =?UTF-8?q?=E6=96=87=E4=B8=AD=E4=B8=BA=E5=90=91=E9=87=8F=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E7=B4=A2=E5=BC=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controllers/AIQuestionsController.cs | 16 +- .../20250812085226_last.Designer.cs | 394 ++++++++++++++++++ .../Migrations/20250812085226_last.cs | 27 ++ ...versalAdminSystemDbContextModelSnapshot.cs | 2 + .../UniversalAdminSystemDbContext.cs | 1 + 5 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.Designer.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.cs diff --git a/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs b/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs index 913abe9..107d2ec 100644 --- a/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs +++ b/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs @@ -40,7 +40,7 @@ public class AIQuestionsController : ControllerBase [HttpPost("user/{id}")] [RequirePermission("document:Read")] - public Task UserAccessById(Guid id,ContentDto content) + public Task UserAccessById(Guid id, ContentDto content) { throw new NotImplementedException();//尚未实现 } @@ -73,4 +73,18 @@ public class AIQuestionsController : ControllerBase return BadRequest(Result.Failure(e.Message)); } } + + [HttpGet("user/message/{ConversationsId}")] + public async Task GetAllConversationMessage(Guid ConversationsId) + { + try + { + var list = await _AIQusetionAppService.GetConversationMessage(ConversationsId); + return Ok(Result>.Success(list)); + } + catch (System.Exception e) + { + return BadRequest(Result.Failure(e.Message)); + } + } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.Designer.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.Designer.cs new file mode 100644 index 0000000..5334287 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.Designer.cs @@ -0,0 +1,394 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Pgvector; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + [DbContext(typeof(UniversalAdminSystemDbContext))] + [Migration("20250812085226_last")] + partial class last + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "vector"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("RolePermissions", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("PermissionId") + .HasColumnType("uuid"); + + b.HasKey("RoleId", "PermissionId"); + + b.HasIndex("PermissionId"); + + b.HasIndex("RoleId"); + + b.ToTable("RolePermissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.FileStorage.Aggregates.File", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AccessLevel") + .HasColumnType("integer"); + + b.Property("IsFolder") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OwnerId") + .HasColumnType("uuid"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.Property("SecurityCheckResult") + .HasColumnType("text"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UploadTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.LogManagement.Aggregates.LogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Context") + .HasColumnType("text"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("LogEntries"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", b => + { + b.Property("PermissionId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PermissionType") + .HasColumnType("integer"); + + b.Property("Resource") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("PermissionId"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("Permissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSupper") + .HasColumnType("boolean"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("RoleId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.SystemSettings.Aggregates.SystemSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Group") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("SystemSettings"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Conversations", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Conversations"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Message", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Messages"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("Account") + .IsRequired() + .HasColumnType("text"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.Property("Password") + .IsRequired() + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.HasKey("UserId"); + + b.HasIndex("Account") + .IsUnique(); + + b.HasIndex("RoleId"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Entities.UserInfo", b => + { + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.Property("Address") + .HasColumnType("text"); + + b.Property("Age") + .HasColumnType("integer"); + + b.Property("Gender") + .HasColumnType("integer"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Phone") + .HasColumnType("text"); + + b.HasKey("UserInfoId"); + + b.ToTable("UserInfos"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.knowledge.Aggregates.DocumentChunk", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("Embedding") + .IsRequired() + .HasColumnType("vector(1536)"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Embedding"); + + b.ToTable("Chunks"); + }); + + modelBuilder.Entity("RolePermissions", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", null) + .WithMany() + .HasForeignKey("PermissionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.SetNull); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.cs new file mode 100644 index 0000000..d47c9b5 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812085226_last.cs @@ -0,0 +1,27 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + /// + public partial class last : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateIndex( + name: "IX_Chunks_Embedding", + table: "Chunks", + column: "Embedding"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "IX_Chunks_Embedding", + table: "Chunks"); + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs index bde02a3..25bc4ff 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs @@ -358,6 +358,8 @@ namespace UniversalAdminSystem.Infrastructure.Migrations b.HasKey("Id"); + b.HasIndex("Embedding"); + b.ToTable("Chunks"); }); diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs index 0173534..c5b88ec 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs @@ -238,6 +238,7 @@ public class UniversalAdminSystemDbContext : DbContext modelBuilder.Entity(entity => { entity.HasKey(d => d.Id); + entity.HasIndex(d => d.Embedding); entity.Property(d => d.FileId) .HasConversion( -- Gitee From 48cf2d2d1d2ad5e52f97a80ec77ce871ea1b9233 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AE=97=E6=97=AD?= <1619917346@qq.com> Date: Wed, 13 Aug 2025 14:08:36 +0800 Subject: [PATCH 3/6] =?UTF-8?q?=E5=AE=8C=E6=88=90=E5=90=8E=E7=AB=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Controllers/AIQuestionsController.cs | 25 +- .../Controllers/FileController.cs | 1 - .../Controllers/TestController.cs | 53 +++ .../PermissionRules.json | 1 - .../src/UniversalAdminSystem.Api/Program.cs | 1 + .../UniversalAdminSystem.Api.http | 9 + .../UniversalAdminSystem.Api/appsettings.json | 16 +- .../Interfaces/IAIQusetionsAppService.cs | 3 +- .../AIQuestions/Interfaces/IAnswerService.cs | 8 + .../Service/AIQusetionsAppService.cs | 24 +- .../FileStorage/Services/FileAppService.cs | 19 +- .../UserConversations/Aggregates/Message.cs | 2 - .../knowledge/Aggregates/DocumentChunk.cs | 10 +- .../Configs/TongyiConfig.cs | 7 + .../AddInfrastructureService.cs | 25 +- .../20250812093344_postgre_vector.Designer.cs | 433 ++++++++++++++++++ .../20250812093344_postgre_vector.cs | 46 ++ ...0250812104226_postgre_vector_2.Designer.cs | 433 ++++++++++++++++++ .../20250812104226_postgre_vector_2.cs | 22 + ...versalAdminSystemDbContextModelSnapshot.cs | 41 ++ .../Consumers/FileProcessingJobConsumer.cs | 143 ++++-- .../RabbitMQ/Jobs/FileProcessingJob.cs | 14 - .../Publishers/FileProcessingJobPublisher.cs | 68 +-- .../RabbitMQ/Queues/EfFileProcessingQueue.cs | 16 +- .../Services/AnswerService.cs | 105 +++++ .../Services/EmbeddingService.cs | 172 +++++++ .../Services/K2ModelService.cs | 111 ++++- .../Services/SpaCyService.cs | 124 +++++ .../Services/TextExtractor.cs | 6 - ...UniversalAdminSystem.Infrastructure.csproj | 2 + 30 files changed, 1801 insertions(+), 139 deletions(-) create mode 100644 backend/src/UniversalAdminSystem.Api/Controllers/TestController.cs create mode 100644 backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAnswerService.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Configs/TongyiConfig.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.Designer.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.Designer.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.cs delete mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Services/EmbeddingService.cs create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Services/SpaCyService.cs delete mode 100644 backend/src/UniversalAdminSystem.Infrastructure/Services/TextExtractor.cs diff --git a/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs b/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs index 913abe9..6c8be7c 100644 --- a/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs +++ b/backend/src/UniversalAdminSystem.Api/Controllers/AIQuestionsController.cs @@ -23,14 +23,15 @@ public class AIQuestionsController : ControllerBase throw new NotImplementedException();//尚未实现 } - [HttpPost("user")] + [HttpPost("chat/{conversationsId}")] [RequirePermission("document:Read")] - public async Task CreateUserConversation(ConversationsDto conversationsDto) + public async Task SendMessageAsync(Guid conversationsId, [FromBody] string userMessage) { try { - var result = await _AIQusetionAppService.CreateConversation(conversationsDto); - return Ok(Result.Success(result)); + + var result = await _AIQusetionAppService.Chat(conversationsId, userMessage); + return Ok(Result.Success(result)); } catch (System.Exception e) { @@ -38,11 +39,19 @@ public class AIQuestionsController : ControllerBase } } - [HttpPost("user/{id}")] + [HttpPost("user")] [RequirePermission("document:Read")] - public Task UserAccessById(Guid id,ContentDto content) + public async Task CreateUserConversation(ConversationsDto conversationsDto) { - throw new NotImplementedException();//尚未实现 + try + { + var result = await _AIQusetionAppService.CreateConversation(conversationsDto); + return Ok(Result.Success(result)); + } + catch (System.Exception e) + { + return BadRequest(Result.Failure(e.Message)); + } } [HttpGet("user/{userid}")] @@ -60,7 +69,7 @@ public class AIQuestionsController : ControllerBase } } - [HttpDelete("user/{ConversationsId}")] + [HttpDelete("user/{conversationsId}")] public async Task DeleteUserConversations(Guid ConversationsId) { try diff --git a/backend/src/UniversalAdminSystem.Api/Controllers/FileController.cs b/backend/src/UniversalAdminSystem.Api/Controllers/FileController.cs index 99b4836..dc9c96e 100644 --- a/backend/src/UniversalAdminSystem.Api/Controllers/FileController.cs +++ b/backend/src/UniversalAdminSystem.Api/Controllers/FileController.cs @@ -31,7 +31,6 @@ public class FileController : ControllerBase try { var res = await _fileAppService.UploadAsync(file); - Console.WriteLine(res); return Ok(Result.Success(res)); } catch (Exception ex) diff --git a/backend/src/UniversalAdminSystem.Api/Controllers/TestController.cs b/backend/src/UniversalAdminSystem.Api/Controllers/TestController.cs new file mode 100644 index 0000000..8d6789b --- /dev/null +++ b/backend/src/UniversalAdminSystem.Api/Controllers/TestController.cs @@ -0,0 +1,53 @@ +using System.Text; +using System.Text.Json; +using Microsoft.AspNetCore.Mvc; +using UniversalAdminSystem.Infrastructure.Services; + +namespace UniversalAdminSystem.Api.Controllers; + +[ApiController] +[Route("api/[controller]")] +public class TestController : ControllerBase +{ + private readonly ILogger _logger; + private readonly SpaCyService _spacy; + private readonly K2ModelService _k2; + private readonly EmbeddingService _embedding; + + public TestController(ILogger logger, K2ModelService k2, SpaCyService spacy, EmbeddingService embedding) + { + _logger = logger; + _k2 = k2; + _spacy = spacy; + _embedding = embedding; + } + + [HttpPost("spacy")] + public async Task Spacy([FromBody] SpaCyRequest text) + { + // 文本分片 + _logger.LogInformation("开始分片文件: {Text}", text); + var preprocessResult = await _spacy.AnalyzeTextAsync(text.Text); + Console.WriteLine(preprocessResult.Entities); + Console.WriteLine(preprocessResult.Sentences); + + var preprocessResultJson = JsonSerializer.Serialize(preprocessResult, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + var chunks = await _k2.SendChunkingRequestAsync(preprocessResultJson); + _logger.LogInformation("分片完成，生成 {Count} 个 chunk", chunks.Count); + + // Embedding + _logger.LogInformation("开始Embedding"); + var embeddings = new List(); + embeddings.AddRange(await _embedding.GetEmbeddingAsync(chunks)); + _logger.LogInformation("Embedding 完成: {Count} 个向量", embeddings.Count); + return Ok(embeddings); + } +} + +public class SpaCyRequest(string text) +{ + public string Text { get; set; } = text; +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Api/PermissionRules.json b/backend/src/UniversalAdminSystem.Api/PermissionRules.json index 7e0ed8b..2c8a6a2 100644 --- a/backend/src/UniversalAdminSystem.Api/PermissionRules.json +++ b/backend/src/UniversalAdminSystem.Api/PermissionRules.json @@ -10,5 +10,4 @@ "Public": ["document"], "Restricted": ["document"], "Private": ["document"] - } diff --git a/backend/src/UniversalAdminSystem.Api/Program.cs b/backend/src/UniversalAdminSystem.Api/Program.cs index 29cae8a..0d372cd 100644 --- a/backend/src/UniversalAdminSystem.Api/Program.cs +++ b/backend/src/UniversalAdminSystem.Api/Program.cs @@ -8,6 +8,7 @@ var builder = WebApplication.CreateBuilder(args); // 加载外部的允许文件配置，避免 appsettings.json 膨胀 builder.Configuration.AddJsonFile("AllowedFiles.json", optional: false, reloadOnChange: true); +builder.Services.AddHttpClient(); // 添加CORS服务 builder.Services.AddCors(options => { diff --git a/backend/src/UniversalAdminSystem.Api/UniversalAdminSystem.Api.http b/backend/src/UniversalAdminSystem.Api/UniversalAdminSystem.Api.http index abcc71f..1596c1b 100644 --- a/backend/src/UniversalAdminSystem.Api/UniversalAdminSystem.Api.http +++ b/backend/src/UniversalAdminSystem.Api/UniversalAdminSystem.Api.http @@ -1,9 +1,18 @@ @url = http://localhost:5101 +### login POST {{url}}/api/auth/login Content-Type: application/json { "account": "manager", "password": "manager123" +} + +### test spacy +POST {{url}}/api/test/spacy +Content-Type: application/json + +{ + "text": "初音未来（日语：初音 ミク／はつねミク Hatsune Miku）是克理普敦未来媒体（简称Crypton Future Media）以雅马哈的VOCALOID语音合成引擎为基础开发的虚拟歌手。她的形象是一位留着青绿色双马尾、身穿未来感服装的16岁少女。自2007年发布以来，她在全球范围内获得了大量粉丝，演唱会甚至使用全息投影技术现场表演。她的粉丝遍布中国、日本、美国等地。Her popularity spread rapidly through platforms like YouTube, NicoNico Douga, and Bilibili.\n\nCrypton在2020年发布了基于全新M9引擎的初音未来NT版本，该版本改进了发音的自然度和表现力。初音未来不仅仅是一款软件，更是音乐创作文化的象征。许多音乐人，包括著名制作人ryo(supercell)、DECO*27，都为她创作了脍炙人口的作品，如《千本樱》《Tell Your World》。这些作品在全球播放量超过数亿次，并被翻唱成多种语言版本。\n以下是一个超长的无标点测试段落它包含了许多描述但是没有任何中文句号逗号或感叹号这种情况下我们的分片算法应该能够在达到最大token限制时自动切割同时保持尽可能的语义完整性而不是让分片在一个人名例如初音未来的中间被截断因为那样会影响RAG的召回效果\n此外，初音未来的成功还催生了大量衍生角色和跨界合作。例如，雪初音是她的冬季特别版本，已经连续多年成为札幌冰雪节的形象大使。她还与赛车、旅游、食品、服装等行业合作，形成了庞大的商业生态。In the future, AI voice synthesis technology may allow for even more natural and expressive performances, and Hatsune Miku could evolve into an interactive virtual idol capable of real-time conversations with fans." } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Api/appsettings.json b/backend/src/UniversalAdminSystem.Api/appsettings.json index 8db1331..8071a8e 100644 --- a/backend/src/UniversalAdminSystem.Api/appsettings.json +++ b/backend/src/UniversalAdminSystem.Api/appsettings.json @@ -8,8 +8,8 @@ }, "AllowedHosts": "*", "ConnectionStrings": { - "pgSql": "server=127.0.0.1;port=5432;uid=postgres;password=031028@yue;database=universal_admin", - "RabbitMq": "amqp://guest:guest@rabbitmq:5672/" + "pgSql": "Server=localhost;Port=5432;Username=admin;Password=031028@yue;Database=rag_vector_db", + "RabbitMq": "amqp://admin:031028%40yue@localhost:5672/" }, "Jwt": { "Key": "YourSuperSecretKey1232347509872093oiqewupori", @@ -18,11 +18,15 @@ "ExpireHours": 2 }, "K2": { - "BaseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1/", - "ApiKey": "sk-a31163f6b59e44dcbdb87c668482ce96" + "ApiKey": "sk-a31163f6b59e44dcbdb87c668482ce96", + "BaseUrl": "https://dashscope.aliyuncs.com/compatible-mode/v1/" + }, + "Tongyi": { + "ApiKey": "sk-a31163f6b59e44dcbdb87c668482ce96", + "BaseUrl": "https://dashscope.aliyuncs.com/" }, "RabbitMq": { - "Host": "rabbitmq", + "Host": "localhost", "Port": 5672, "Username": "guest", "Password": "guest", @@ -30,4 +34,4 @@ "Queue": "file-processing-queue", "RoutingKey": "file-processing" } -} \ No newline at end of file +} diff --git a/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAIQusetionsAppService.cs b/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAIQusetionsAppService.cs index 54072f3..c00924e 100644 --- a/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAIQusetionsAppService.cs +++ b/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAIQusetionsAppService.cs @@ -1,5 +1,4 @@ using UniversalAdminSystem.Application.AIQuestions.DTOs; -using UniversalAdminSystem.Domian.UserConversations.Aggregates; namespace UniversalAdminSystem.Application.AIQuestions.Interfaces; @@ -17,4 +16,6 @@ public interface IAIQusetionsAppService Task> GetConversationMessage(Guid Id); Task CreateConversation(ConversationsDto conversationsDto); + + Task Chat(Guid conversationId, string userMessage); } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAnswerService.cs b/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAnswerService.cs new file mode 100644 index 0000000..4ae1201 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Application/AIQuestions/Interfaces/IAnswerService.cs @@ -0,0 +1,8 @@ +using UniversalAdminSystem.Domian.UserConversations.Aggregates; + +namespace UniversalAdminSystem.Application.AIQuestions.Interfaces; + +public interface IAnswerService +{ + Task AnswerAsync(string userInput, IEnumerable? messages); +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs b/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs index 101ccb6..fa932ae 100644 --- a/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs +++ b/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs @@ -2,6 +2,7 @@ using UniversalAdminSystem.Application.AIQuestions.DTOs; using UniversalAdminSystem.Application.AIQuestions.Interfaces; using UniversalAdminSystem.Application.Common.Interfaces; using UniversalAdminSystem.Application.PermissionManagement.Interfaces; +using UniversalAdminSystem.Domian.Core.ValueObjects; using UniversalAdminSystem.Domian.UserConversations.Aggregates; using UniversalAdminSystem.Domian.UserConversations.IRepository; using UniversalAdminSystem.Domian.UserManagement.IRepository; @@ -19,18 +20,21 @@ public class AIQusetionsAppService : IAIQusetionsAppService private readonly IUserRepository _userRepository; private readonly IUnitOfWork _work; + private readonly IAnswerService _answerService; public AIQusetionsAppService(IMessageRepository message, IConversationsRepository conversations, IPermissionCheckService permissioncheck, IUnitOfWork work, - IUserRepository userRepository) + IUserRepository userRepository, + IAnswerService answerService) { _messageRepo = message; _conversationsRepo = conversations; _permissioncheck = permissioncheck; _work = work; _userRepository = userRepository; + _answerService = answerService; } public async Task CreateConversation(ConversationsDto conversationsDto) @@ -47,7 +51,7 @@ public class AIQusetionsAppService : IAIQusetionsAppService catch (System.Exception e) { await _work.RollbackAsync(); - throw new Exception( $"用户会话创建失败:{e.Message}"); + throw new Exception($"用户会话创建失败:{e.Message}"); } } @@ -72,7 +76,7 @@ public class AIQusetionsAppService : IAIQusetionsAppService try { var list = await _messageRepo.GetByConversationIdAsync(Id); - return list.Select(m => new ContentResultDto(m.Role,m.Content)); + return list.Select(m => new ContentResultDto(m.Role, m.Content)); } catch (System.Exception) { @@ -94,6 +98,20 @@ public class AIQusetionsAppService : IAIQusetionsAppService } } + public async Task Chat(Guid conversationId, string userInput) + { + // 开启事务 + await _work.BeginTransactionAsync(); + + // 创建消息并持久化 + var message = Message.Create(ConversationId.Create(conversationId), "user", userInput); + await _messageRepo.AddAsync(message); + + // 回答 + var messages = await _messageRepo.GetByConversationIdAsync(conversationId); + return await _answerService.AnswerAsync(userInput, messages); + } + public Task UserAccess(Guid id, ContentDto content) { throw new NotImplementedException(); diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs index 2fad793..5f81df6 100644 --- a/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs @@ -35,9 +35,8 @@ public class FileAppService : IFileAppService var (isValid, message, format) = _fileValidator.ValidateFile(file.Length, fileStream); if (!isValid) throw new Exception($"校验结果: {message}"); var fileInfo = await _localfileStorage.SaveAsync(file); - Console.WriteLine("开启事务..."); + Console.WriteLine("开启事务------------------"); await _unitOfWork.BeginTransactionAsync(); - Console.WriteLine("创建实体..."); var fileEntity = File.Create ( FileName.Create(fileInfo.Name), @@ -48,22 +47,24 @@ public class FileAppService : IFileAppService false, null ); - Console.WriteLine($"文件信息打印------------"); + Console.WriteLine($"文件信息打印---------------"); Console.WriteLine($"文件名：{fileEntity.Name}"); Console.WriteLine($"文件url：{fileEntity.Path}"); Console.WriteLine($"文件大小：{fileEntity.Size}"); Console.WriteLine($"文件类型：{fileEntity.Type}"); - Console.WriteLine("添加数据到数据库..."); + Console.WriteLine("---------------------------"); await _fileRepository.AddAsync(fileEntity); - Console.WriteLine("添加数据到成功！提交事务！"); - await _unitOfWork.CommitAsync(); + Console.WriteLine("添加数据成功！"); // 文件处理任务入队 await _fileProcessingQueue.EnqueueAsync(new FileProcessingJobDto( - fileEntity.Id.Value, - fileInfo.FullName, - file.ContentType, + fileEntity.Id.Value, + fileInfo.FullName, + file.ContentType, file.Length), CancellationToken.None); + Console.WriteLine("文件处理任务入队成功！提交事务！"); + await _unitOfWork.CommitAsync(); + Console.WriteLine("提交事务成功！"); return new FileUploadResultDto ( diff --git a/backend/src/UniversalAdminSystem.Domian/UserConversations/Aggregates/Message.cs b/backend/src/UniversalAdminSystem.Domian/UserConversations/Aggregates/Message.cs index c6ff4ff..1e4c6cc 100644 --- a/backend/src/UniversalAdminSystem.Domian/UserConversations/Aggregates/Message.cs +++ b/backend/src/UniversalAdminSystem.Domian/UserConversations/Aggregates/Message.cs @@ -1,6 +1,4 @@ -using System.Data.Common; using UniversalAdminSystem.Domian.Core.ValueObjects; -using UniversalAdminSystem.Domian.PermissionManagement.ValueObjects; namespace UniversalAdminSystem.Domian.UserConversations.Aggregates; diff --git a/backend/src/UniversalAdminSystem.Domian/knowledge/Aggregates/DocumentChunk.cs b/backend/src/UniversalAdminSystem.Domian/knowledge/Aggregates/DocumentChunk.cs index 99a063b..8dbefec 100644 --- a/backend/src/UniversalAdminSystem.Domian/knowledge/Aggregates/DocumentChunk.cs +++ b/backend/src/UniversalAdminSystem.Domian/knowledge/Aggregates/DocumentChunk.cs @@ -1,3 +1,4 @@ +using System.ComponentModel.DataAnnotations.Schema; using UniversalAdminSystem.Domian.Core; using UniversalAdminSystem.Domian.Core.ValueObjects; using UniversalAdminSystem.Domian.FileStorage.ValueObjects; @@ -15,12 +16,13 @@ public class DocumentChunk : AggregateRoot public string Content { get; private set; } = string.Empty; public TextEmbedding Embedding { get; private set; } = null!; - + [NotMapped] // EF Core 不会映射到表 + public double SimilarityScore { get; set; } public FileAccessLevel Level { get; private set; } protected DocumentChunk() { } - private DocumentChunk(ChunkId id, FileId fileId, string content, TextEmbedding embedding,FileAccessLevel level) + private DocumentChunk(ChunkId id, FileId fileId, string content, TextEmbedding embedding, FileAccessLevel level) { Id = id; FileId = fileId; @@ -29,7 +31,7 @@ public class DocumentChunk : AggregateRoot Level = level; } - public static DocumentChunk CreateDocumentChunk(ChunkId id, FileId fileId, string content, TextEmbedding embedding,FileAccessLevel level = FileAccessLevel.Public) + public static DocumentChunk CreateDocumentChunk(ChunkId id, FileId fileId, string content, TextEmbedding embedding, FileAccessLevel level = FileAccessLevel.Public) { if (string.IsNullOrWhiteSpace(content)) { @@ -41,7 +43,7 @@ public class DocumentChunk : AggregateRoot throw new ArgumentException("嵌入向量不能为空"); } - return new DocumentChunk(id, fileId, content, embedding,level); + return new DocumentChunk(id, fileId, content, embedding, level); } diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Configs/TongyiConfig.cs b/backend/src/UniversalAdminSystem.Infrastructure/Configs/TongyiConfig.cs new file mode 100644 index 0000000..a0a0f2e --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Configs/TongyiConfig.cs @@ -0,0 +1,7 @@ +namespace UniversalAdminSystem.Infrastructure.Configs; + +public class TongyiConfig +{ + public string BaseUrl { get; set; } = string.Empty; + public string ApiKey { get; set; } = string.Empty; +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs b/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs index 0d5aba2..58842c8 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs @@ -28,8 +28,8 @@ using UniversalAdminSystem.Infrastructure.RabbitMQ.Queues; using UniversalAdminSystem.Infrastructure.RabbitMQ; using UniversalAdminSystem.Infrastructure.RabbitMQ.Publishers; using UniversalAdminSystem.Infrastructure.RabbitMQ.Consumers; -using Microsoft.AspNetCore.Connections; using RabbitMQ.Client; +using UniversalAdminSystem.Application.AIQuestions.Interfaces; namespace UniversalAdminSystem.Infrastructure.DependencyInject; @@ -111,14 +111,12 @@ public static class AddInfrastrutureService return services; } - public static IServiceCollection AddInfrastruture(this IServiceCollection services, IConfiguration configuration) { - // services.Configure(configuration.GetSection("AllowedFiles")); // 注册数据库上下文 services.AddDbContext(x => x.UseNpgsql(configuration.GetConnectionString("pgSql"), x => x.UseVector())); - // 注册配置 + // 自动注册所有配置 services.AddAllConfig(configuration); // 自动注册所有仓储 @@ -157,11 +155,21 @@ public static class AddInfrastrutureService services.AddSingleton(); services.AddScoped(); + // 注册ali的HttpClient服务 + services.AddHttpClient(); + // 注册SpaCy的HttpClient服务 + services.AddHttpClient(); + // 注册K2模型相关服务 - services.AddHttpClient(); services.AddScoped(); services.AddScoped(); + // 注册SpaCy服务 + services.AddScoped(); + + // 注册Embedding服务 + services.AddScoped(); + // 注册权限相关服务 services.AddScoped(); services.AddScoped(); @@ -178,8 +186,8 @@ public static class AddInfrastrutureService services.AddHostedService(); services.AddSingleton(); - // 注册文件处理相关服务 - services.AddSingleton(); + // // 注册文件处理相关服务 + services.AddScoped(); services.AddSingleton(sp => { var factory = new ConnectionFactory @@ -203,11 +211,12 @@ public static class AddInfrastrutureService ch.QueueBind(queue, exchange, routingKey); return ch; }); - services.AddSingleton(); services.AddHostedService(); services.AddHostedService(); + // 注册聊天相关服务 + services.AddScoped(); return services; } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.Designer.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.Designer.cs new file mode 100644 index 0000000..a18b7af --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.Designer.cs @@ -0,0 +1,433 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Pgvector; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + [DbContext(typeof(UniversalAdminSystemDbContext))] + [Migration("20250812093344_postgre_vector")] + partial class postgre_vector + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "vector"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("RolePermissions", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("PermissionId") + .HasColumnType("uuid"); + + b.HasKey("RoleId", "PermissionId"); + + b.HasIndex("PermissionId"); + + b.HasIndex("RoleId"); + + b.ToTable("RolePermissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.FileStorage.Aggregates.File", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AccessLevel") + .HasColumnType("integer"); + + b.Property("IsFolder") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OwnerId") + .HasColumnType("uuid"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.Property("SecurityCheckResult") + .HasColumnType("text"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UploadTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.LogManagement.Aggregates.LogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Context") + .HasColumnType("text"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("LogEntries"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", b => + { + b.Property("PermissionId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PermissionType") + .HasColumnType("integer"); + + b.Property("Resource") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("PermissionId"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("Permissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSupper") + .HasColumnType("boolean"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("RoleId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.SystemSettings.Aggregates.SystemSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Group") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("SystemSettings"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Conversations", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Conversations"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Message", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Messages"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("Account") + .IsRequired() + .HasColumnType("text"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.Property("Password") + .IsRequired() + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.HasKey("UserId"); + + b.HasIndex("Account") + .IsUnique(); + + b.HasIndex("RoleId"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Entities.UserInfo", b => + { + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.Property("Address") + .HasColumnType("text"); + + b.Property("Age") + .HasColumnType("integer"); + + b.Property("Gender") + .HasColumnType("integer"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Phone") + .HasColumnType("text"); + + b.HasKey("UserInfoId"); + + b.ToTable("UserInfos"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.knowledge.Aggregates.DocumentChunk", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("Embedding") + .IsRequired() + .HasColumnType("vector(1536)"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Chunks"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs.FileProcessingJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ContentType") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("Status", "NextAttemptAt", "CreatedAt"); + + b.ToTable("FileProcessingJobs"); + }); + + modelBuilder.Entity("RolePermissions", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", null) + .WithMany() + .HasForeignKey("PermissionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.SetNull); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.cs new file mode 100644 index 0000000..3c14a14 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812093344_postgre_vector.cs @@ -0,0 +1,46 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + /// + public partial class postgre_vector : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "FileProcessingJobs", + columns: table => new + { + Id = table.Column(type: "uuid", nullable: false), + FileId = table.Column(type: "uuid", nullable: false), + FilePath = table.Column(type: "text", nullable: false), + ContentType = table.Column(type: "text", nullable: false), + Size = table.Column(type: "bigint", nullable: false), + Status = table.Column(type: "character varying(32)", maxLength: 32, nullable: false), + RetryCount = table.Column(type: "integer", nullable: false), + CreatedAt = table.Column(type: "timestamp with time zone", nullable: false), + NextAttemptAt = table.Column(type: "timestamp with time zone", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_FileProcessingJobs", x => x.Id); + }); + + migrationBuilder.CreateIndex( + name: "IX_FileProcessingJobs_Status_NextAttemptAt_CreatedAt", + table: "FileProcessingJobs", + columns: new[] { "Status", "NextAttemptAt", "CreatedAt" }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "FileProcessingJobs"); + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.Designer.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.Designer.cs new file mode 100644 index 0000000..ddf6436 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.Designer.cs @@ -0,0 +1,433 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Pgvector; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + [DbContext(typeof(UniversalAdminSystemDbContext))] + [Migration("20250812104226_postgre_vector_2")] + partial class postgre_vector_2 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "vector"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("RolePermissions", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("PermissionId") + .HasColumnType("uuid"); + + b.HasKey("RoleId", "PermissionId"); + + b.HasIndex("PermissionId"); + + b.HasIndex("RoleId"); + + b.ToTable("RolePermissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.FileStorage.Aggregates.File", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AccessLevel") + .HasColumnType("integer"); + + b.Property("IsFolder") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OwnerId") + .HasColumnType("uuid"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.Property("SecurityCheckResult") + .HasColumnType("text"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UploadTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.LogManagement.Aggregates.LogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Context") + .HasColumnType("text"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("LogEntries"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", b => + { + b.Property("PermissionId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PermissionType") + .HasColumnType("integer"); + + b.Property("Resource") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("PermissionId"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("Permissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSupper") + .HasColumnType("boolean"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("RoleId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.SystemSettings.Aggregates.SystemSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Group") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("SystemSettings"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Conversations", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Conversations"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Message", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Messages"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("Account") + .IsRequired() + .HasColumnType("text"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.Property("Password") + .IsRequired() + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.HasKey("UserId"); + + b.HasIndex("Account") + .IsUnique(); + + b.HasIndex("RoleId"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Entities.UserInfo", b => + { + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.Property("Address") + .HasColumnType("text"); + + b.Property("Age") + .HasColumnType("integer"); + + b.Property("Gender") + .HasColumnType("integer"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Phone") + .HasColumnType("text"); + + b.HasKey("UserInfoId"); + + b.ToTable("UserInfos"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.knowledge.Aggregates.DocumentChunk", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("Embedding") + .IsRequired() + .HasColumnType("vector(1536)"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Chunks"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs.FileProcessingJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ContentType") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("Status", "NextAttemptAt", "CreatedAt"); + + b.ToTable("FileProcessingJobs"); + }); + + modelBuilder.Entity("RolePermissions", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", null) + .WithMany() + .HasForeignKey("PermissionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.SetNull); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.cs new file mode 100644 index 0000000..1852276 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250812104226_postgre_vector_2.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + /// + public partial class postgre_vector_2 : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs index bde02a3..41a3d70 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs @@ -361,6 +361,47 @@ namespace UniversalAdminSystem.Infrastructure.Migrations b.ToTable("Chunks"); }); + modelBuilder.Entity("UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs.FileProcessingJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ContentType") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("Status", "NextAttemptAt", "CreatedAt"); + + b.ToTable("FileProcessingJobs"); + }); + modelBuilder.Entity("RolePermissions", b => { b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", null) diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs index e47d80a..17cacda 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs @@ -1,11 +1,16 @@ using System.Text; using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; +using UniversalAdminSystem.Domian.knowledge.Aggregates; +using UniversalAdminSystem.Domian.knowledge.IRepository; using UniversalAdminSystem.Infrastructure.FileStorage.Parsers; using UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Models.Messages; +using UniversalAdminSystem.Infrastructure.Services; namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Consumers; @@ -13,65 +18,127 @@ public class FileProcessingJobConsumer : BackgroundService { private readonly IModel _ch; private readonly ILogger _logger; + private readonly IServiceScopeFactory _scopeFactory; + private readonly int _consumerCount = 5; - public FileProcessingJobConsumer(IModel ch, ILogger logger) + public FileProcessingJobConsumer(IModel ch, ILogger logger, IServiceScopeFactory scopeFactory) { _ch = ch; _logger = logger; + _scopeFactory = scopeFactory; } protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + _logger.LogInformation("正在连接到 RabbitMQ..."); try { - var consumer = new EventingBasicConsumer(_ch); - consumer.Received += async (sender, e) => + var consumerTasks = new List(); + for (int i = 0; i < _consumerCount; i++) { - var body = e.Body.ToArray(); - var message = Encoding.UTF8.GetString(body); - _logger.LogInformation("收到文件处理任务: {Message}", message); - try + consumerTasks.Add(AssignConsumerAsync(stoppingToken, i)); + } + await Task.WhenAll(consumerTasks); + } + catch (Exception ex) { _logger.LogError(ex, "文件处理任务消费失败"); } + } + + private async Task AssignConsumerAsync(CancellationToken stoppingToken, int consumerIndex) + { + _logger.LogInformation($"消费者{consumerIndex}正在启动并等待消息..."); + var consumer = new AsyncEventingBasicConsumer(_ch); + _logger.LogInformation("正在订阅队列: file-processing-queue"); + consumer.Received += async (sender, e) => + { + var body = e.Body.ToArray(); + var message = Encoding.UTF8.GetString(body); + _logger.LogInformation($"消费者 {consumerIndex} 收到文件处理任务: {message}"); + try + { + var fpm = JsonSerializer.Deserialize(message); + if (fpm == null) { - await ProcessMessageAsync(message, stoppingToken); - _ch.BasicAck(e.DeliveryTag, false); - _logger.LogInformation("文件处理任务处理成功: {Message}", message); + _logger.LogError("消息格式错误，跳过处理"); + _ch.BasicNack(e.DeliveryTag, false, false); + return; } - catch (Exception ex) + if (fpm.NextAttemptAt.HasValue && fpm.NextAttemptAt > DateTime.UtcNow) { - _ch.BasicNack(e.DeliveryTag, false, true); - _logger.LogError(ex, "文件处理任务处理失败: {Message}", message); + _logger.LogInformation($"等待重试，当前时间：{DateTime.UtcNow}，下一次尝试时间：{fpm.NextAttemptAt.Value}"); + await Task.Delay(fpm.NextAttemptAt.Value - DateTime.UtcNow, stoppingToken); } - }; - _ch.BasicConsume(queue: "file-processing-queue", autoAck: false, consumer: consumer); - await Task.Delay(Timeout.Infinite, stoppingToken); - } - catch (Exception ex) { _logger.LogError(ex, "文件处理任务消费失败"); } + await ProcessMessageAsync(fpm, stoppingToken); + _ch.BasicAck(e.DeliveryTag, false); + _logger.LogInformation($"消费者 {consumerIndex} 文件处理任务处理成功: {message}"); + } + catch (Exception ex) + { + var fpm = JsonSerializer.Deserialize(message); + if (fpm == null) return; + fpm.RetryCount++; + if (fpm.RetryCount >= 3) + { + _logger.LogError($"消息 {fpm.JobId} 达到最大重试次数，丢弃消息: {message}"); + _ch.BasicNack(e.DeliveryTag, false, false); + return; + } + var retryDelay = Math.Min(300, Math.Pow(2, fpm.RetryCount)); + fpm.NextAttemptAt = DateTime.UtcNow.AddSeconds(retryDelay); + var payload = JsonSerializer.Serialize(fpm); + _ch.BasicPublish("file-processing", "file-processing", null, Encoding.UTF8.GetBytes(payload)); + _logger.LogError(ex, $"消费者 {consumerIndex} 文件处理任务失败: {message}, 重试次数: {fpm.RetryCount}, 下次重试延迟: {retryDelay} 秒"); + } + }; + _ch.BasicConsume(queue: "file-processing-queue", autoAck: false, consumer: consumer); + await Task.Delay(Timeout.Infinite, stoppingToken); } - private async Task ProcessMessageAsync(string message, CancellationToken stoppingToken) + private async Task ProcessMessageAsync(FileProcessingMessage fpm, CancellationToken stoppingToken) { - var job = JsonSerializer.Deserialize(message); - if (job == null) + try { - _logger.LogError("文件处理任务反序列化失败: {Message}", message); - return; - } - // 文本提取 - _logger.LogInformation("开始处理文件: {FilePath}", job.FilePath); - var parser = new DocParserFactory().GetParser(job.ContentType); - var text = await parser.ParseAsync(job.FilePath); - _logger.LogInformation("文件处理完成: {FilePath}", job.FilePath); + // 文本提取 + _logger.LogInformation("开始处理文件: {FilePath}", fpm.FilePath); + var parser = new DocParserFactory().GetParser(fpm.ContentType); + var text = await parser.ParseAsync(fpm.FilePath); + _logger.LogInformation("文件处理完成: {FilePath}", fpm.FilePath); + + using (var scope = _scopeFactory.CreateScope()) + { + var spacy = scope.ServiceProvider.GetRequiredService(); + var k2 = scope.ServiceProvider.GetRequiredService(); + var embedding = scope.ServiceProvider.GetRequiredService(); + var docChunkRepo = scope.ServiceProvider.GetRequiredService(); - // 文本分片 - _logger.LogInformation("开始分片文件: {FilePath}", job.FilePath); + // 文本分片 + _logger.LogInformation("开始分片文件: {Text}", text); + var preprocessResult = await spacy.AnalyzeTextAsync(text); + var preprocessResultJson = JsonSerializer.Serialize(preprocessResult, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + var chunks = await k2.SendChunkingRequestAsync(preprocessResultJson); + _logger.LogInformation("分片完成，生成 {Count} 个 chunk", chunks.Count); + + // Embedding + _logger.LogInformation("开始Embedding"); + var embeddings = new List(); + embeddings.AddRange(await embedding.GetEmbeddingAsync(chunks)); + _logger.LogInformation("Embedding 完成: {Count} 个向量", embeddings.Count); - // Embedding - _logger.LogInformation("开始Embedding文件: {FilePath}", job.FilePath); + // Vector Store + _logger.LogInformation("开始Vector Store文件: {FilePath}", fpm.FilePath); + var docChunks = new List(); + await docChunkRepo.BulkAddDocumentChunkAsync(docChunks); - // Vector Store - _logger.LogInformation("开始Vector Store文件: {FilePath}", job.FilePath); - - // 文件处理完成 - _logger.LogInformation("文件处理完成: {FilePath}", job.FilePath); + // 文件处理完成 + _logger.LogInformation("文件处理完成: {FilePath}", fpm.FilePath); + } + } + catch (Exception ex) + { + _logger.LogError(ex, "文件处理任务失败: {Message}", fpm); + throw; + } } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs deleted file mode 100644 index c41aaf7..0000000 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Jobs/FileProcessingJob.cs +++ /dev/null @@ -1,14 +0,0 @@ -namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; - -public class FileProcessingJob -{ - public Guid Id { get; set; } = Guid.NewGuid(); - public Guid FileId { get; set; } - public string FilePath { get; set; } = default!; - public string ContentType { get; set; } = default!; - public long Size { get; set; } - public string Status { get; set; } = "Pending"; // Pending|Processing|Succeeded|Failed - public int RetryCount { get; set; } = 0; - public DateTime CreatedAt { get; set; } = DateTime.UtcNow; - public DateTime? NextAttemptAt { get; set; } -} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs index 645c421..d9bde02 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using UniversalAdminSystem.Application.Common.Interfaces; using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; +using UniversalAdminSystem.Infrastructure.RabbitMQ.Models.Messages; namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Publishers; @@ -21,40 +22,51 @@ public class FileProcessingJobPublisher : BackgroundService { while (!stoppingToken.IsCancellationRequested) { - try + await Task.Run(async () => { - using var scope = _sf.CreateScope(); - var db = scope.ServiceProvider.GetRequiredService(); - var jobs = await db.FileProcessingJobs - .Where(x => x.Status == "Pending" && (x.NextAttemptAt == null || x.NextAttemptAt <= DateTime.UtcNow)) - .OrderBy(x => x.CreatedAt) - .Take(100) - .ToListAsync(stoppingToken); + try + { + using var scope = _sf.CreateScope(); + var db = scope.ServiceProvider.GetRequiredService(); + var jobs = await db.FileProcessingJobs + .Where(x => x.Status == "Pending" && (x.NextAttemptAt == null || x.NextAttemptAt <= DateTime.UtcNow)) + .OrderBy(x => x.CreatedAt) + .Take(100) + .ToListAsync(stoppingToken); - foreach (var j in jobs) j.Status = "Processing"; - await db.SaveChangesAsync(stoppingToken); + foreach (var j in jobs) j.Status = "Processing"; + await db.SaveChangesAsync(stoppingToken); - foreach (var j in jobs) - { - try + foreach (var j in jobs) { - var payload = JsonSerializer.Serialize(new { j.Id, j.FileId, j.FilePath, j.ContentType, j.Size }); - await _bus.PublishAsync("file-processing", "file-processing", payload, messageId: j.Id.ToString(), stoppingToken); - j.Status = "Succeeded"; - } - catch (Exception ex) - { - j.RetryCount += 1; - j.Status = "Pending"; - j.NextAttemptAt = DateTime.UtcNow.AddSeconds(Math.Min(300, Math.Pow(2, j.RetryCount))); - _logger.LogError(ex, "发布失败: {JobId}", j.Id); + var message = new FileProcessingMessage + { + JobId = j.Id, + FilePath = j.FilePath, + ContentType = j.ContentType, + Size = j.Size, + RetryCount = j.RetryCount + }; + var payload = JsonSerializer.Serialize(message); + try + { + await _bus.PublishAsync("file-processing", "file-processing", payload, messageId: j.Id.ToString(), stoppingToken); + j.Status = "Succeeded"; + } + catch (Exception ex) + { + j.RetryCount++; + j.Status = "Pending"; + j.NextAttemptAt = DateTime.UtcNow.AddSeconds(Math.Min(300, Math.Pow(2, j.RetryCount))); + _logger.LogError(ex, "发布失败: {JobId}", j.Id); + } } + await db.SaveChangesAsync(stoppingToken); } - await db.SaveChangesAsync(stoppingToken); - } - catch (Exception ex) { _logger.LogError(ex, "文件处理任务发布循环错误"); } - - await Task.Delay(TimeSpan.FromSeconds(2), stoppingToken); + catch (Exception ex) { _logger.LogError(ex, "文件处理任务发布循环错误"); } + }); + + await Task.Delay(TimeSpan.FromSeconds(5), stoppingToken); } } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs index 2a0bbf0..26339ae 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Queues/EfFileProcessingQueue.cs @@ -9,9 +9,9 @@ namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Queues; public class EfFileProcessingQueue : IFileProcessingQueue { private readonly UniversalAdminSystemDbContext _dbContext; - private readonly Logger _logger; + private readonly ILogger _logger; - public EfFileProcessingQueue(UniversalAdminSystemDbContext dbContext, Logger logger) + public EfFileProcessingQueue(UniversalAdminSystemDbContext dbContext, ILogger logger) { _dbContext = dbContext; _logger = logger; @@ -21,6 +21,7 @@ public class EfFileProcessingQueue : IFileProcessingQueue { try { + _logger.LogInformation("准备插入任务: {FileId}, FilePath: {FilePath}", jd.FileId, jd.FilePath); _dbContext.FileProcessingJobs.Add(new FileProcessingJob { FileId = jd.FileId, @@ -28,6 +29,7 @@ public class EfFileProcessingQueue : IFileProcessingQueue ContentType = jd.ContentType, Size = jd.Size }); + _logger.LogInformation("DbContext IsConnected: {IsConnected}", _dbContext.Database.CanConnect()); await _dbContext.SaveChangesAsync(ct); _logger.LogInformation("文件处理任务已入队: {FileId}", jd.FileId); } @@ -37,4 +39,12 @@ public class EfFileProcessingQueue : IFileProcessingQueue throw; } } -} \ No newline at end of file +} +// public class EfFileProcessingQueue : IFileProcessingQueue +// { +// public Task EnqueueAsync(FileProcessingJobDto jd, CancellationToken ct = default) +// { +// // 什么都不做，直接返回 +// return Task.CompletedTask; +// } +// } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs new file mode 100644 index 0000000..f6685df --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs @@ -0,0 +1,105 @@ +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.DependencyInjection; +using UniversalAdminSystem.Application.AIQuestions.Interfaces; +using UniversalAdminSystem.Domian.FileStorage.ValueObjects; +using UniversalAdminSystem.Domian.knowledge.Aggregates; +using UniversalAdminSystem.Domian.knowledge.IRepository; +using UniversalAdminSystem.Domian.knowledge.ValueObj; +using UniversalAdminSystem.Domian.UserConversations.Aggregates; +using UniversalAdminSystem.Domian.UserConversations.IRepository; +using UniversalAdminSystem.Infrastructure.Configs; + +namespace UniversalAdminSystem.Infrastructure.Services; + +public class AnswerService : IAnswerService +{ + private readonly IServiceScopeFactory _scopeFactory; + private readonly IMessageRepository _messageRepo; + private readonly IDocumentChunkRepository _docChunkRepo; + + public AnswerService(IServiceScopeFactory scopeFactory, IMessageRepository messageRepo, IDocumentChunkRepository docChunkRepo) + { + _scopeFactory = scopeFactory; + _messageRepo = messageRepo; + _docChunkRepo = docChunkRepo; + } + + public async Task AnswerAsync(string userInput, IEnumerable? messages) + { + using var scope = _scopeFactory.CreateScope(); + var spacy = scope.ServiceProvider.GetRequiredService(); + var k2 = scope.ServiceProvider.GetRequiredService(); + var embedding = scope.ServiceProvider.GetRequiredService(); + + + // 1. 解析用户输入 + var parsedUserInput = await spacy.ParseUserInputAsync(userInput); + System.Console.WriteLine(parsedUserInput); + var parsedUserInputJson = JsonSerializer.Serialize(parsedUserInput, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + + // 2. 让 K2 进行分片 + var chunksText = await k2.SendChunkingRequestAsync(parsedUserInputJson); + + // 3. 向量化分片 + var userInputEmbeddings = await embedding.GetEmbeddingAsync(chunksText); + + // 4. 相似文档检索（并行） + var allChunks = new List(); + + var retrievalTasks = userInputEmbeddings.Select(async vec => + { + var publicDocs = await _docChunkRepo.FindSimilarDocumentsAsync(TextEmbedding.Create(vec), FileAccessLevel.Public); + var restrictedDocs = await _docChunkRepo.FindSimilarDocumentsAsync(TextEmbedding.Create(vec), FileAccessLevel.Restricted); + return publicDocs.Concat(restrictedDocs); + }); + + var retrievedResults = await Task.WhenAll(retrievalTasks); + allChunks.AddRange(retrievedResults.SelectMany(r => r)); + + // 5. 重排文档 + var sortedChunks = allChunks + .Where(c => !string.IsNullOrWhiteSpace(c.Content)) + .OrderByDescending(chunk => chunk.SimilarityScore) + .Distinct() + .ToList(); + + // 6. 构造增强 Prompt（取前 N 个最相关文档） + var topContext = string.Join("\n\n", sortedChunks.Take(5).Select(c => c.Content)); + + var systemPrompt = $@"你是一个具有超长记忆的语言大师，你的名字叫做初音未来。请基于以下参考文档内容和用户问题，精准、详细地回答问题。参考文档：{topContext}".Trim(); + + // 7. 构造对话消息 + List chatMessages; + if (messages != null && messages.Any()) + { + chatMessages = messages.Select(m => new K2Message + { + Role = m.Role, + Content = m.Content + }).ToList(); + chatMessages.Insert(0, new K2Message { Role = "system", Content = systemPrompt }); + } + else + { + chatMessages = new List + { + new K2Message { Role = "system", Content = systemPrompt }, + new K2Message { Role = "user", Content = userInput } + }; + } + + // 8. 调用 K2 模型并接收流式结果 + var sb = new StringBuilder(); + await foreach (var chunk in k2.SendChatRequestAsync(chatMessages)) + { + sb.Append(chunk); + } + + return sb.ToString(); + } + +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/EmbeddingService.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/EmbeddingService.cs new file mode 100644 index 0000000..218a343 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Services/EmbeddingService.cs @@ -0,0 +1,172 @@ +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Options; +using UniversalAdminSystem.Infrastructure.Configs; + +namespace UniversalAdminSystem.Infrastructure.Services; + +public class EmbeddingService +{ + private readonly TongyiConfig _tongyiConfig; + private readonly HttpClient _httpClient; + + public EmbeddingService(IOptions tongyiConfig, IHttpClientFactory httpClientFactory) + { + _tongyiConfig = tongyiConfig.Value; + _httpClient = httpClientFactory.CreateClient("EmbeddingClient"); + _httpClient.BaseAddress = new Uri(_tongyiConfig.BaseUrl); + _httpClient.DefaultRequestHeaders.Add("Authorization", $"Bearer {_tongyiConfig.ApiKey}"); + _httpClient.DefaultRequestHeaders.Add("Accept", "application/json"); + } + + public async Task> GetEmbeddingAsync(List texts) + { + HttpRequestMessage request; + HttpResponseMessage response; + const int batchSize = 10; + var allEmbeddings = new List(); + + for (int i = 0; i < texts.Count; i += batchSize) + { + var batch = texts.Skip(i).Take(batchSize).ToArray(); + var payload = new + { + model = "text-embedding-v4", + input = batch, + encodingFormat = "float" + }; + Console.WriteLine("Request Body: " + JsonSerializer.Serialize(payload)); + request = new HttpRequestMessage(HttpMethod.Post, $"{_tongyiConfig.BaseUrl}compatible-mode/v1/embeddings") + { + Content = new StringContent(JsonSerializer.Serialize(payload, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }), Encoding.UTF8, "application/json") + }; + response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationToken.None); + // response.EnsureSuccessStatusCode(); + var json = await response.Content.ReadAsStringAsync(); + Console.WriteLine($"状态码: {response.StatusCode}, 响应: {json}"); + var embeddingResponse = JsonSerializer.Deserialize(json, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + if (embeddingResponse?.Data != null) + { + allEmbeddings.AddRange(embeddingResponse.Data.Select(d => d.Embedding.ToArray())); + } + } + + return allEmbeddings; + } + + public async Task SubmitEmbeddingTaskAsync(string url) + { + HttpRequestMessage request; + HttpResponseMessage response; + var payload = new + { + model = "text-embedding-async-v2", + input = url, + parameters = new { textType = "query" } + }; + request = new HttpRequestMessage(HttpMethod.Post, $"{_tongyiConfig.BaseUrl}api/v1/services/embeddings/text-embedding/text-embedding") + { + Content = new StringContent(JsonSerializer.Serialize(payload, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }), Encoding.UTF8, "application/json") + }; + request.Headers.Add("X-DashScope-Async", "enable"); + response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var res = JsonSerializer.Deserialize(json); + + return res.Output.TaskId; + } + + public async Task WaitForTaskAsync(string taskId, int pollIntervalSeconds = 5) + { + while (true) + { + var response = await _httpClient.GetAsync($"tasks/{taskId}"); + response.EnsureSuccessStatusCode(); + + var json = await response.Content.ReadAsStringAsync(); + var status = JsonSerializer.Deserialize(json); + + Console.WriteLine($"任务状态: {status.Output.TaskStatus}"); + + if (status.Output.TaskStatus == "SUCCEEDED") + { + return status.Output.ResultUrl; + } + else if (status.Output.TaskStatus == "FAILED") + { + throw new Exception("向量化任务失败"); + } + + await Task.Delay(TimeSpan.FromSeconds(pollIntervalSeconds)); + } + } + + public async Task DownloadResultAsync(string resultUrl) + { + var result = await _httpClient.GetStringAsync(resultUrl); + return result; + } +} +public class EmbeddingResponse +{ + public string Id { get; set; } + + public List Data { get; set; } + + public string Object { get; set; } + + public string Model { get; set; } + + public Usage Usage { get; set; } + +} + +public class EmbeddingData +{ + public string Object { get; set; } + public int Index { get; set; } + public List Embedding { get; set; } +} + +public class Usage +{ + public int PromptTokens { get; set; } + + public int TotalTokens { get; set; } +} + +public class AsyncEmbeddingResponse +{ + public string RequestId { get; set; } + public OutputData Output { get; set; } +} + +public class OutputData +{ + public string TaskId { get; set; } + public string TaskStatus { get; set; } +} + +public class TaskStatusResponse +{ + public string RequestId { get; set; } + public TaskOutput Output { get; set; } +} + +public class TaskOutput +{ + public string TaskStatus { get; set; } + public string ResultUrl { get; set; } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs index 600c365..ff210c6 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs @@ -1,12 +1,13 @@ using System.Text; using System.Text.Json; +using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using UniversalAdminSystem.Infrastructure.Configs; namespace UniversalAdminSystem.Infrastructure.Services; -public class K2ModelService +public partial class K2ModelService { private readonly K2Config _k2Config; private readonly ILogger _logger; @@ -39,7 +40,7 @@ public class K2ModelService Temperature = temperature, MaxTokens = maxTokens }; - request = new HttpRequestMessage(HttpMethod.Post, $"{_k2Config.BaseUrl}/chat/completions") + request = new HttpRequestMessage(HttpMethod.Post, $"{_k2Config.BaseUrl}chat/completions") { Content = new StringContent(JsonSerializer.Serialize(k2Request, new JsonSerializerOptions { @@ -55,17 +56,17 @@ public class K2ModelService throw; } await using var responseStream = await response.Content.ReadAsStreamAsync(); - await foreach (var chunk in ParseStreamingResponseAsync(responseStream)) + var sb = new StringBuilder(); + await foreach (var chunk in ParseStreamingResponseAsync(responseStream, sb)) { yield return chunk; } _logger.LogInformation("K2模型请求成功完成"); } - private async IAsyncEnumerable ParseStreamingResponseAsync(Stream responseStream) + private async IAsyncEnumerable ParseStreamingResponseAsync(Stream responseStream, StringBuilder answer) { using var reader = new StreamReader(responseStream); - var fullContent = new StringBuilder(); var responseId = ""; var responseModel = ""; long responseCreated = 0; @@ -99,8 +100,8 @@ public class K2ModelService // 保存响应元数据 if (!string.IsNullOrEmpty(chunk.Id)) responseId = chunk.Id; if (!string.IsNullOrEmpty(chunk.Model)) responseModel = chunk.Model; - if (chunk.Created > 0) responseCreated = chunk.Created; if (!string.IsNullOrEmpty(chunk.Object)) responseObject = chunk.Object; + if (chunk.Created > 0) responseCreated = chunk.Created; // 处理choices if (chunk.Choices != null && chunk.Choices.Count > 0) @@ -108,7 +109,7 @@ public class K2ModelService var choice = chunk.Choices[0]; if (choice.Delta != null && !string.IsNullOrEmpty(choice.Delta.Content)) { - fullContent.Append(choice.Delta.Content); + answer.Append(choice.Delta.Content); OnContentChunkReceived?.Invoke(choice.Delta.Content); yield return choice.Delta.Content; } @@ -117,4 +118,100 @@ public class K2ModelService } } } + + public async Task> SendChunkingRequestAsync(string preprocessedJson) + { + HttpRequestMessage request; + HttpResponseMessage response; + try + { + string model = "Moonshot-Kimi-K2-Instruct"; float temperature = 0.7f; int maxTokens = 1000; + _logger.LogInformation("开始发送K2模型请求，使用模型: {Model}", model); + var k2Request = new K2Request + { + Model = model, + Messages = { + new K2Message { Role = "system", Content = "你是一个文本分片助手，任务是将长文本分割成合适大小的块（chunk），每个 chunk 不超过 500 个 token。在分片时需要遵守以下规则：1. 尽量在句子边界进行切分。2. 不要将命名实体（例如人名、地名、作品名等）拆分到不同 chunk 中。3. 尽可能保证每个 chunk 的语义完整。4. 如果文本中有无标点长句，可以在合适位置手动加切分点，但要保证语义不被破坏。" }, + new K2Message { Role = "user", Content = $"以下是文本的预处理结果，请进行精准分片：\n{preprocessedJson}, 以json格式返回结果，每个分片包含chunk_id、content、token_count字段" }, + }, + Temperature = temperature, + MaxTokens = maxTokens, + }; + request = new HttpRequestMessage(HttpMethod.Post, $"{_k2Config.BaseUrl}chat/completions") + { + Content = new StringContent(JsonSerializer.Serialize(k2Request, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }), Encoding.UTF8, "application/json") + }; + response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationToken.None); + response.EnsureSuccessStatusCode(); + } + catch (Exception ex) + { + _logger.LogError(ex, "K2模型请求过程中发生错误"); + throw; + } + await using var responseStream = await response.Content.ReadAsStreamAsync(); + var sb = new StringBuilder(); + await foreach (var chunk in ParseStreamingResponseAsync(responseStream, sb)) { } + + return ParseChunksFromLLMAnswer(sb.ToString()); + } + + // private List ParseChunksFromLLMAnswer(string answer) + // { + // var chunks = new List(); + // using (JsonDocument jsonDoc = JsonDocument.Parse(answer)) + // { + // foreach (var chunk in jsonDoc.RootElement.EnumerateArray()) + // { + // if (chunk.TryGetProperty("content", out var jsonContent)) + // { + // var content = jsonContent.GetString(); + // if (content == null) continue; + // _logger.LogInformation("文本分片: {content} ", content); + // chunks.Add(content); + // } + // } + // } + + // if (chunks.Count > 0) return chunks; + // return []; + // } + + private List ParseChunksFromLLMAnswer(string answer) + { + var chunks = new List(); + using var jsonDoc = JsonDocument.Parse(answer); + + JsonElement root = jsonDoc.RootElement; + + // 如果根是对象并且包含 "chunks" + if (root.ValueKind == JsonValueKind.Object && root.TryGetProperty("chunks", out var chunksProp)) + { + if (chunksProp.ValueKind == JsonValueKind.Array) + { + foreach (var chunk in chunksProp.EnumerateArray()) + ExtractContent(chunk, chunks); + } + } + else if (root.ValueKind == JsonValueKind.Array) + { + foreach (var chunk in root.EnumerateArray()) + ExtractContent(chunk, chunks); + } + + return chunks; + } + + private void ExtractContent(JsonElement chunk, List chunks) + { + if (chunk.TryGetProperty("content", out var jsonContent)) + { + var content = jsonContent.GetString(); + if (!string.IsNullOrEmpty(content)) + chunks.Add(content); + } + } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/SpaCyService.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/SpaCyService.cs new file mode 100644 index 0000000..8d041d0 --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/Services/SpaCyService.cs @@ -0,0 +1,124 @@ +using System.Text.Json; + +namespace UniversalAdminSystem.Infrastructure.Services; + +public class SpaCyService +{ + private readonly HttpClient _httpClient; + private readonly string _baseUrl = "http://127.0.0.1:5000"; + + // 构造函数，注入 IHttpClientFactory + public SpaCyService(IHttpClientFactory httpClientFactory) + { + _httpClient = httpClientFactory.CreateClient("SpaCyClient"); + _httpClient.BaseAddress = new Uri(_baseUrl); + } + + // 发送请求到 Flask API 获取 文本分析 结果 + public async Task AnalyzeTextAsync(string text) + { + HttpRequestMessage request; + HttpResponseMessage response; + try + { + // 构造请求内容 + request = new HttpRequestMessage(HttpMethod.Post, $"{_baseUrl}/spacy") + { + Content = new StringContent(JsonSerializer.Serialize(new { text }), System.Text.Encoding.UTF8, "application/json") + }; + + // 发送请求 + response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + // 读取响应内容 + var result = await response.Content.ReadAsStringAsync(); + Console.WriteLine("Response from Flask: " + result); // 打印响应数据，方便调试 + + // 解析响应 JSON + var preprocessResult = JsonSerializer.Deserialize(result); + Console.WriteLine("preprocessResult: " + preprocessResult); + + if (preprocessResult != null && preprocessResult.Entities.Count > 0) + { + // 打印解析后的对象（调试） + Console.WriteLine("Entities:"); + foreach (var entity in preprocessResult.Entities) + { + Console.WriteLine($"Text: {entity.Text}, Label: {entity.Label}"); + } + + Console.WriteLine("\nSentences:"); + foreach (var sentence in preprocessResult.Sentences) + { + Console.WriteLine(sentence); + } + } + + return preprocessResult ?? throw new Exception("文本预处理错误"); + } + catch (Exception ex) + { + // 处理错误，例如网络问题或 API 服务不可用 + Console.WriteLine($"Error: {ex.Message}"); + throw; + } + } + + // 发送请求到 Flask API 获取 用户输入解析 结果 + public async Task ParseUserInputAsync(string userInput) + { + HttpRequestMessage request; + HttpResponseMessage response; + try + { + // 构造请求内容 + request = new HttpRequestMessage(HttpMethod.Post, $"{_baseUrl}/parse_input") + { + Content = new StringContent(JsonSerializer.Serialize(new { content = userInput }), System.Text.Encoding.UTF8, "application/json") + }; + + // 发送请求 + response = await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, CancellationToken.None); + response.EnsureSuccessStatusCode(); + + // 读取响应内容 + var result = await response.Content.ReadAsStringAsync(); + Console.WriteLine("Response from Flask: " + result); // 打印响应数据，方便调试 + + // 解析响应结果 + var userInputParseResult = JsonSerializer.Deserialize(result); + Console.WriteLine("parseResult: " + userInputParseResult); + + return userInputParseResult ?? throw new Exception("文本预处理错误"); + } + catch (Exception ex) + { + // 处理错误，例如网络问题或 API 服务不可用 + Console.WriteLine($"Error: {ex.Message}"); + throw; + } + } +} + +// 预处理结果类 +public class PreprocessResult +{ + public List Entities { get; set; } = []; + public List Sentences { get; set; } = []; +} + +// 实体类 +public class Entity +{ + public string Text { get; set; } = string.Empty; + public string Label { get; set; } = string.Empty; +} + +// 用户输入解析结果类 +public class UserInputParseResult +{ + public string Text { get; set; } = string.Empty; + public List Entities { get; set; } = []; + public List Tokens { get; set; } = []; +} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/TextExtractor.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/TextExtractor.cs deleted file mode 100644 index 6c35810..0000000 --- a/backend/src/UniversalAdminSystem.Infrastructure/Services/TextExtractor.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace UniversalAdminSystem.Infrastructure.Services; - -public class TextExtractor -{ - -} \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj b/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj index 9a5269a..b2eab81 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj +++ b/backend/src/UniversalAdminSystem.Infrastructure/UniversalAdminSystem.Infrastructure.csproj @@ -13,7 +13,9 @@ + + -- Gitee From ec42080ac7e70fd8a4e3dadc675f10f725b43ff1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AE=97=E6=97=AD?= <1619917346@qq.com> Date: Wed, 13 Aug 2025 20:23:25 +0800 Subject: [PATCH 4/6] =?UTF-8?q?fix:=E4=BF=AE=E5=A4=8D=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E6=A0=BC=E5=BC=8F=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Service/AIQusetionsAppService.cs | 34 ++++++++++---- .../Interfaces/IFileValidationService.cs | 2 +- .../FileStorage/Services/FileAppService.cs | 2 +- .../FileStorage/FileValidationService.cs | 45 ++++++++++++++----- .../Consumers/FileProcessingJobConsumer.cs | 17 +++++++ .../Publishers/FileProcessingJobPublisher.cs | 1 + .../Services/AnswerService.cs | 4 ++ .../Services/K2ModelService.cs | 31 +++---------- 8 files changed, 90 insertions(+), 46 deletions(-) diff --git a/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs b/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs index fa932ae..dbebcef 100644 --- a/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs +++ b/backend/src/UniversalAdminSystem.Application/AIQuestions/Service/AIQusetionsAppService.cs @@ -100,16 +100,34 @@ public class AIQusetionsAppService : IAIQusetionsAppService public async Task Chat(Guid conversationId, string userInput) { - // 开启事务 - await _work.BeginTransactionAsync(); + try + { + // 开启事务 + await _work.BeginTransactionAsync(); - // 创建消息并持久化 - var message = Message.Create(ConversationId.Create(conversationId), "user", userInput); - await _messageRepo.AddAsync(message); + // 创建消息并持久化 + var message = Message.Create(ConversationId.Create(conversationId), "user", userInput); + await _messageRepo.AddAsync(message); + + await _work.CommitAsync(); + // 回答 + await _work.BeginTransactionAsync(); + var messages = await _messageRepo.GetByConversationIdAsync(conversationId); + System.Console.WriteLine("消息列表:",messages); + var temp = await _answerService.AnswerAsync(userInput, messages); + System.Console.WriteLine(temp); + var Systemmessage = Message.Create(ConversationId.Create(conversationId), "system", temp); + await _messageRepo.AddAsync(Systemmessage); + await _work.CommitAsync(); + return temp; + } + catch (System.Exception e) + { + await _work.RollbackAsync(); + throw new Exception(e.Message); + } + - // 回答 - var messages = await _messageRepo.GetByConversationIdAsync(conversationId); - return await _answerService.AnswerAsync(userInput, messages); } public Task UserAccess(Guid id, ContentDto content) diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileValidationService.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileValidationService.cs index 95d65cb..a871dba 100644 --- a/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileValidationService.cs +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/Interfaces/IFileValidationService.cs @@ -2,5 +2,5 @@ namespace UniversalAdminSystem.Application.FileStorage.Interfaces; public interface IFileValidationService { - (bool isValid, string message, string? format) ValidateFile(long fileSize, Stream stream); + (bool isValid, string message, string? format) ValidateFile(string fileName, long fileSize, Stream stream); } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs b/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs index 5f81df6..84d144a 100644 --- a/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs +++ b/backend/src/UniversalAdminSystem.Application/FileStorage/Services/FileAppService.cs @@ -32,7 +32,7 @@ public class FileAppService : IFileAppService try { using var fileStream = file.OpenReadStream(); - var (isValid, message, format) = _fileValidator.ValidateFile(file.Length, fileStream); + var (isValid, message, format) = _fileValidator.ValidateFile(file.FileName, file.Length, fileStream); if (!isValid) throw new Exception($"校验结果: {message}"); var fileInfo = await _localfileStorage.SaveAsync(file); Console.WriteLine("开启事务------------------"); diff --git a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/FileValidationService.cs b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/FileValidationService.cs index 13ef65d..621d7a3 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/FileValidationService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/FileStorage/FileValidationService.cs @@ -19,7 +19,7 @@ public class FileValidationService : IFileValidationService _logger = logger; } - public (bool isValid, string message, string? format) ValidateFile(long fileSize, Stream stream) + public (bool isValid, string message, string? format) ValidateFile(string fileName, long fileSize, Stream stream) { try { @@ -32,29 +32,49 @@ public class FileValidationService : IFileValidationService // 检查流状态 Console.WriteLine($"流状态 - CanRead: {stream.CanRead}, CanSeek: {stream.CanSeek}, Length: {stream.Length}"); - // 检查MIME类型是否匹配 + // 允许的无签名类型映射 + var noSignatureMimeMap = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { ".txt", new[] { "text/plain" } }, + { ".md", new[] { "text/markdown" } }, + { ".json", new[] { "application/json" } }, + { ".xml", new[] { "application/xml", "text/xml" } } + }; + + // 用 FileSignatures 检测文件签名 Console.WriteLine("开始检查文件格式..."); var fileFormat = _fileFormatInspector.DetermineFileFormat(stream); Console.WriteLine($"文件格式检测结果: {fileFormat?.MediaType ?? "未知"}"); - Console.WriteLine($"允许的文件类型数量: {_allowedFiles.AllowedFiles?.Count() ?? 0}"); - if (_allowedFiles.AllowedFiles?.FirstOrDefault(f => f.Mime == fileFormat?.MediaType) == null) + Console.WriteLine($"_allowedFiles 是否为空: {_allowedFiles == null}"); + Console.WriteLine($"_allowedFiles.AllowedFiles 是否为空: {_allowedFiles.AllowedFiles == null}"); + Console.WriteLine($"fileFormat.MediaType 是否为空: {string.IsNullOrEmpty(fileFormat?.MediaType)}"); + + + // 有签名 + if (!string.IsNullOrEmpty(fileFormat?.MediaType) && _allowedFiles.AllowedFiles?.Any(f => f.Mime == fileFormat.MediaType) == true) { - return (false, $"❌ 不允许的文件类型: {fileFormat?.MediaType ?? "未知"}", fileFormat?.MediaType); + ResetStream(stream); + return (true, $"✅ 文件校验通过: {fileFormat.MediaType}", fileFormat.MediaType); } - // 重置流位置，以便后续使用 - if (stream.CanSeek) + Console.WriteLine($"✅ 文件名称+扩展名：{fileName}"); + // 无签名（比如 txt/md/json/xml），走扩展名映射 + var ext = Path.GetExtension(fileName); + if (!string.IsNullOrEmpty(ext) && noSignatureMimeMap.TryGetValue(ext, out var mimeList)) { - stream.Position = 0; + if (mimeList.Any(mime => _allowedFiles.AllowedFiles?.Any(f => f.Mime == mime) == true)) + { + ResetStream(stream); + return (true, $"✅ 文件校验通过: {string.Join(", ", mimeList)}", mimeList.First()); + } } - return (true, $"✅ 文件校验通过: {fileFormat?.MediaType}", fileFormat?.MediaType); + return (false, $"❌ 不允许的文件类型: {fileFormat?.MediaType ?? "未知"}", fileFormat?.MediaType); } catch (Exception ex) { _logger.LogError(ex, "文件验证过程中发生异常"); - Console.WriteLine($"文件验证异常: {ex.Message}"); return (false, $"❌ 文件验证失败: {ex.Message}", string.Empty); } } @@ -71,6 +91,11 @@ public class FileValidationService : IFileValidationService } return $"{len:0.##} {sizes[order]}"; } + private void ResetStream(Stream stream) + { + if (stream.CanSeek) + stream.Position = 0; + } // public static bool IsPlainText(Stream stream, int maxBytes = 512) // { diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs index 17cacda..06e4fe1 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs @@ -5,8 +5,11 @@ using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; +using UniversalAdminSystem.Domian.Core.ValueObjects; +using UniversalAdminSystem.Domian.FileStorage.ValueObjects; using UniversalAdminSystem.Domian.knowledge.Aggregates; using UniversalAdminSystem.Domian.knowledge.IRepository; +using UniversalAdminSystem.Domian.knowledge.ValueObj; using UniversalAdminSystem.Infrastructure.FileStorage.Parsers; using UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; using UniversalAdminSystem.Infrastructure.RabbitMQ.Models.Messages; @@ -129,6 +132,20 @@ public class FileProcessingJobConsumer : BackgroundService // Vector Store _logger.LogInformation("开始Vector Store文件: {FilePath}", fpm.FilePath); var docChunks = new List(); + for (int i = 0; i < chunks.Count; i++) + { + var c = chunks[i]; + var e = embeddings[i]; + + var docChunk = DocumentChunk.CreateDocumentChunk( + ChunkId.Create(Guid.NewGuid()), + fpm.FileId, + c, + TextEmbedding.Create(e) + ); + + docChunks.Add(docChunk); + } await docChunkRepo.BulkAddDocumentChunkAsync(docChunks); // 文件处理完成 diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs index d9bde02..66a76ec 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs @@ -42,6 +42,7 @@ public class FileProcessingJobPublisher : BackgroundService var message = new FileProcessingMessage { JobId = j.Id, + FileId = j.FileId, FilePath = j.FilePath, ContentType = j.ContentType, Size = j.Size, diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs index f6685df..1a5745a 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Services/AnswerService.cs @@ -76,15 +76,19 @@ public class AnswerService : IAnswerService List chatMessages; if (messages != null && messages.Any()) { + System.Console.WriteLine("列表为不为空"); + System.Console.WriteLine($"输入消息列表为:{messages}"); chatMessages = messages.Select(m => new K2Message { Role = m.Role, Content = m.Content }).ToList(); + System.Console.WriteLine($"输出消息列表为:{chatMessages}"); chatMessages.Insert(0, new K2Message { Role = "system", Content = systemPrompt }); } else { + System.Console.WriteLine("列表为为空"); chatMessages = new List { new K2Message { Role = "system", Content = systemPrompt }, diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs b/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs index ff210c6..7422ee8 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Services/K2ModelService.cs @@ -131,8 +131,8 @@ public partial class K2ModelService { Model = model, Messages = { - new K2Message { Role = "system", Content = "你是一个文本分片助手，任务是将长文本分割成合适大小的块（chunk），每个 chunk 不超过 500 个 token。在分片时需要遵守以下规则：1. 尽量在句子边界进行切分。2. 不要将命名实体（例如人名、地名、作品名等）拆分到不同 chunk 中。3. 尽可能保证每个 chunk 的语义完整。4. 如果文本中有无标点长句，可以在合适位置手动加切分点，但要保证语义不被破坏。" }, - new K2Message { Role = "user", Content = $"以下是文本的预处理结果，请进行精准分片：\n{preprocessedJson}, 以json格式返回结果，每个分片包含chunk_id、content、token_count字段" }, + new K2Message { Role = "system", Content = "你是一个文本分片专家，任务是将文本分割成合适大小的块（chunk），每个 chunk 不超过 500 个 token。在分片时需要遵守以下规则：1. 尽量在句子边界进行切分。2. 不要将命名实体（例如人名、地名、作品名等）拆分到不同 chunk 中。3. 尽可能保证每个 chunk 的语义完整。4. 如果文本中有无标点长句，可以在合适位置手动加切分点，但要保证语义不被破坏。" }, + new K2Message { Role = "user", Content = $"帮我完成精准的分片，以下是文本的预处理结果：\n{preprocessedJson}。\n必须以Json格式返回结果，每个分片包含chunk_id、content、token_count字段" }, }, Temperature = temperature, MaxTokens = maxTokens, @@ -156,34 +156,13 @@ public partial class K2ModelService var sb = new StringBuilder(); await foreach (var chunk in ParseStreamingResponseAsync(responseStream, sb)) { } - return ParseChunksFromLLMAnswer(sb.ToString()); + return ParseChunk(sb.ToString()); } - // private List ParseChunksFromLLMAnswer(string answer) - // { - // var chunks = new List(); - // using (JsonDocument jsonDoc = JsonDocument.Parse(answer)) - // { - // foreach (var chunk in jsonDoc.RootElement.EnumerateArray()) - // { - // if (chunk.TryGetProperty("content", out var jsonContent)) - // { - // var content = jsonContent.GetString(); - // if (content == null) continue; - // _logger.LogInformation("文本分片: {content} ", content); - // chunks.Add(content); - // } - // } - // } - - // if (chunks.Count > 0) return chunks; - // return []; - // } - - private List ParseChunksFromLLMAnswer(string answer) + public List ParseChunk(string chunkJson) { var chunks = new List(); - using var jsonDoc = JsonDocument.Parse(answer); + using var jsonDoc = JsonDocument.Parse(chunkJson); JsonElement root = jsonDoc.RootElement; -- Gitee From 8b09662e7ed5ad6207937d4548028fc19be86a7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AE=97=E6=97=AD?= <1619917346@qq.com> Date: Wed, 13 Aug 2025 20:25:56 +0800 Subject: [PATCH 5/6] =?UTF-8?q?=E4=BD=BF=E7=94=A8FlaskAPI=E5=B0=81?= =?UTF-8?q?=E8=A3=85Spacy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- FlaskAPI/app.py | 67 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 FlaskAPI/app.py diff --git a/FlaskAPI/app.py b/FlaskAPI/app.py new file mode 100644 index 0000000..31acf95 --- /dev/null +++ b/FlaskAPI/app.py @@ -0,0 +1,67 @@ +import spacy +import json +from flask import Flask, jsonify, request, Response + +# 加载 SpaCy 中文模型 +nlp = spacy.load("zh_core_web_sm") + +app = Flask(__name__) + +def preprocess_text(text): + # 使用 SpaCy 进行命名实体识别 + doc = nlp(text) + + # 识别实体 + entities = [{"Text": ent.text, "Label": ent.label_} for ent in doc.ents] + + # 按句子拆分文本 + sentences = [sent.text for sent in doc.sents] + + # 返回预处理后的数据 + result = { + "Entities": list(entities), # 确保将实体列表转换为标准列表 + "Sentences": sentences + } + + return result + +@app.route('/spacy', methods=['POST']) +def preprocess(): + data = request.get_json() + text = data.get("text", "") + + # 进行预处理 + processed_result = preprocess_text(text) + print(f"Entities: {processed_result.values()}") + + # 返回预处理后的数据 + res = Response(json.dumps(processed_result, ensure_ascii=False), content_type="application/json; charset=utf-8") + + return res + +@app.route('/parse_input', methods=['POST']) +def parse_text(): + # 从请求中获取用户输入 + content = request.json.get('content', '') + + if not content: + return jsonify({"error": "Content is required"}), 400 + + # 使用 spaCy 进行文本处理 + doc = nlp(content) + + # 提取实体和关键词 + entities = [{"text": ent.text, "label": ent.label_} for ent in doc.ents] + tokens = [token.text for token in doc if not token.is_stop] # 去除停用词 + + # 构造响应 + response = { + "entities": entities, + "tokens": tokens, + "text": content + } + + return jsonify(response) + +if __name__ == '__main__': + app.run(debug=True) \ No newline at end of file -- Gitee From a24bd136a21b48b6b7f8af9cb98dd6788cb3c48a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=88=98=E5=AE=97=E6=97=AD?= <1619917346@qq.com> Date: Thu, 14 Aug 2025 08:31:05 +0800 Subject: [PATCH 6/6] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E9=98=9F=E5=88=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../UniversalAdminSystem.Api/appsettings.json | 2 +- .../IRepository/IDocumentChunkRepository.cs | 1 + .../AddInfrastructureService.cs | 18 +- ...r\357\274\2101024\357\274\211.Designer.cs" | 435 ++++++++++++++++++ ...630_vector\357\274\2101024\357\274\211.cs" | 50 ++ ...357\274\2101024\357\274\211_2.Designer.cs" | 433 +++++++++++++++++ ...8_vector\357\274\2101024\357\274\211_2.cs" | 51 ++ ...versalAdminSystemDbContextModelSnapshot.cs | 4 +- .../UniversalAdminSystemDbContext.cs | 4 +- .../Repositories/DocumentChunkRepository.cs | 8 +- .../Consumers/FileProcessingJobConsumer.cs | 157 +++++-- .../Publishers/FileProcessingJobPublisher.cs | 2 +- .../RabbitMQ/RabbitMqEventBus.cs | 21 +- .../RabbitMQ/RabbitMqTopologyInitializer.cs | 37 ++ 14 files changed, 1145 insertions(+), 78 deletions(-) create mode 100644 "backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.Designer.cs" create mode 100644 "backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.cs" create mode 100644 "backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.Designer.cs" create mode 100644 "backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.cs" create mode 100644 backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqTopologyInitializer.cs diff --git a/backend/src/UniversalAdminSystem.Api/appsettings.json b/backend/src/UniversalAdminSystem.Api/appsettings.json index 8071a8e..070b4fd 100644 --- a/backend/src/UniversalAdminSystem.Api/appsettings.json +++ b/backend/src/UniversalAdminSystem.Api/appsettings.json @@ -9,7 +9,7 @@ "AllowedHosts": "*", "ConnectionStrings": { "pgSql": "Server=localhost;Port=5432;Username=admin;Password=031028@yue;Database=rag_vector_db", - "RabbitMq": "amqp://admin:031028%40yue@localhost:5672/" + "RabbitMq": "amqp://admin:admin@localhost:5672/" }, "Jwt": { "Key": "YourSuperSecretKey1232347509872093oiqewupori", diff --git a/backend/src/UniversalAdminSystem.Domian/knowledge/IRepository/IDocumentChunkRepository.cs b/backend/src/UniversalAdminSystem.Domian/knowledge/IRepository/IDocumentChunkRepository.cs index 5258578..3361790 100644 --- a/backend/src/UniversalAdminSystem.Domian/knowledge/IRepository/IDocumentChunkRepository.cs +++ b/backend/src/UniversalAdminSystem.Domian/knowledge/IRepository/IDocumentChunkRepository.cs @@ -37,4 +37,5 @@ public interface IDocumentChunkRepository : IRepository /// /// Task BulkAddDocumentChunkAsync(IEnumerable chunks); + Task ExistsByFileIdAsync(Guid fileId); } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs b/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs index 58842c8..18f400e 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/DependencyInject/AddInfrastructureService.cs @@ -192,25 +192,13 @@ public static class AddInfrastrutureService { var factory = new ConnectionFactory { - Uri = new Uri(sp.GetRequiredService().GetConnectionString("RabbitMq") ?? throw new InvalidOperationException("Missing RabbitMq connection string")), + Uri = new Uri(sp.GetRequiredService().GetConnectionString("RabbitMq") + ?? throw new InvalidOperationException("Missing RabbitMq connection string")), DispatchConsumersAsync = true }; return factory.CreateConnection(); }); - services.AddTransient(sp => - { - var conn = sp.GetRequiredService(); - var ch = conn.CreateModel(); - ch.ConfirmSelect(); - var exchange = configuration["RabbitMq:Exchange"] ?? "file-processing"; - var queue = configuration["RabbitMq:Queue"] ?? "file-processing-queue"; - var routingKey = configuration["RabbitMq:RoutingKey"] ?? "file-processing"; - - ch.ExchangeDeclare(exchange, "topic", durable: true, autoDelete: false); - ch.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false, arguments: null); - ch.QueueBind(queue, exchange, routingKey); - return ch; - }); + services.AddHostedService(); services.AddSingleton(); services.AddHostedService(); services.AddHostedService(); diff --git "a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.Designer.cs" "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.Designer.cs" new file mode 100644 index 0000000..cf937a1 --- /dev/null +++ "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.Designer.cs" @@ -0,0 +1,435 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Pgvector; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + [DbContext(typeof(UniversalAdminSystemDbContext))] + [Migration("20250813131630_vector（1024）")] + partial class vector1024 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "vector"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("RolePermissions", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("PermissionId") + .HasColumnType("uuid"); + + b.HasKey("RoleId", "PermissionId"); + + b.HasIndex("PermissionId"); + + b.HasIndex("RoleId"); + + b.ToTable("RolePermissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.FileStorage.Aggregates.File", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AccessLevel") + .HasColumnType("integer"); + + b.Property("IsFolder") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OwnerId") + .HasColumnType("uuid"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.Property("SecurityCheckResult") + .HasColumnType("text"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UploadTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.LogManagement.Aggregates.LogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Context") + .HasColumnType("text"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("LogEntries"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", b => + { + b.Property("PermissionId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PermissionType") + .HasColumnType("integer"); + + b.Property("Resource") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("PermissionId"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("Permissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSupper") + .HasColumnType("boolean"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("RoleId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.SystemSettings.Aggregates.SystemSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Group") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("SystemSettings"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Conversations", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Conversations"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Message", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Messages"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("Account") + .IsRequired() + .HasColumnType("text"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.Property("Password") + .IsRequired() + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.HasKey("UserId"); + + b.HasIndex("Account") + .IsUnique(); + + b.HasIndex("RoleId"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Entities.UserInfo", b => + { + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.Property("Address") + .HasColumnType("text"); + + b.Property("Age") + .HasColumnType("integer"); + + b.Property("Gender") + .HasColumnType("integer"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Phone") + .HasColumnType("text"); + + b.HasKey("UserInfoId"); + + b.ToTable("UserInfos"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.knowledge.Aggregates.DocumentChunk", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("Embedding") + .IsRequired() + .HasColumnType("vector(1024)"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.HasIndex("Embedding"); + + b.ToTable("Chunks"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs.FileProcessingJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ContentType") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("Status", "NextAttemptAt", "CreatedAt"); + + b.ToTable("FileProcessingJobs"); + }); + + modelBuilder.Entity("RolePermissions", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", null) + .WithMany() + .HasForeignKey("PermissionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.SetNull); + }); +#pragma warning restore 612, 618 + } + } +} diff --git "a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.cs" "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.cs" new file mode 100644 index 0000000..e8761a9 --- /dev/null +++ "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813131630_vector\357\274\2101024\357\274\211.cs" @@ -0,0 +1,50 @@ +using Microsoft.EntityFrameworkCore.Migrations; +using Pgvector; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + public partial class vector1024 : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + // 修改列类型为 vector(1024) + migrationBuilder.AlterColumn( + name: "Embedding", + table: "Chunks", + type: "vector(1024)", + nullable: false, + oldClrType: typeof(Vector), + oldType: "vector(1536)"); + + // 删除旧索引（如果存在） + migrationBuilder.Sql(@"DROP INDEX IF EXISTS ""IX_Chunks_Embedding_Vector"";"); + + // 建立向量索引（pgvector ivfflat） + migrationBuilder.Sql(@" + CREATE INDEX ""IX_Chunks_Embedding_Vector"" + ON ""Chunks"" + USING ivfflat (""Embedding"" vector_l2_ops) WITH (lists = 100); + "); + + // 可选：分析表统计信息 + migrationBuilder.Sql(@"ANALYZE ""Chunks"";"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + // 删除向量索引 + migrationBuilder.Sql(@"DROP INDEX IF EXISTS ""IX_Chunks_Embedding_Vector"";"); + + // 恢复列类型为 vector(1536) + migrationBuilder.AlterColumn( + name: "Embedding", + table: "Chunks", + type: "vector(1536)", + nullable: false, + oldClrType: typeof(Vector), + oldType: "vector(1024)"); + } + } +} \ No newline at end of file diff --git "a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.Designer.cs" "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.Designer.cs" new file mode 100644 index 0000000..812f8ee --- /dev/null +++ "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.Designer.cs" @@ -0,0 +1,433 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using Npgsql.EntityFrameworkCore.PostgreSQL.Metadata; +using Pgvector; +using UniversalAdminSystem.Infrastructure.Persistence.DbContexts; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + [DbContext(typeof(UniversalAdminSystemDbContext))] + [Migration("20250813133848_vector（1024）_2")] + partial class vector1024_2 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "8.0.0") + .HasAnnotation("Relational:MaxIdentifierLength", 63); + + NpgsqlModelBuilderExtensions.HasPostgresExtension(modelBuilder, "vector"); + NpgsqlModelBuilderExtensions.UseIdentityByDefaultColumns(modelBuilder); + + modelBuilder.Entity("RolePermissions", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("PermissionId") + .HasColumnType("uuid"); + + b.HasKey("RoleId", "PermissionId"); + + b.HasIndex("PermissionId"); + + b.HasIndex("RoleId"); + + b.ToTable("RolePermissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.FileStorage.Aggregates.File", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("AccessLevel") + .HasColumnType("integer"); + + b.Property("IsFolder") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("OwnerId") + .HasColumnType("uuid"); + + b.Property("ParentId") + .HasColumnType("uuid"); + + b.Property("Path") + .IsRequired() + .HasColumnType("text"); + + b.Property("SecurityCheckResult") + .HasColumnType("text"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Type") + .IsRequired() + .HasColumnType("text"); + + b.Property("UploadTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.LogManagement.Aggregates.LogEntry", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Context") + .HasColumnType("text"); + + b.Property("Exception") + .HasColumnType("text"); + + b.Property("Level") + .IsRequired() + .HasColumnType("text"); + + b.Property("Message") + .IsRequired() + .HasColumnType("text"); + + b.Property("Source") + .IsRequired() + .HasColumnType("text"); + + b.Property("Timestamp") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("LogEntries"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", b => + { + b.Property("PermissionId") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("Action") + .HasColumnType("integer"); + + b.Property("Code") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("PermissionType") + .HasColumnType("integer"); + + b.Property("Resource") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("PermissionId"); + + b.HasIndex("Code") + .IsUnique(); + + b.ToTable("Permissions"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", b => + { + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("IsSupper") + .HasColumnType("boolean"); + + b.Property("IsSystem") + .HasColumnType("boolean"); + + b.Property("Name") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.HasKey("RoleId"); + + b.HasIndex("Name") + .IsUnique(); + + b.ToTable("Roles"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.SystemSettings.Aggregates.SystemSetting", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("CreateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Description") + .HasColumnType("text"); + + b.Property("Group") + .HasColumnType("text"); + + b.Property("Key") + .IsRequired() + .HasColumnType("text"); + + b.Property("UpdateTime") + .HasColumnType("timestamp with time zone"); + + b.Property("Value") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("SystemSettings"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Conversations", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Title") + .HasColumnType("text"); + + b.Property("UpdateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("UserId") + .HasColumnType("uuid"); + + b.HasKey("Id"); + + b.ToTable("Conversations"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserConversations.Aggregates.Message", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("ConversationId") + .HasColumnType("uuid"); + + b.Property("CreateDate") + .HasColumnType("timestamp with time zone"); + + b.Property("Role") + .IsRequired() + .HasColumnType("text"); + + b.HasKey("Id"); + + b.ToTable("Messages"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.Property("UserId") + .HasColumnType("uuid"); + + b.Property("Account") + .IsRequired() + .HasColumnType("text"); + + b.Property("Email") + .IsRequired() + .HasColumnType("text"); + + b.Property("Password") + .IsRequired() + .HasColumnType("text"); + + b.Property("RoleId") + .HasColumnType("uuid"); + + b.Property("Salt") + .IsRequired() + .HasColumnType("text"); + + b.Property("Status") + .HasColumnType("integer"); + + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.HasKey("UserId"); + + b.HasIndex("Account") + .IsUnique(); + + b.HasIndex("RoleId"); + + b.ToTable("Users"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Entities.UserInfo", b => + { + b.Property("UserInfoId") + .HasColumnType("uuid"); + + b.Property("Address") + .HasColumnType("text"); + + b.Property("Age") + .HasColumnType("integer"); + + b.Property("Gender") + .HasColumnType("integer"); + + b.Property("Name") + .HasColumnType("text"); + + b.Property("Phone") + .HasColumnType("text"); + + b.HasKey("UserInfoId"); + + b.ToTable("UserInfos"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.knowledge.Aggregates.DocumentChunk", b => + { + b.Property("Id") + .HasColumnType("uuid"); + + b.Property("Content") + .IsRequired() + .HasColumnType("text"); + + b.Property("Embedding") + .IsRequired() + .HasColumnType("vector(1024)"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("Level") + .HasColumnType("integer"); + + b.HasKey("Id"); + + b.ToTable("Chunks"); + }); + + modelBuilder.Entity("UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs.FileProcessingJob", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uuid"); + + b.Property("ContentType") + .IsRequired() + .HasColumnType("text"); + + b.Property("CreatedAt") + .HasColumnType("timestamp with time zone"); + + b.Property("FileId") + .HasColumnType("uuid"); + + b.Property("FilePath") + .IsRequired() + .HasColumnType("text"); + + b.Property("NextAttemptAt") + .HasColumnType("timestamp with time zone"); + + b.Property("RetryCount") + .HasColumnType("integer"); + + b.Property("Size") + .HasColumnType("bigint"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(32) + .HasColumnType("character varying(32)"); + + b.HasKey("Id"); + + b.HasIndex("Status", "NextAttemptAt", "CreatedAt"); + + b.ToTable("FileProcessingJobs"); + }); + + modelBuilder.Entity("RolePermissions", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Permission", null) + .WithMany() + .HasForeignKey("PermissionId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("UniversalAdminSystem.Domian.UserManagement.Aggregates.User", b => + { + b.HasOne("UniversalAdminSystem.Domian.PermissionManagement.Aggregate.Role", null) + .WithMany() + .HasForeignKey("RoleId") + .OnDelete(DeleteBehavior.SetNull); + }); +#pragma warning restore 612, 618 + } + } +} diff --git "a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.cs" "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.cs" new file mode 100644 index 0000000..88b3f4a --- /dev/null +++ "b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/20250813133848_vector\357\274\2101024\357\274\211_2.cs" @@ -0,0 +1,51 @@ +using Pgvector; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace UniversalAdminSystem.Infrastructure.Migrations +{ + /// + public partial class vector1024_2 : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + // 修改列类型为 vector(1024) + migrationBuilder.AlterColumn( + name: "Embedding", + table: "Chunks", + type: "vector(1024)", + nullable: false, + oldClrType: typeof(Vector), + oldType: "vector(1536)"); + + // 删除旧索引（如果存在） + migrationBuilder.Sql(@"DROP INDEX IF EXISTS ""IX_Chunks_Embedding_Vector"";"); + + // 建立向量索引（pgvector ivfflat） + migrationBuilder.Sql(@" + CREATE INDEX ""IX_Chunks_Embedding_Vector"" + ON ""Chunks"" + USING ivfflat (""Embedding"" vector_l2_ops) WITH (lists = 100); + "); + + // 可选：分析表统计信息 + migrationBuilder.Sql(@"ANALYZE ""Chunks"";"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + // 删除向量索引 + migrationBuilder.Sql(@"DROP INDEX IF EXISTS ""IX_Chunks_Embedding_Vector"";"); + + // 恢复列类型为 vector(1536) + migrationBuilder.AlterColumn( + name: "Embedding", + table: "Chunks", + type: "vector(1536)", + nullable: false, + oldClrType: typeof(Vector), + oldType: "vector(1024)"); + } + } +} diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs index 2900335..fe4f5d2 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Migrations/UniversalAdminSystemDbContextModelSnapshot.cs @@ -348,7 +348,7 @@ namespace UniversalAdminSystem.Infrastructure.Migrations b.Property("Embedding") .IsRequired() - .HasColumnType("vector(1536)"); + .HasColumnType("vector(1024)"); b.Property("FileId") .HasColumnType("uuid"); @@ -358,8 +358,6 @@ namespace UniversalAdminSystem.Infrastructure.Migrations b.HasKey("Id"); - b.HasIndex("Embedding"); - b.ToTable("Chunks"); }); diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs index 5f21cee..c42369c 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/DbContexts/UniversalAdminSystemDbContext.cs @@ -238,7 +238,7 @@ public class UniversalAdminSystemDbContext : DbContext modelBuilder.Entity(entity => { entity.HasKey(d => d.Id); - entity.HasIndex(d => d.Embedding); + entity.Property(d => d.FileId) .HasConversion( @@ -251,7 +251,7 @@ public class UniversalAdminSystemDbContext : DbContext value => ChunkId.Create(value)); entity.Property(d => d.Embedding) - .HasColumnType("vector(1536)") + .HasColumnType("vector(1024)") .HasConversion( embedding => new Vector(embedding.Value), value => TextEmbedding.Create(value.ToArray()) diff --git a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/Repositories/DocumentChunkRepository.cs b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/Repositories/DocumentChunkRepository.cs index 16e13e5..b87ca1f 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/Persistence/Repositories/DocumentChunkRepository.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/Persistence/Repositories/DocumentChunkRepository.cs @@ -46,5 +46,11 @@ public class DocumentChunkRepository : BaseRepository, IDocumentC return results; } - + public async Task ExistsByFileIdAsync(Guid fileId) + { + var fileid = FileId.Create(fileId); + return await _TDs + .AsNoTracking() + .AnyAsync(x => x.FileId == fileid); + } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs index 06e4fe1..ac40e67 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Consumers/FileProcessingJobConsumer.cs @@ -6,27 +6,28 @@ using Microsoft.Extensions.Logging; using RabbitMQ.Client; using RabbitMQ.Client.Events; using UniversalAdminSystem.Domian.Core.ValueObjects; -using UniversalAdminSystem.Domian.FileStorage.ValueObjects; using UniversalAdminSystem.Domian.knowledge.Aggregates; using UniversalAdminSystem.Domian.knowledge.IRepository; using UniversalAdminSystem.Domian.knowledge.ValueObj; using UniversalAdminSystem.Infrastructure.FileStorage.Parsers; -using UniversalAdminSystem.Infrastructure.RabbitMQ.Jobs; using UniversalAdminSystem.Infrastructure.RabbitMQ.Models.Messages; using UniversalAdminSystem.Infrastructure.Services; +using UniversalAdminSystem.Application.Common.Interfaces; + namespace UniversalAdminSystem.Infrastructure.RabbitMQ.Consumers; public class FileProcessingJobConsumer : BackgroundService { - private readonly IModel _ch; + private readonly IConnection _conn; private readonly ILogger _logger; private readonly IServiceScopeFactory _scopeFactory; + private readonly int _consumerCount = 5; - public FileProcessingJobConsumer(IModel ch, ILogger logger, IServiceScopeFactory scopeFactory) + public FileProcessingJobConsumer(IConnection conn, ILogger logger, IServiceScopeFactory scopeFactory) { - _ch = ch; + _conn = conn; _logger = logger; _scopeFactory = scopeFactory; } @@ -48,86 +49,141 @@ public class FileProcessingJobConsumer : BackgroundService private async Task AssignConsumerAsync(CancellationToken stoppingToken, int consumerIndex) { - _logger.LogInformation($"消费者{consumerIndex}正在启动并等待消息..."); - var consumer = new AsyncEventingBasicConsumer(_ch); - _logger.LogInformation("正在订阅队列: file-processing-queue"); - consumer.Received += async (sender, e) => + var exchange = "file-processing"; + var queue = "file-processing-queue"; + var routingKey = "file-processing"; + + using var ch = _conn.CreateModel(); + ch.ExchangeDeclare(exchange, "topic", durable: true, autoDelete: false); + ch.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false, arguments: null); + ch.QueueBind(queue, exchange, routingKey); + ch.BasicQos(0, 1, false); + + _logger.LogInformation("消费者{consumerIndex}正在启动并等待消息...", consumerIndex); + + var consumer = new AsyncEventingBasicConsumer(ch); + consumer.Received += async (sender, ea) => { - var body = e.Body.ToArray(); - var message = Encoding.UTF8.GetString(body); - _logger.LogInformation($"消费者 {consumerIndex} 收到文件处理任务: {message}"); + var message = Encoding.UTF8.GetString(ea.Body.ToArray()); + + _logger.LogInformation("消费者 {consumerIndex} 收到文件处理任务: {message}", consumerIndex, message); + try { + _logger.LogInformation("消费者 {consumerIndex} 收到文件处理任务: {message}", consumerIndex, message); var fpm = JsonSerializer.Deserialize(message); + if (fpm == null) { - _logger.LogError("消息格式错误，跳过处理"); - _ch.BasicNack(e.DeliveryTag, false, false); + _logger.LogError("消息格式错误，跳过处理: {message}", message); + ch.BasicNack(ea.DeliveryTag, false, false); return; } + if (fpm.NextAttemptAt.HasValue && fpm.NextAttemptAt > DateTime.UtcNow) { - _logger.LogInformation($"等待重试，当前时间：{DateTime.UtcNow}，下一次尝试时间：{fpm.NextAttemptAt.Value}"); + _logger.LogInformation("等待重试，当前时间：{DateTime.UtcNow}，下一次尝试时间：{fpm.NextAttemptAt.Value}", DateTime.UtcNow, fpm.NextAttemptAt.Value); await Task.Delay(fpm.NextAttemptAt.Value - DateTime.UtcNow, stoppingToken); } - await ProcessMessageAsync(fpm, stoppingToken); - _ch.BasicAck(e.DeliveryTag, false); - _logger.LogInformation($"消费者 {consumerIndex} 文件处理任务处理成功: {message}"); + + await ProcessMessageAsync(fpm, stoppingToken, ea, ch); + + ch.BasicAck(ea.DeliveryTag, false); + _logger.LogInformation("消费者 {consumerIndex} 文件处理任务处理成功: {message}", consumerIndex, message); } catch (Exception ex) { var fpm = JsonSerializer.Deserialize(message); - if (fpm == null) return; + if (fpm == null) + { + ch.BasicNack(ea.DeliveryTag, false, requeue: false); + return; + } fpm.RetryCount++; if (fpm.RetryCount >= 3) { - _logger.LogError($"消息 {fpm.JobId} 达到最大重试次数，丢弃消息: {message}"); - _ch.BasicNack(e.DeliveryTag, false, false); + _logger.LogError("消息 {fpm.JobId} 达到最大重试次数，丢弃消息: {message}", fpm.Id, message); + ch.BasicNack(ea.DeliveryTag, false, false); return; } var retryDelay = Math.Min(300, Math.Pow(2, fpm.RetryCount)); fpm.NextAttemptAt = DateTime.UtcNow.AddSeconds(retryDelay); - var payload = JsonSerializer.Serialize(fpm); - _ch.BasicPublish("file-processing", "file-processing", null, Encoding.UTF8.GetBytes(payload)); - _logger.LogError(ex, $"消费者 {consumerIndex} 文件处理任务失败: {message}, 重试次数: {fpm.RetryCount}, 下次重试延迟: {retryDelay} 秒"); + // 先 nack 再重新发布（都在同一个 ch） + ch.BasicNack(ea.DeliveryTag, false, requeue: false); + ch.BasicPublish("file-processing", "file-processing", null, Encoding.UTF8.GetBytes(JsonSerializer.Serialize(fpm))); + + _logger.LogError(ex, "消费者 {consumerIndex} 文件处理任务失败: {message}, 重试次数: {fpm.RetryCount}, 下次重试延迟: {retryDelay} 秒", consumerIndex, message, fpm.RetryCount, retryDelay); } }; - _ch.BasicConsume(queue: "file-processing-queue", autoAck: false, consumer: consumer); - await Task.Delay(Timeout.Infinite, stoppingToken); + + var consumerTag = ch.BasicConsume(queue: queue, autoAck: false, consumer: consumer); + + try + { + // 阻塞直到停止 + while (!stoppingToken.IsCancellationRequested) + { + await Task.Delay(1000, stoppingToken); + } + } + finally + { + // 关闭 + try { ch.BasicCancel(consumerTag); } catch { } + try { ch.Close(); } catch { } + } } - private async Task ProcessMessageAsync(FileProcessingMessage fpm, CancellationToken stoppingToken) + private async Task ProcessMessageAsync(FileProcessingMessage fpm, CancellationToken stoppingToken, BasicDeliverEventArgs ea, IModel ch) { try { + if (!File.Exists(fpm.FilePath)) + { + _logger.LogWarning("文件不存在，丢弃任务: {Path}", fpm.FilePath); + ch.BasicAck(ea.DeliveryTag, false); // 确认消费，RabbitMQ 删除消息 + return; + } + // 文本提取 _logger.LogInformation("开始处理文件: {FilePath}", fpm.FilePath); var parser = new DocParserFactory().GetParser(fpm.ContentType); var text = await parser.ParseAsync(fpm.FilePath); _logger.LogInformation("文件处理完成: {FilePath}", fpm.FilePath); - using (var scope = _scopeFactory.CreateScope()) + using var scope = _scopeFactory.CreateScope(); + var spacy = scope.ServiceProvider.GetRequiredService(); + var k2 = scope.ServiceProvider.GetRequiredService(); + var embedding = scope.ServiceProvider.GetRequiredService(); + var docChunkRepo = scope.ServiceProvider.GetRequiredService(); + var work = scope.ServiceProvider.GetRequiredService(); + + // 去重 + var exists = await docChunkRepo.ExistsByFileIdAsync(fpm.FileId); + if (exists) { - var spacy = scope.ServiceProvider.GetRequiredService(); - var k2 = scope.ServiceProvider.GetRequiredService(); - var embedding = scope.ServiceProvider.GetRequiredService(); - var docChunkRepo = scope.ServiceProvider.GetRequiredService(); - - // 文本分片 - _logger.LogInformation("开始分片文件: {Text}", text); - var preprocessResult = await spacy.AnalyzeTextAsync(text); - var preprocessResultJson = JsonSerializer.Serialize(preprocessResult, new JsonSerializerOptions - { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }); - var chunks = await k2.SendChunkingRequestAsync(preprocessResultJson); - _logger.LogInformation("分片完成，生成 {Count} 个 chunk", chunks.Count); + _logger.LogInformation("文件 {fpm.FileId} 已处理，跳过", fpm.FileId); + return; + } - // Embedding - _logger.LogInformation("开始Embedding"); - var embeddings = new List(); - embeddings.AddRange(await embedding.GetEmbeddingAsync(chunks)); - _logger.LogInformation("Embedding 完成: {Count} 个向量", embeddings.Count); + // 文本分片 + _logger.LogInformation("开始分片文件: {Text}", text); + var preprocessResult = await spacy.AnalyzeTextAsync(text); + var preprocessResultJson = JsonSerializer.Serialize(preprocessResult, new JsonSerializerOptions + { + PropertyNamingPolicy = JsonNamingPolicy.CamelCase + }); + var chunks = await k2.SendChunkingRequestAsync(preprocessResultJson); + _logger.LogInformation("分片完成，生成 {Count} 个 chunk", chunks.Count); + + // Embedding + _logger.LogInformation("开始Embedding"); + var embeddings = new List(); + embeddings.AddRange(await embedding.GetEmbeddingAsync(chunks)); + _logger.LogInformation("Embedding 完成: {Count} 个向量", embeddings.Count); + try + { + await work.BeginTransactionAsync(); // Vector Store _logger.LogInformation("开始Vector Store文件: {FilePath}", fpm.FilePath); @@ -146,11 +202,18 @@ public class FileProcessingJobConsumer : BackgroundService docChunks.Add(docChunk); } + await docChunkRepo.BulkAddDocumentChunkAsync(docChunks); + await work.CommitAsync(); // 文件处理完成 _logger.LogInformation("文件处理完成: {FilePath}", fpm.FilePath); } + catch (System.Exception) + { + await work.RollbackAsync(); + throw; + } } catch (Exception ex) { diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs index 66a76ec..597597c 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/Publishers/FileProcessingJobPublisher.cs @@ -41,7 +41,7 @@ public class FileProcessingJobPublisher : BackgroundService { var message = new FileProcessingMessage { - JobId = j.Id, + Id = j.Id, FileId = j.FileId, FilePath = j.FilePath, ContentType = j.ContentType, diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs index 2065305..66cd7b9 100644 --- a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqEventBus.cs @@ -7,24 +7,29 @@ namespace UniversalAdminSystem.Infrastructure.RabbitMQ; public class RabbitMqEventBus : IEventBus, IDisposable { - private readonly IModel _ch; + private readonly IConnection _conn; - public RabbitMqEventBus(IModel ch, IConfiguration cfg) + public RabbitMqEventBus(IConnection conn, IConfiguration _) { - _ch = ch; - _ch.ConfirmSelect(); + _conn = conn; } public Task PublishAsync(string exchange, string routingKey, string payload, string? messageId = null, CancellationToken ct = default) { - var props = _ch.CreateBasicProperties(); + // 每次发布各自创建 channel，线程安全 & 生命周期正确 + using var ch = _conn.CreateModel(); + ch.ConfirmSelect(); + var props = ch.CreateBasicProperties(); props.Persistent = true; props.MessageId = messageId; props.ContentType = "application/json"; - _ch.BasicPublish(exchange, routingKey, mandatory: true, basicProperties: props, body: Encoding.UTF8.GetBytes(payload)); - _ch.WaitForConfirmsOrDie(TimeSpan.FromSeconds(5)); + var body = Encoding.UTF8.GetBytes(payload); + ch.BasicPublish(exchange, routingKey, mandatory: false, basicProperties: props, body: body); + + // publisher confirm，确保消息真正进 broker + ch.WaitForConfirmsOrDie(TimeSpan.FromSeconds(5)); return Task.CompletedTask; } - public void Dispose() { _ch?.Dispose(); } + public void Dispose() { } } \ No newline at end of file diff --git a/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqTopologyInitializer.cs b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqTopologyInitializer.cs new file mode 100644 index 0000000..04da06a --- /dev/null +++ b/backend/src/UniversalAdminSystem.Infrastructure/RabbitMQ/RabbitMqTopologyInitializer.cs @@ -0,0 +1,37 @@ +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Configuration; +using RabbitMQ.Client; + +namespace UniversalAdminSystem.Infrastructure.RabbitMQ; + +public class RabbitMqTopologyInitializer : IHostedService +{ + private readonly IConnection _conn; + private readonly IConfiguration _cfg; + private readonly ILogger _logger; + + public RabbitMqTopologyInitializer(IConnection conn, IConfiguration cfg, ILogger logger) + { + _conn = conn; _cfg = cfg; _logger = logger; + } + + public Task StartAsync(CancellationToken cancellationToken) + { + using var ch = _conn.CreateModel(); + var exchange = _cfg["RabbitMq:Exchange"] ?? "file-processing"; + var queue = _cfg["RabbitMq:Queue"] ?? "file-processing-queue"; + var routingKey = _cfg["RabbitMq:RoutingKey"] ?? "file-processing"; + + ch.ExchangeDeclare(exchange, type: "topic", durable: true, autoDelete: false); + ch.QueueDeclare(queue, durable: true, exclusive: false, autoDelete: false, arguments: null); + ch.QueueBind(queue, exchange, routingKey); + + _logger.LogInformation("RabbitMQ 拓扑已就绪: exchange={Exchange}, queue={Queue}, routingKey={RoutingKey}", + exchange, queue, routingKey); + + return Task.CompletedTask; + } + + public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; +} -- Gitee