ow
This commit is contained in:
1 parent
2e990e687f
commit
4ec2502778
53 files changed
+365
-1854
No files matched your search
@@ -1,61 +0,0 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using CloudBuilder.Core.DatabaseAccessor;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.Policy;
|
||||
using CloudBuilder.AI.SearchCriteria;
|
||||
namespace CloudBuilder.AI.Service
|
||||
{
|
||||
public class AiBookPersonService : IAiBookPersonService
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
private IRepository<AiBookPersonEntity> repository;
|
||||
public AiBookPersonService(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
repository = service.GetRepository<IRepository<AiBookPersonEntity>>();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBookPerson, operate: Operate.Find)]
|
||||
public AiBookPersonEntity FindByKey(string key)
|
||||
{
|
||||
return repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
}
|
||||
|
||||
//[PermissionOperate(name: AIPermissionPolicy.AiBookPerson, operate: Operate.Find)]
|
||||
public dynamic FindBySearchCriteria(AiBookPersonSearchCriteria criteria)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
//[PermissionOperate(name: AIPermissionPolicy.AiBookPerson, operate: Operate.Print)]
|
||||
public dynamic PrintBySearchCriteria(AiBookPersonSearchCriteria criteria)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBookPerson, operate: Operate.Insert)]
|
||||
[UnitOfWork]
|
||||
public AiBookPersonEntity Insert(AiBookPersonEntity data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBookPerson, operate: Operate.Update)]
|
||||
[UnitOfWork]
|
||||
public AiBookPersonEntity Update(AiBookPersonEntity data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBookPerson, operate: Operate.Delete)]
|
||||
[UnitOfWork]
|
||||
public void Delete(string key)
|
||||
{
|
||||
var ent= repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
if (ent != null) { repository.DeleteNow(ent); }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@ using CloudBuilder.AI.Data;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.Policy;
|
||||
using CloudBuilder.AI.SearchCriteria;
|
||||
using CloudBuilder.AI.Service.Utility;
|
||||
using CloudBuilder.AI.Utility;
|
||||
using CloudBuilder.Core.DatabaseAccessor;
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.Service;
|
||||
@@ -29,7 +29,6 @@ namespace CloudBuilder.AI.Service
|
||||
|
||||
private IRepository<AiBookEntity> repository;
|
||||
|
||||
private ChineseNameExtractor chineseNameExtractor;
|
||||
public AiBookService(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
@@ -40,7 +39,7 @@ namespace CloudBuilder.AI.Service
|
||||
public AiBookData FindByKey(string key)
|
||||
{
|
||||
AiBookData data = new AiBookData();
|
||||
data.AiBook = repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
data.AiBook = repository.DetachedEntities.Where(x => x.BookId == key).FirstOrDefault()!;
|
||||
|
||||
if (data.AiBook.FileId > 0)
|
||||
{
|
||||
@@ -129,7 +128,7 @@ namespace CloudBuilder.AI.Service
|
||||
|
||||
ISecurityGeneratorNumberService securityGeneratorNumberService = service.ServiceProvider.GetService<ISecurityGeneratorNumberService>()!;
|
||||
|
||||
data.AiBook.Guid = securityGeneratorNumberService.GetNo(AIPrefixPolicy.AI_BOOK, AIPrefixPolicy.GUID, AIPrefixPolicy.AI_BOOK_PREFIX, YesNoPolicy.NO, AIPrefixPolicy.AI_BOOK_GUID_FORMAT);
|
||||
data.AiBook.BookId = securityGeneratorNumberService.GetNo(AIPrefixPolicy.AI_BOOK, AIPrefixPolicy.GUID, AIPrefixPolicy.AI_BOOK_PREFIX, YesNoPolicy.NO, AIPrefixPolicy.AI_BOOK_GUID_FORMAT);
|
||||
|
||||
repository.InsertNow(data.AiBook);
|
||||
|
||||
@@ -144,7 +143,7 @@ namespace CloudBuilder.AI.Service
|
||||
[UnitOfWork]
|
||||
public AiBookData Update(AiBookData data)
|
||||
{
|
||||
AiBookEntity aiBook = repository.DetachedEntities.Where(x => x.Guid == data.AiBook.Guid).FirstOrDefault();
|
||||
AiBookEntity aiBook = repository.DetachedEntities.Where(x => x.BookId == data.AiBook.BookId).FirstOrDefault();
|
||||
if (aiBook == null)
|
||||
{
|
||||
throw new ValidatedException("记录不存在.");
|
||||
@@ -164,7 +163,7 @@ namespace CloudBuilder.AI.Service
|
||||
[UnitOfWork]
|
||||
public void Delete(string key)
|
||||
{
|
||||
var ent = repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
var ent = repository.DetachedEntities.Where(x => x.BookId == key).FirstOrDefault()!;
|
||||
if (ent != null) { repository.DeleteNow(ent); }
|
||||
}
|
||||
|
||||
@@ -203,110 +202,53 @@ namespace CloudBuilder.AI.Service
|
||||
|
||||
string context = File.ReadAllText(file, Encoding.GetEncoding(encoding));
|
||||
|
||||
List<AiChapterEntity> aiChapterEntities = chapterSplitter.SplitNovelIntoChapters(context, pattern);
|
||||
List<AiChapterDisplayData> aiChapterEntities = chapterSplitter.SplitNovelIntoChapters(context, pattern);
|
||||
|
||||
if (aiChapterEntities == null || aiChapterEntities.Count() == 0) return data;
|
||||
|
||||
aiChapterEntities.ToLookup(x => x.BookGuid = data.AiBook.Guid);
|
||||
aiChapterEntities.ToLookup(x => x.Status = ChapterStatusPolicy.PROCESS);
|
||||
int chapterIndex = 1;
|
||||
foreach (var chapter in aiChapterEntities)
|
||||
{
|
||||
chapter.ChapterId = chapterIndex++;
|
||||
chapter.BookId = data.AiBook.BookId;
|
||||
chapter.Status = ChapterStatusPolicy.PROCESS;
|
||||
}
|
||||
|
||||
data.AiChapters = aiChapterEntities.ToArray();
|
||||
|
||||
paragraphSplitter.PackParagraphs(data);
|
||||
|
||||
IRepository<AiChapterEntity> repositoryAiChapterEntity = repository.GetRepository<IRepository<AiChapterEntity>>();
|
||||
IRepository<AiParagraphEntity> repositoryAiParagraphEntity = repository.GetRepository<IRepository<AiParagraphEntity>>();
|
||||
|
||||
IRepository<AiSentenceEntity> repositoryAiSentenceEntity = repository.GetRepository<IRepository<AiSentenceEntity>>();
|
||||
|
||||
AiChapterEntity[] aiChapters = repositoryAiChapterEntity.DetachedEntities.Where(x => x.BookGuid == data.AiBook.Guid).ToArray();
|
||||
AiChapterEntity[] aiChapters = repositoryAiChapterEntity.DetachedEntities.Where(x => x.BookId == data.AiBook.BookId).ToArray();
|
||||
|
||||
if (aiChapters != null && aiChapters.Length > 0)
|
||||
{
|
||||
foreach (AiChapterEntity aiChapter in aiChapters)
|
||||
{
|
||||
AiParagraphEntity[] aiParagraphs = repositoryAiParagraphEntity.DetachedEntities.Where(x => x.ChapterGuid == aiChapter.Guid).ToArray();
|
||||
AiSentenceEntity[] aiSentences = repositoryAiSentenceEntity.DetachedEntities.Where(x => x.ChapterId == aiChapter.ChapterId).ToArray();
|
||||
|
||||
if (aiParagraphs != null && aiParagraphs.Length > 0)
|
||||
if (aiSentences != null && aiSentences.Length > 0)
|
||||
{
|
||||
foreach (AiParagraphEntity aiParagraph in aiParagraphs)
|
||||
if (aiSentences != null && aiSentences.Length > 0)
|
||||
{
|
||||
AiSentenceEntity[] aiSentences = repositoryAiSentenceEntity.DetachedEntities.Where(x => x.ParagraphGuid == aiParagraph.Guid).ToArray();
|
||||
|
||||
if (aiSentences != null && aiSentences.Length > 0)
|
||||
{
|
||||
repositoryAiSentenceEntity.DeleteNow(aiSentences);
|
||||
}
|
||||
repositoryAiSentenceEntity.DeleteNow(aiSentences);
|
||||
}
|
||||
|
||||
repositoryAiParagraphEntity.DeleteNow(aiParagraphs);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
repositoryAiChapterEntity.DeleteNow(aiChapters);
|
||||
}
|
||||
|
||||
repositoryAiChapterEntity.InsertNow(data.AiChapters);
|
||||
|
||||
if (data.AiParagraphs == null || data.AiParagraphs.Count() == 0) return data;
|
||||
|
||||
repositoryAiParagraphEntity.InsertNow(data.AiParagraphs);
|
||||
|
||||
if (data.AiSentences == null || data.AiSentences.Count() == 0) return data;
|
||||
repositoryAiSentenceEntity.InsertNow(data.AiSentences);
|
||||
|
||||
return FindByKey(key);
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Update)]
|
||||
[UnitOfWork]
|
||||
public void ExtractNames(string bookGuid)
|
||||
{
|
||||
ExtractNamesByChapterGuid(bookGuid, null);
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Update)]
|
||||
[UnitOfWork]
|
||||
public void ExtractNamesByChapterGuid(string bookGuid, string chapterGuid)
|
||||
{
|
||||
if (chineseNameExtractor == null) chineseNameExtractor = new ChineseNameExtractor(service);
|
||||
|
||||
IRepository<AiChapterPersonEntity> repositoryAiChapterPersonEntity = repository.GetRepository<IRepository<AiChapterPersonEntity>>();
|
||||
|
||||
IRepository<AiParagraphViewEntity> repositoryAiParagraphEntity = repository.GetRepository<IRepository<AiParagraphViewEntity>>();
|
||||
|
||||
AiParagraphViewEntity[] paragraphs = repositoryAiParagraphEntity.DetachedEntities.Where(x => x.BookGuid == bookGuid && (string.IsNullOrEmpty(chapterGuid) || x.ChapterGuid == chapterGuid)).ToArray();
|
||||
|
||||
if (paragraphs == null || paragraphs.Length == 0) return;
|
||||
|
||||
List<string> names = new List<string>();
|
||||
List<AiChapterPersonEntity> aiChapterPersonEntities = new List<AiChapterPersonEntity>();
|
||||
AiChapterPersonEntity entity;
|
||||
int orderIndex = 0;
|
||||
foreach (var paragraph in paragraphs)
|
||||
{
|
||||
names = chineseNameExtractor.ExtractNames(paragraph.Content);
|
||||
|
||||
if (names == null || names.Count() == 0) continue;
|
||||
|
||||
orderIndex = 1;
|
||||
foreach (var name in names)
|
||||
{
|
||||
entity = new AiChapterPersonEntity();
|
||||
entity.Guid = Guid.NewGuid().ToString();
|
||||
entity.ChapterGuid = paragraph.ChapterGuid;
|
||||
entity.ParagraphGuid = paragraph.Guid;
|
||||
entity.ParagraphIndex = paragraph.ParagraphIndex;
|
||||
entity.OrderIndex = orderIndex++;
|
||||
entity.PersonName = name;
|
||||
aiChapterPersonEntities.Add(entity);
|
||||
}
|
||||
}
|
||||
|
||||
if (aiChapterPersonEntities != null && aiChapterPersonEntities.Count() > 0)
|
||||
repositoryAiChapterPersonEntity.InsertNow(aiChapterPersonEntities);
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
public IActionResult GetTtsStream(string text)
|
||||
|
||||
@@ -1,61 +0,0 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using CloudBuilder.Core.DatabaseAccessor;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.Policy;
|
||||
using CloudBuilder.AI.SearchCriteria;
|
||||
namespace CloudBuilder.AI.Service
|
||||
{
|
||||
public class AiChapterPersonService : IAiChapterPersonService
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
private IRepository<AiChapterPersonEntity> repository;
|
||||
public AiChapterPersonService(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
repository = service.GetRepository<IRepository<AiChapterPersonEntity>>();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiChapterPerson, operate: Operate.Find)]
|
||||
public AiChapterPersonEntity FindByKey(string key)
|
||||
{
|
||||
return repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
}
|
||||
|
||||
//[PermissionOperate(name: AIPermissionPolicy.AiChapterPerson, operate: Operate.Find)]
|
||||
public dynamic FindBySearchCriteria(AiChapterPersonSearchCriteria criteria)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
//[PermissionOperate(name: AIPermissionPolicy.AiChapterPerson, operate: Operate.Print)]
|
||||
public dynamic PrintBySearchCriteria(AiChapterPersonSearchCriteria criteria)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiChapterPerson, operate: Operate.Insert)]
|
||||
[UnitOfWork]
|
||||
public AiChapterPersonEntity Insert(AiChapterPersonEntity data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiChapterPerson, operate: Operate.Update)]
|
||||
[UnitOfWork]
|
||||
public AiChapterPersonEntity Update(AiChapterPersonEntity data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiChapterPerson, operate: Operate.Delete)]
|
||||
[UnitOfWork]
|
||||
public void Delete(string key)
|
||||
{
|
||||
var ent= repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
if (ent != null) { repository.DeleteNow(ent); }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,20 +20,9 @@ namespace CloudBuilder.AI.Service
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
public AiChapterEntity FindByKey(string key)
|
||||
public AiChapterEntity FindByKey(AiChapterEntityKey key)
|
||||
{
|
||||
return repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
public AiChapterEntity[] FindByTitle(string title)
|
||||
{
|
||||
|
||||
IRepository<AiBookEntity> repositoryAiBookEntity = repository.GetRepository<IRepository<AiBookEntity>>();
|
||||
AiBookEntity aiBookEntity = repositoryAiBookEntity.DetachedEntities.Where(x => x.Title == title).FirstOrDefault()!;
|
||||
if (aiBookEntity == null) return null;
|
||||
|
||||
return repository.DetachedEntities.Where(x => x.BookGuid == aiBookEntity.Guid).ToArray();
|
||||
return repository.DetachedEntities.Where(x => x.ChapterId == key.ChapterId && x.BookId == key.BookId).FirstOrDefault()!;
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
@@ -73,7 +62,7 @@ namespace CloudBuilder.AI.Service
|
||||
|
||||
data.Limit = criteria.limit;
|
||||
data.Page = criteria.page;
|
||||
data.Datas = list.OrderBy(x=>x.ChapterIndex).ToArray();
|
||||
data.Datas = list.OrderBy(x => x.ChapterId).ToArray();
|
||||
data.Total = count;
|
||||
//data.Sql = sql;
|
||||
data.ErrorMessage = err;
|
||||
@@ -103,9 +92,9 @@ namespace CloudBuilder.AI.Service
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Delete)]
|
||||
[UnitOfWork]
|
||||
public void Delete(string key)
|
||||
public void Delete(AiChapterEntityKey key)
|
||||
{
|
||||
var ent = repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
var ent = repository.DetachedEntities.Where(x => x.ChapterId == key.ChapterId && x.BookId == key.BookId).FirstOrDefault()!;
|
||||
if (ent != null) { repository.DeleteNow(ent); }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using Microsoft.AspNetCore.Authorization;
|
||||
using CloudBuilder.Core.DatabaseAccessor;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.Policy;
|
||||
using CloudBuilder.AI.SearchCriteria;
|
||||
namespace CloudBuilder.AI.Service
|
||||
{
|
||||
public class AiParagraphService : IAiParagraphService
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
private IRepository<AiParagraphEntity> repository;
|
||||
public AiParagraphService(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
repository = service.GetRepository<IRepository<AiParagraphEntity>>();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Find)]
|
||||
public AiParagraphEntity FindByKey(string key)
|
||||
{
|
||||
return repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
}
|
||||
|
||||
//[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Find)]
|
||||
public dynamic FindBySearchCriteria(AiParagraphSearchCriteria criteria)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
//[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Print)]
|
||||
public dynamic PrintBySearchCriteria(AiParagraphSearchCriteria criteria)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Insert)]
|
||||
[UnitOfWork]
|
||||
public AiParagraphEntity Insert(AiParagraphEntity data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Insert)]
|
||||
[UnitOfWork]
|
||||
public void Inserts(AiParagraphEntity[] datas)
|
||||
{
|
||||
repository.InsertNow(datas);
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Update)]
|
||||
[UnitOfWork]
|
||||
public AiParagraphEntity Update(AiParagraphEntity data)
|
||||
{
|
||||
throw new NotImplementedException();
|
||||
}
|
||||
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiParagraph, operate: Operate.Delete)]
|
||||
[UnitOfWork]
|
||||
public void Delete(string key)
|
||||
{
|
||||
var ent= repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
if (ent != null) { repository.DeleteNow(ent); }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,18 +22,18 @@ namespace CloudBuilder.AI.Service
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
public AiSentenceEntity FindByKey(string key)
|
||||
public AiSentenceEntity FindByKey(long key)
|
||||
{
|
||||
return repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
return repository.DetachedEntities.Where(x => x.SentenceId == key).FirstOrDefault()!;
|
||||
}
|
||||
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
public AiSentenceViewEntity[] FindByChapterGuid(string chapterGuid)
|
||||
public AiSentenceViewEntity[] FindByChapterId(long chapterId)
|
||||
{
|
||||
IRepository<AiSentenceViewEntity> repositoryAiSentenceViewEntity = repository.GetRepository<IRepository<AiSentenceViewEntity>>();
|
||||
|
||||
return repositoryAiSentenceViewEntity.DetachedEntities.Where(x => x.ChapterGuid == chapterGuid).OrderBy(x => x.ParagraphIndex).ThenBy(x=>x.SentenceIndex).ToArray();
|
||||
return repositoryAiSentenceViewEntity.DetachedEntities.Where(x => x.ChapterId == chapterId).OrderBy(x => x.ParagraphId).ThenBy(x=>x.SentenceIndex).ToArray();
|
||||
}
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Find)]
|
||||
@@ -111,9 +111,9 @@ namespace CloudBuilder.AI.Service
|
||||
|
||||
[PermissionOperate(name: AIPermissionPolicy.AiBook, operate: Operate.Delete)]
|
||||
[UnitOfWork]
|
||||
public void Delete(string key)
|
||||
public void Delete(long key)
|
||||
{
|
||||
var ent = repository.DetachedEntities.Where(x => x.Guid == key).FirstOrDefault()!;
|
||||
var ent = repository.DetachedEntities.Where(x => x.SentenceId == key).FirstOrDefault()!;
|
||||
if (ent != null) { repository.DeleteNow(ent); }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,10 +22,6 @@
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyModel" Version="8.0.0" />
|
||||
<PackageReference Include="Microsoft.ML" Version="5.0.0" />
|
||||
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.26.0" />
|
||||
<PackageReference Include="Microsoft.ML.OnnxTransformer" Version="5.0.0" />
|
||||
<PackageReference Include="NAudio.Core" Version="2.3.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
@@ -39,6 +35,10 @@
|
||||
<HintPath>..\CloudBuilder.Security\bin\Debug\net8.0\CloudBuilder.Security.dll</HintPath>
|
||||
</Reference>
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Folder Include="Utility\" />
|
||||
</ItemGroup>
|
||||
|
||||
<Target Name="PostBuild" AfterTargets="PostBuildEvent">
|
||||
<Exec Command="D:\MAFAI\CodeGenerator\CodeGenerator\bin\Debug\net8.0-windows\CodeGenerator.exe $(TargetPath)" />
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
using CloudBuilder.AI.Entity;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.AI.Service.Utility
|
||||
{
|
||||
/// <summary>
|
||||
/// 章节分割器
|
||||
/// 不同的小说有自己的分割方法,可以传入参数
|
||||
/// </summary>
|
||||
public class ChapterSplitter
|
||||
{
|
||||
public ChapterSplitter()
|
||||
{
|
||||
|
||||
}
|
||||
/// <summary>
|
||||
/// 按章节分割小说文本
|
||||
/// </summary>
|
||||
public List<AiChapterEntity> SplitNovelIntoChapters(string novelText, string chapterPatterns)
|
||||
{
|
||||
var chapters = new List<AiChapterEntity>();
|
||||
var lines = novelText.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var currentAiChapterEntity = new StringBuilder();
|
||||
var currentAiChapterEntityTitle = "未命名章节";
|
||||
var chapterCount = 1;
|
||||
|
||||
foreach (var line in lines)
|
||||
{
|
||||
// 检查是否是章节标题
|
||||
if (IsChapterTitle(line, chapterPatterns))
|
||||
{
|
||||
// 如果当前章节有内容,则保存
|
||||
if (currentAiChapterEntity.Length > 0)
|
||||
{
|
||||
chapters.Add(new AiChapterEntity
|
||||
{
|
||||
Title = currentAiChapterEntityTitle,
|
||||
Guid = Guid.NewGuid().ToString(),
|
||||
Content = currentAiChapterEntity.ToString(),
|
||||
ChapterIndex = chapterCount
|
||||
});
|
||||
currentAiChapterEntity.Clear();
|
||||
chapterCount++;
|
||||
}
|
||||
|
||||
// 更新章节标题
|
||||
currentAiChapterEntityTitle = line.Trim();
|
||||
// 添加章节标题
|
||||
currentAiChapterEntity.AppendLine(line.Trim());
|
||||
// 添加一个空行作为分隔,确保章节标题和正文内容分离
|
||||
currentAiChapterEntity.AppendLine();
|
||||
}
|
||||
else
|
||||
{
|
||||
// 添加到当前章节,确保每行独立
|
||||
currentAiChapterEntity.AppendLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加最后一个章节
|
||||
if (currentAiChapterEntity.Length > 0)
|
||||
{
|
||||
chapters.Add(new AiChapterEntity
|
||||
{
|
||||
Title = currentAiChapterEntityTitle,
|
||||
Content = currentAiChapterEntity.ToString(),
|
||||
Guid = Guid.NewGuid().ToString(),
|
||||
ChapterIndex = chapterCount
|
||||
});
|
||||
}
|
||||
|
||||
return chapters;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 检查是否为章节标题
|
||||
/// </summary>
|
||||
private bool IsChapterTitle(string line, string cptPattern)
|
||||
{
|
||||
// 去除首尾空白字符
|
||||
var trimmedLine = line.Trim();
|
||||
|
||||
// 常见的章节标题模式
|
||||
var chapterPatterns = new[]
|
||||
{
|
||||
@"^第\s*[\u4e00-\u9fa50-9]+\s*章", // 第X章, 第XX章等
|
||||
@"^第\s*[0-9]+\s*章", // 第1章, 第2章等
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(cptPattern))
|
||||
{
|
||||
Array.Resize(ref chapterPatterns, chapterPatterns.Length + 1);
|
||||
chapterPatterns[chapterPatterns.Length - 1] = cptPattern;
|
||||
}
|
||||
|
||||
foreach (var pattern in chapterPatterns)
|
||||
{
|
||||
if (Regex.IsMatch(trimmedLine, pattern, RegexOptions.IgnoreCase))
|
||||
{
|
||||
//第一章内容说明了什么,请描述一下:
|
||||
if (trimmedLine.Length > 20) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 保存章节到单独的文件
|
||||
/// </summary>
|
||||
public void SaveChaptersToFiles(List<AiChapterEntity> chapters, string outputDirectory)
|
||||
{
|
||||
// 确保输出目录存在
|
||||
if (!Directory.Exists(outputDirectory))
|
||||
{
|
||||
Directory.CreateDirectory(outputDirectory);
|
||||
}
|
||||
|
||||
foreach (var chapter in chapters)
|
||||
{
|
||||
// 使用章节标题作为文件名,清理非法字符
|
||||
var fileName = CleanFileName(chapter.Title) + ".txt";
|
||||
var filePath = Path.Combine(outputDirectory, fileName);
|
||||
File.WriteAllText(filePath, chapter.Content, Encoding.UTF8);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 清理文件名中的非法字符
|
||||
/// </summary>
|
||||
private string CleanFileName(string fileName)
|
||||
{
|
||||
var invalidChars = Path.GetInvalidFileNameChars();
|
||||
foreach (var invalidChar in invalidChars)
|
||||
{
|
||||
fileName = fileName.Replace(invalidChar, '_');
|
||||
}
|
||||
|
||||
// 限制文件名长度
|
||||
if (fileName.Length > 100)
|
||||
{
|
||||
fileName = fileName.Substring(0, 100);
|
||||
}
|
||||
|
||||
return fileName;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,361 +0,0 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.Service;
|
||||
using Microsoft.ML;
|
||||
using Microsoft.ML.Data;
|
||||
using Microsoft.ML.Transforms.Onnx;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
||||
namespace CloudBuilder.AI.Service.Utility
|
||||
{
|
||||
/// <summary>
|
||||
/// 基于 ONNX 模型的命名实体识别器,支持提取人名及人物角色(人名+职位)。
|
||||
/// </summary>
|
||||
public class ChineseNameExtractor
|
||||
{
|
||||
private BertChineseTokenizer tokenizer;
|
||||
private OnnxScoringEstimator pipeline;
|
||||
private PredictionEngine<NerInput, NerOutput> engine;
|
||||
|
||||
/// <summary>
|
||||
/// 开启后会将每个 token 的预测标签及置信度输出到控制台,方便调试。
|
||||
/// </summary>
|
||||
public bool DebugMode { get; set; } = false;
|
||||
|
||||
public ChineseNameExtractor(IApplicationService service)
|
||||
{
|
||||
InitMLContext(service);
|
||||
}
|
||||
|
||||
private void InitMLContext(IApplicationService service)
|
||||
{
|
||||
var mlContext = new MLContext();
|
||||
string modelDir = service.Configuration["FileServiceSettings:OnnxDirectory"];
|
||||
string modelPath = Path.Combine(modelDir, "model.onnx");
|
||||
string vocabPath = Path.Combine(modelDir, "vocab.txt");
|
||||
|
||||
if (!File.Exists(modelPath) || !File.Exists(vocabPath))
|
||||
throw new Exception("请确保 model.onnx 和 vocab.txt 在指定目录下。");
|
||||
|
||||
tokenizer = new BertChineseTokenizer(vocabPath);
|
||||
|
||||
var shapeDictionary = new Dictionary<string, int[]>
|
||||
{
|
||||
{ "input_ids", new[] { 1, 128 } },
|
||||
{ "attention_mask", new[] { 1, 128 } },
|
||||
{ "token_type_ids", new[] { 1, 128 } },
|
||||
{ "logits", new[] { 1, 128, tokenizer.LabelCount } }
|
||||
};
|
||||
|
||||
var emptyData = mlContext.Data.LoadFromEnumerable(new List<NerInput>());
|
||||
pipeline = mlContext.Transforms.ApplyOnnxModel(
|
||||
modelFile: modelPath,
|
||||
inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" },
|
||||
outputColumnNames: new[] { "logits" },
|
||||
shapeDictionary: shapeDictionary,
|
||||
gpuDeviceId: null,
|
||||
fallbackToCpu: true
|
||||
);
|
||||
|
||||
var transformer = pipeline.Fit(emptyData);
|
||||
engine = mlContext.Model.CreatePredictionEngine<NerInput, NerOutput>(transformer);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 提取纯人名(标签为 B-PER / I-PER 的连续实体)。
|
||||
/// </summary>
|
||||
public List<string> ExtractNames(string text)
|
||||
{
|
||||
return ExtractPersonEntities(text, onlyPureName: true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 提取人物角色(人名 + 紧接其后的职位,例如“岳堂主”、“张经理”等)。
|
||||
/// </summary>
|
||||
public List<string> ExtractPersonRoles(string text)
|
||||
{
|
||||
return ExtractPersonEntities(text, onlyPureName: false);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 核心实体提取方法,支持合并人名与其后的职位标签。
|
||||
/// </summary>
|
||||
/// <param name="text">输入文本</param>
|
||||
/// <param name="onlyPureName">true 只提取纯人名;false 提取人名+职位复合实体</param>
|
||||
private List<string> ExtractPersonEntities(string text, bool onlyPureName)
|
||||
{
|
||||
var tokenized = tokenizer.Tokenize(text);
|
||||
var input = new NerInput
|
||||
{
|
||||
InputIds = tokenized.InputIds,
|
||||
AttentionMask = tokenized.AttentionMask,
|
||||
TokenTypeIds = tokenized.TokenTypeIds
|
||||
};
|
||||
|
||||
var prediction = engine.Predict(input);
|
||||
var idToLabel = tokenizer.IdToLabel;
|
||||
int seqLen = 128;
|
||||
int numLabels = idToLabel.Length;
|
||||
|
||||
var entities = new List<string>();
|
||||
string currentEntity = "";
|
||||
bool insideEntity = false;
|
||||
bool isPersonEntity = false; // 当前实体是否以 B-PER 开始
|
||||
|
||||
for (int i = 0; i < seqLen; i++)
|
||||
{
|
||||
if (tokenized.AttentionMask[i] == 0) continue; // 跳过 padding
|
||||
|
||||
// 找到当前 token 的预测标签
|
||||
int startIdx = i * numLabels;
|
||||
float maxVal = float.MinValue;
|
||||
int maxIdx = 0;
|
||||
for (int j = 0; j < numLabels; j++)
|
||||
{
|
||||
float val = prediction.Logits[startIdx + j];
|
||||
if (val > maxVal)
|
||||
{
|
||||
maxVal = val;
|
||||
maxIdx = j;
|
||||
}
|
||||
}
|
||||
|
||||
string label = idToLabel[maxIdx];
|
||||
string token = tokenized.Tokens[i];
|
||||
|
||||
if (DebugMode)
|
||||
Console.WriteLine($"{token} -> {label} (置信度: {maxVal:F3})");
|
||||
|
||||
// 判断当前标签类型
|
||||
bool isPerStart = label == "B-PER";
|
||||
bool isPerInside = label == "I-PER";
|
||||
bool isPositionStart = label == "B-POSITION";
|
||||
bool isPositionInside = label == "I-POSITION";
|
||||
|
||||
// ---- 仅提取纯人名的逻辑 ----
|
||||
if (onlyPureName)
|
||||
{
|
||||
if (isPerStart)
|
||||
{
|
||||
if (insideEntity && currentEntity.Length > 0) entities.Add(currentEntity);
|
||||
currentEntity = token;
|
||||
insideEntity = true;
|
||||
isPersonEntity = true;
|
||||
}
|
||||
else if (isPerInside && insideEntity && isPersonEntity)
|
||||
{
|
||||
currentEntity += token;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (insideEntity && currentEntity.Length > 0)
|
||||
{
|
||||
entities.Add(currentEntity);
|
||||
currentEntity = "";
|
||||
insideEntity = false;
|
||||
isPersonEntity = false;
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// ---- 提取人物角色(人名+职位)的逻辑 ----
|
||||
if (isPerStart)
|
||||
{
|
||||
// 无论之前是否有未结束的实体,都结束之前的,并开始一个新的人名
|
||||
if (insideEntity && currentEntity.Length > 0)
|
||||
entities.Add(currentEntity);
|
||||
|
||||
currentEntity = token;
|
||||
insideEntity = true;
|
||||
isPersonEntity = true;
|
||||
}
|
||||
else if (isPositionStart)
|
||||
{
|
||||
if (insideEntity && isPersonEntity)
|
||||
{
|
||||
// 当前正在收集人名,且遇到 B-POSITION,则合并职位
|
||||
currentEntity += token;
|
||||
// 保持 isPersonEntity = true 不变,这样后续的 I-POSITION 也能继续合并
|
||||
}
|
||||
else
|
||||
{
|
||||
// 不在人名内部,则视作新实体(提取单独的职位,如“堂主”)
|
||||
if (insideEntity && currentEntity.Length > 0)
|
||||
entities.Add(currentEntity);
|
||||
currentEntity = token;
|
||||
insideEntity = true;
|
||||
isPersonEntity = false; // 标记为纯职位实体
|
||||
}
|
||||
}
|
||||
else if ((isPerInside || isPositionInside) && insideEntity)
|
||||
{
|
||||
// 无论是 I-PER 还是 I-POSITION,只要在实体内部都追加
|
||||
currentEntity += token;
|
||||
}
|
||||
else
|
||||
{
|
||||
// 遇到其他标签(O 或其他 B- 开头),结束当前实体
|
||||
if (insideEntity && currentEntity.Length > 0)
|
||||
{
|
||||
entities.Add(currentEntity);
|
||||
currentEntity = "";
|
||||
insideEntity = false;
|
||||
isPersonEntity = false;
|
||||
}
|
||||
// 注意:不处理非目标标签本身,因为它们不属于角色
|
||||
}
|
||||
}
|
||||
|
||||
// 收尾最后一个实体
|
||||
if (insideEntity && currentEntity.Length > 0)
|
||||
entities.Add(currentEntity);
|
||||
|
||||
return entities;
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== 数据模型 ====================
|
||||
public class NerInput
|
||||
{
|
||||
[VectorType(1, 128)]
|
||||
[ColumnName("input_ids")]
|
||||
public long[] InputIds { get; set; }
|
||||
|
||||
[VectorType(1, 128)]
|
||||
[ColumnName("attention_mask")]
|
||||
public long[] AttentionMask { get; set; }
|
||||
|
||||
[VectorType(1, 128)]
|
||||
[ColumnName("token_type_ids")]
|
||||
public long[] TokenTypeIds { get; set; }
|
||||
}
|
||||
|
||||
public class NerOutput
|
||||
{
|
||||
[VectorType(1, 128, 29)] // 维度需与模型输出一致
|
||||
[ColumnName("logits")]
|
||||
public float[] Logits { get; set; }
|
||||
}
|
||||
|
||||
// ==================== 分词器 ====================
|
||||
public class BertChineseTokenizer
|
||||
{
|
||||
private readonly Dictionary<string, int> _tokenToId;
|
||||
|
||||
/// <summary>
|
||||
/// 标签映射,务必与模型训练时的标签顺序一致。
|
||||
/// </summary>
|
||||
public readonly string[] IdToLabel;
|
||||
public int LabelCount => IdToLabel.Length;
|
||||
|
||||
public BertChineseTokenizer(string vocabPath, int maxSeqLength = 128)
|
||||
{
|
||||
_tokenToId = new Dictionary<string, int>();
|
||||
var lines = File.ReadAllLines(vocabPath);
|
||||
for (int i = 0; i < lines.Length; i++)
|
||||
_tokenToId[lines[i]] = i;
|
||||
|
||||
// 请根据实际模型输出维度调整此数组!
|
||||
// 下面的顺序仅为示例,必须与模型 logits 第2维一一对应。
|
||||
IdToLabel = new string[] {
|
||||
"O", // 0
|
||||
"B-ORG", // 1
|
||||
"I-ORG", // 2
|
||||
"B-PER", // 3
|
||||
"I-PER", // 4
|
||||
"B-TIME", // 5
|
||||
"I-TIME", // 6
|
||||
"B-LOC", // 7
|
||||
"I-LOC", // 8
|
||||
"B-POSITION", // 9
|
||||
"I-POSITION", // 10
|
||||
"B-COMPANY", // 11
|
||||
"I-COMPANY", // 12
|
||||
"B-GAME", // 13
|
||||
"I-GAME", // 14
|
||||
"B-GOVERNMENT", // 15
|
||||
"I-GOVERNMENT", // 16
|
||||
"B-SCENE", // 17
|
||||
"I-SCENE", // 18
|
||||
"B-SUBJECT", // 19
|
||||
"I-SUBJECT", // 20
|
||||
"B-CREATION", // 21
|
||||
"I-CREATION", // 22
|
||||
"B-FOOD", // 23
|
||||
"I-FOOD", // 24
|
||||
"B-MOVIE", // 25
|
||||
"I-MOVIE", // 26
|
||||
"[CLS]", // 27 (如果模型没有这些特殊标签,请删除)
|
||||
"[SEP]" // 28
|
||||
};
|
||||
}
|
||||
|
||||
public TokenizedResult Tokenize(string text)
|
||||
{
|
||||
const int maxLen = 128;
|
||||
var inputIds = new long[maxLen];
|
||||
var attentionMask = new long[maxLen];
|
||||
var tokenTypeIds = new long[maxLen];
|
||||
var tokens = new string[maxLen];
|
||||
|
||||
var chars = text.ToCharArray();
|
||||
int charIndex = 0;
|
||||
|
||||
for (int i = 0; i < maxLen; i++)
|
||||
{
|
||||
if (i == 0)
|
||||
{
|
||||
inputIds[i] = 101;
|
||||
tokens[i] = "[CLS]";
|
||||
}
|
||||
else if (i == chars.Length + 1)
|
||||
{
|
||||
inputIds[i] = 102;
|
||||
tokens[i] = "[SEP]";
|
||||
}
|
||||
else if (i > chars.Length + 1)
|
||||
{
|
||||
inputIds[i] = 0;
|
||||
tokens[i] = "[PAD]";
|
||||
}
|
||||
else
|
||||
{
|
||||
string c = chars[charIndex].ToString();
|
||||
charIndex++;
|
||||
if (_tokenToId.TryGetValue(c, out int id))
|
||||
{
|
||||
inputIds[i] = id;
|
||||
tokens[i] = c;
|
||||
}
|
||||
else
|
||||
{
|
||||
inputIds[i] = _tokenToId["[UNK]"];
|
||||
tokens[i] = "[UNK]";
|
||||
}
|
||||
}
|
||||
|
||||
attentionMask[i] = (i <= chars.Length + 1) ? 1 : 0;
|
||||
tokenTypeIds[i] = 0;
|
||||
}
|
||||
|
||||
return new TokenizedResult
|
||||
{
|
||||
InputIds = inputIds,
|
||||
AttentionMask = attentionMask,
|
||||
TokenTypeIds = tokenTypeIds,
|
||||
Tokens = tokens
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public class TokenizedResult
|
||||
{
|
||||
public long[] InputIds { get; set; }
|
||||
public long[] AttentionMask { get; set; }
|
||||
public long[] TokenTypeIds { get; set; }
|
||||
public string[] Tokens { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
using CloudBuilder.AI.Data;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using DocumentFormat.OpenXml.Spreadsheet;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.AI.Service.Utility
|
||||
{
|
||||
public class ParagraphSplitter
|
||||
{
|
||||
public ParagraphSplitter()
|
||||
{
|
||||
}
|
||||
public void PackParagraphs(AiBookData data)
|
||||
{
|
||||
if (data.AiChapters == null || data.AiChapters.Length == 0) return;
|
||||
|
||||
var paragraphs = new List<AiParagraphEntity>();
|
||||
var sentences = new List<AiSentenceEntity>();
|
||||
|
||||
foreach (AiChapterEntity aiChapterEntity in data.AiChapters)
|
||||
{
|
||||
SplitChapterIntoParagraphs(data, aiChapterEntity);
|
||||
if (data.AiParagraphs != null && data.AiParagraphs.Length > 0)
|
||||
paragraphs.AddRange(data.AiParagraphs);
|
||||
if (data.AiSentences != null && data.AiSentences.Length > 0)
|
||||
sentences.AddRange(data.AiSentences);
|
||||
}
|
||||
data.AiParagraphs = paragraphs.ToArray();
|
||||
data.AiSentences = sentences.ToArray();
|
||||
}
|
||||
|
||||
public AiBookData SplitChapterIntoParagraphs(AiBookData data, AiChapterEntity aiChapter)
|
||||
{
|
||||
data.AiParagraphs = null;
|
||||
data.AiSentences = null;
|
||||
|
||||
var paragraphs = new List<AiParagraphEntity>();
|
||||
var sentences = new List<AiSentenceEntity>();
|
||||
var temps = new List<AiSentenceEntity>();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(aiChapter.Content)) return data;
|
||||
|
||||
// 按换行拆分段落(过滤空段落)
|
||||
var paragraphTexts = aiChapter.Content.Split("\r\n")
|
||||
.Select(p => p.Trim())
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p))
|
||||
.ToList();
|
||||
|
||||
SentenceSplitter sentenceSplitter = new SentenceSplitter();
|
||||
|
||||
AiParagraphEntity aiParagraph;
|
||||
int index = 1;
|
||||
|
||||
foreach (var text in paragraphTexts)
|
||||
{
|
||||
if (!ValidText(text)) continue;
|
||||
|
||||
aiParagraph = new AiParagraphEntity();
|
||||
aiParagraph.Content = text;
|
||||
aiParagraph.ChapterGuid = aiChapter.Guid;
|
||||
aiParagraph.Guid = Guid.NewGuid().ToString();
|
||||
aiParagraph.ParagraphIndex = index++;
|
||||
aiParagraph.DialogueIndc = sentenceSplitter.IsDialogue(text) ? YesNoPolicy.YES : YesNoPolicy.NO;
|
||||
paragraphs.Add(aiParagraph);
|
||||
|
||||
temps = sentenceSplitter.SplitByConfigurableQuotes(aiParagraph);
|
||||
|
||||
if (temps != null && temps.Count() > 0) sentences.AddRange(temps);
|
||||
}
|
||||
|
||||
data.AiParagraphs = paragraphs.ToArray();
|
||||
data.AiSentences = sentences.ToArray();
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private bool ValidText(string text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text.Trim())) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,197 +0,0 @@
|
||||
using CloudBuilder.AI.Entity;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.AI.Service.Utility
|
||||
{
|
||||
/// <summary>
|
||||
/// 智能句子分割器
|
||||
/// </summary>
|
||||
public class SentenceSplitter
|
||||
{
|
||||
// 成对引号配置数组(可自由扩展,格式:[左引号, 右引号])
|
||||
private readonly List<(string LeftQuote, string RightQuote)> _quotePairs;
|
||||
// 动态生成的正则表达式(匹配所有配置的引号类型)
|
||||
private Regex _dynamicQuotePattern;
|
||||
// 引号内必须包含的结尾标点集合
|
||||
private readonly string _endPunctuations = "?!。,;:、……~·";
|
||||
|
||||
/// <summary>
|
||||
/// 默认构造函数:初始化常用成对引号(中文、英文、直角引号)
|
||||
/// </summary>
|
||||
public SentenceSplitter()
|
||||
{
|
||||
_quotePairs = new List<(string, string)>
|
||||
{
|
||||
("“", "”"), // 中文弯引号
|
||||
("\"", "\""), // 英文双引号(需转义)
|
||||
("〝", "〞"), // 中文直角引号
|
||||
("「", "」") // 中文方头引号(可选扩展)
|
||||
};
|
||||
InitDynamicRegex();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 自定义构造函数:允许外部传入任意成对引号配置
|
||||
/// </summary>
|
||||
/// <param name="customQuotePairs">自定义成对引号列表</param>
|
||||
public SentenceSplitter(List<(string LeftQuote, string RightQuote)> customQuotePairs)
|
||||
{
|
||||
_quotePairs = customQuotePairs ?? new List<(string, string)>();
|
||||
InitDynamicRegex();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 初始化动态正则:根据配置的成对引号生成匹配规则
|
||||
/// </summary>
|
||||
private void InitDynamicRegex()
|
||||
{
|
||||
if (_quotePairs.Count == 0)
|
||||
{
|
||||
// 空配置时生成空正则,避免匹配任何内容
|
||||
_dynamicQuotePattern = new Regex(@"^$", RegexOptions.Compiled);
|
||||
return;
|
||||
}
|
||||
|
||||
// 步骤1:转义所有引号(避免正则特殊字符冲突),拼接左/右引号匹配组
|
||||
string escapedLeftQuotes = string.Join("|", _quotePairs.Select(p => Regex.Escape(p.LeftQuote)));
|
||||
string escapedRightQuotes = string.Join("|", _quotePairs.Select(p => Regex.Escape(p.RightQuote)));
|
||||
// 步骤2:转义标点集合,避免正则解析错误
|
||||
string escapedPunctuations = Regex.Escape(_endPunctuations);
|
||||
|
||||
// 步骤3:构建动态正则表达式
|
||||
// 分组说明:
|
||||
// ?<left>:匹配配置的任意左引号
|
||||
// ?<content>:匹配引号内的内容(必须以指定标点结尾)
|
||||
// ?<right>:匹配配置的任意右引号
|
||||
string regexPattern = $@"(?<left>{escapedLeftQuotes})(?<content>.+?[{escapedPunctuations}])(?<right>{escapedRightQuotes})";
|
||||
_dynamicQuotePattern = new Regex(regexPattern, RegexOptions.Compiled | RegexOptions.Singleline);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 核心分割方法:按配置的成对引号分割文本,结果无引号字符
|
||||
/// </summary>
|
||||
/// <param name="input">待分割的文本(如:“其他标点符号,如?”紫茜的秀目 或 "其他标点符号,如?"紫茜的秀目)</param>
|
||||
/// <returns>分割后的纯文本列表</returns>
|
||||
public List<AiSentenceEntity> SplitByConfigurableQuotes(AiParagraphEntity aiParagraphEntity)
|
||||
{
|
||||
AiSentenceEntity aiSentence;
|
||||
int sentenceIndex = 1;
|
||||
// 空值/空白校验
|
||||
if (string.IsNullOrWhiteSpace(aiParagraphEntity.Content))
|
||||
{
|
||||
return new List<AiSentenceEntity>();
|
||||
}
|
||||
|
||||
List<AiSentenceEntity> result = new List<AiSentenceEntity>();
|
||||
string remainingText = aiParagraphEntity.Content.Trim();
|
||||
int lastMatchEnd = 0;
|
||||
|
||||
// 遍历所有匹配的「引号+内容+标点+引号」文本块
|
||||
foreach (Match match in _dynamicQuotePattern.Matches(remainingText))
|
||||
{
|
||||
// 1. 提取匹配的左/右引号,校验是否为成对配置
|
||||
string matchedLeft = match.Groups["left"].Value;
|
||||
string matchedRight = match.Groups["right"].Value;
|
||||
bool isPairMatched = _quotePairs.Any(p => p.LeftQuote == matchedLeft && p.RightQuote == matchedRight);
|
||||
if (!isPairMatched)
|
||||
{
|
||||
continue; // 非成对引号,跳过该匹配(避免“xxx"这类混合匹配)
|
||||
}
|
||||
|
||||
// 2. 提取引号前的文本(过滤空值)
|
||||
string prefixText = remainingText.Substring(lastMatchEnd, match.Index - lastMatchEnd).Trim();
|
||||
if (!string.IsNullOrEmpty(prefixText))
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = prefixText;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.NO;
|
||||
aiSentence.SentenceIndex = sentenceIndex++;
|
||||
aiSentence.ParagraphGuid = aiParagraphEntity.Guid;
|
||||
aiSentence.Status = "G";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
result.Add(aiSentence);
|
||||
}
|
||||
|
||||
// 3. 提取引号内的纯内容(无引号字符)
|
||||
string quotedContent = match.Groups["content"].Value.Trim();
|
||||
if (!string.IsNullOrEmpty(quotedContent))
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = quotedContent;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.YES;
|
||||
aiSentence.SentenceIndex = sentenceIndex++;
|
||||
aiSentence.ParagraphGuid = aiParagraphEntity.Guid;
|
||||
aiSentence.Status = "P";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
result.Add(aiSentence);
|
||||
}
|
||||
|
||||
// 更新最后匹配位置
|
||||
lastMatchEnd = match.Index + match.Length;
|
||||
}
|
||||
|
||||
// 4. 提取最后一个匹配项后的剩余文本
|
||||
string suffixText = remainingText.Substring(lastMatchEnd).Trim();
|
||||
if (!string.IsNullOrEmpty(suffixText))
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = suffixText;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.NO;
|
||||
aiSentence.SentenceIndex = sentenceIndex++;
|
||||
aiSentence.ParagraphGuid = aiParagraphEntity.Guid;
|
||||
aiSentence.Status = "G";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
result.Add(aiSentence);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 新增:判断文本是否包含对话内容
|
||||
/// </summary>
|
||||
/// <param name="text">待判断的文本(句子/段落)</param>
|
||||
/// <returns>true=包含对话,false=纯旁白/空文本</returns>
|
||||
public bool IsDialogue(string text)
|
||||
{
|
||||
// 1. 处理空文本/NULL,直接返回false
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
return false;
|
||||
|
||||
// 2. 文本预处理:去空格、统一全角/半角引号
|
||||
string cleanedText = text.Trim();
|
||||
// 替换全角空格为半角,避免干扰判断
|
||||
cleanedText = Regex.Replace(cleanedText, @" ", " ");
|
||||
// 统一引号格式(中文弯引号、直引号 → 标准双引号)
|
||||
cleanedText = cleanedText.Replace("“", "\"")
|
||||
.Replace("”", "\"")
|
||||
.Replace("〝", "\"")
|
||||
.Replace("〞", "\"")
|
||||
.Replace("「", "\"")
|
||||
.Replace("」", "\"")
|
||||
.Replace("‘", "'")
|
||||
.Replace("’", "'");
|
||||
|
||||
// 3. 核心判断规则:
|
||||
// - 包含成对/单个双引号(排除仅含引号无内容的情况)
|
||||
// - 包含冒号+引号(如:"XXX说:"XXX"" 场景)
|
||||
var hasQuote = Regex.IsMatch(cleanedText, @""".+?""") || // 成对双引号(含内容)
|
||||
Regex.IsMatch(cleanedText, @":\s*"""); // 冒号后接引号(说话人+对话)
|
||||
var hasSingleQuote = Regex.IsMatch(cleanedText, @"'[^']+'"); // 成对单引号(英文对话)
|
||||
|
||||
// 4. 排除仅含引号符号的无效情况(如 ""、''、“”)
|
||||
var onlyQuote = Regex.IsMatch(cleanedText, @"^\s*[""''“”]{1,2}\s*$");
|
||||
|
||||
// 最终判断:包含有效引号且不是仅含引号 → 判定为对话
|
||||
return (hasQuote || hasSingleQuote) && !onlyQuote;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user