ow
This commit is contained in:
owenchen committed
1 parent
597b88d075
commit
527205520b
12 files changed
+104
-43
No files matched your search
@@ -2,12 +2,12 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.DependencyInjection.Task;
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using CloudBuilder.Topshelf.Task.Novel.Process;
|
||||
using CloudBuilder.Topshelf.Utility;
|
||||
using static System.Net.Mime.MediaTypeNames;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task
|
||||
{
|
||||
//task:ChineseNameExtractorTask guid:B000006
|
||||
//task:ChineseNameExtractorTask book_id:B000008
|
||||
public class ChineseNameExtractorTask : IScheduleTask
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
@@ -19,16 +19,16 @@ namespace CloudBuilder.Topshelf.Task
|
||||
|
||||
public void Run(Dictionary<string, string> bodyDict)
|
||||
{
|
||||
if (!bodyDict.ContainsKey("guid")) return;
|
||||
if (!bodyDict.ContainsKey("book_id")) return;
|
||||
|
||||
string guid = bodyDict["guid"];
|
||||
string guid = bodyDict["book_id"];
|
||||
|
||||
PythonCommand python = new PythonCommand(service);
|
||||
try
|
||||
{
|
||||
Hanlp2 hanlp = new Hanlp2(python);
|
||||
|
||||
IRepository<AiSentenceDialogueViewEntity> repository = service.GetRepository<IRepository<AiSentenceDialogueViewEntity>>();
|
||||
IRepository<AiSentenceViewEntity> repository = service.GetRepository<IRepository<AiSentenceViewEntity>>();
|
||||
|
||||
IRepository<AiSentenceEntity> repositoryAiSentenceEntity = service.GetRepository<IRepository<AiSentenceEntity>>();
|
||||
|
||||
@@ -42,10 +42,12 @@ namespace CloudBuilder.Topshelf.Task
|
||||
|
||||
SpeakerAnalysisHelper speakerFeatureLearner = new SpeakerAnalysisHelper();
|
||||
|
||||
AiSentenceDialogueViewEntity[] ents = repository.DetachedEntities.Where(x => x.BookGuid == guid).OrderBy(x => x.ChapterIndex).ThenBy(x => x.ParagraphIndex).ThenBy(x => x.DialogueIndc).ToArray();
|
||||
AiSentenceViewEntity[] ents = repository.DetachedEntities.Where(x => x.BookId == guid).OrderBy(x => x.ChapterId).ThenBy(x => x.ParagraphId).ThenBy(x => x.SentenceIndex).ToArray();
|
||||
if (ents == null || ents.Length == 0) return;
|
||||
|
||||
AiSentenceEntity aiSentenceEntity;
|
||||
string person = null;
|
||||
int paragraphIndex = 0;
|
||||
long paragraphIndex = 0;
|
||||
|
||||
int totalLines = ents.Length;
|
||||
int processedLines = 0; // 已处理行数计数器
|
||||
@@ -54,23 +56,17 @@ namespace CloudBuilder.Topshelf.Task
|
||||
{
|
||||
processedLines++;
|
||||
|
||||
if (!string.IsNullOrEmpty(person) && paragraphIndex == ent.ParagraphIndex)
|
||||
person = speakerFeatureLearner.GetPerson(hanlp, ent.Content);
|
||||
paragraphIndex = ent.ParagraphId;
|
||||
|
||||
if (!string.IsNullOrEmpty(person))
|
||||
{
|
||||
paragraphIndex = 0;
|
||||
aiSentenceEntity = repositoryAiSentenceEntity.Entities.Where(x => x.Guid == ent.Guid).FirstOrDefault();
|
||||
aiSentenceEntity = repositoryAiSentenceEntity.Entities.Where(x => x.SentenceId == ent.SentenceId).FirstOrDefault();
|
||||
aiSentenceEntity.PersonName = person;
|
||||
|
||||
repositoryAiSentenceEntity.UpdateNow(aiSentenceEntity);
|
||||
continue;
|
||||
}
|
||||
|
||||
depRoot = hanlp.GetDepRoot(ent.Content);
|
||||
ners = hanlp.GetHanlpNer(ent.Content);
|
||||
srls = hanlp.GetHanlpSrl(ent.Content);
|
||||
|
||||
person = speakerFeatureLearner.GetPerson(depRoot, ners, srls, cons);
|
||||
paragraphIndex = ent.ParagraphIndex;
|
||||
|
||||
UpdateProgress(processedLines, totalLines);
|
||||
}
|
||||
|
||||
@@ -81,6 +77,7 @@ namespace CloudBuilder.Topshelf.Task
|
||||
}
|
||||
finally
|
||||
{
|
||||
Console.WriteLine("");
|
||||
python.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,102 +0,0 @@
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel.Process
|
||||
{
|
||||
public class SpeakerAnalysisHelper
|
||||
{
|
||||
public string GetPerson(string depRoot, List<NamedEntity> ners, List<List<MeaningRepresentationParsingEntity>> srls, List<HanlpConstituencyNode> cons)
|
||||
{
|
||||
string arg0 = string.Empty;
|
||||
|
||||
if (!string.IsNullOrEmpty(depRoot) && srls != null && srls.Count() > 0)
|
||||
{
|
||||
foreach (var srl in srls)
|
||||
{
|
||||
if (srl.Where(x => x.Text == depRoot).Any())
|
||||
{
|
||||
foreach (var mrp in srl)
|
||||
{
|
||||
if (mrp.Type.ToUpper() != "ARG0") continue;
|
||||
|
||||
if (ners != null && ners.Where(x => mrp.Text.Contains(x.Text)).Any() &&
|
||||
ners.Where(x => mrp.Text.Contains(x.Text)).FirstOrDefault().Text.Length > 1)
|
||||
return ners.Where(x => mrp.Text.Contains(x.Text)).FirstOrDefault().Text;
|
||||
|
||||
return mrp.Text.Length > 6 ? mrp.Text.Substring(mrp.Text.Length - 6, 6) : mrp.Text;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var mrp in srl)
|
||||
{
|
||||
if (mrp.Type.ToUpper() != "ARG0") continue;
|
||||
arg0 = mrp.Text.Length > 6 ? mrp.Text.Substring(mrp.Text.Length - 6, 6) : mrp.Text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ners != null && ners.Count() > 0)
|
||||
return ners.FirstOrDefault().Text;
|
||||
|
||||
return arg0;
|
||||
}
|
||||
|
||||
public AiParagraphEntity[] AiParagraphs { get; set; }
|
||||
|
||||
public AiSentenceEntity[] AiSentences { get; set; }
|
||||
|
||||
public AiChapterPersonEntity[] AiChapterPersons { get; set; }
|
||||
}
|
||||
}
|
||||
/*
|
||||
“文本解析→人物提取→上下文匹配→结果输出”
|
||||
|
||||
graph TD
|
||||
A[对话行] --> B{规则1：文本特征匹配};
|
||||
B -->|匹配成功| C[直接赋值说话者];
|
||||
B -->|匹配失败| D{规则2：紧邻旁白匹配};
|
||||
D -->|匹配成功| C;
|
||||
D -->|匹配失败| E{规则3：段落内最近人物};
|
||||
E -->|匹配成功| C;
|
||||
E -->|匹配失败| F[赋值默认值+标记异常];
|
||||
|
||||
规则 1：文本特征精准匹配（最高优先级）
|
||||
基于对话内容中的 “专属特征” 直接匹配说话者，避免上下文干扰：
|
||||
示例规则：
|
||||
对话内容特征 匹配的说话者
|
||||
包含 “第二元婴” 韩立
|
||||
包含 “银月所化” 银月
|
||||
包含 “本座 / 本尊” 反派角色
|
||||
|
||||
落地：维护一个 “特征 - 人物” 映射字典，匹配时优先使用：
|
||||
规则 2：紧邻旁白匹配（次高优先级）只找对话行 “紧邻的前 1-2 行旁白”，而非整个段落的旁白，避免跨越多个人物导致匹配错误：
|
||||
规则：仅检查对话行的前 1 行、前 2 行旁白（最多前 3 行），超过则停止；
|
||||
落地：修改向前查找的逻辑，限制查找范围
|
||||
|
||||
规则 3：排除无效人物（避免干扰）从旁白中提取人物时，过滤掉 “非核心人物”（如 “店小二”“路人”），只保留主要人物：
|
||||
落地：维护 “核心人物白名单”，提取人物时只保留白名单内的名称
|
||||
|
||||
3. 增加异常处理和人工校验（兜底保障）
|
||||
即使逻辑再完善，也会有边缘场景匹配错误，需通过 “异常标记 + 人工复核” 兜底：
|
||||
|
||||
步骤 1：标记可疑匹配结果对以下情况标记为 “可疑”，存入日志或单独字段：
|
||||
匹配到的人物在旁白中出现次数≥2（比如同一段落有韩立和银月，无法确定）；
|
||||
对话行前后 3 行都没有找到人物，最终赋值为 “未知人物”；
|
||||
对话内容无任何特征，且段落内人物数量≥3；
|
||||
落地：为实体增加扩展字段（如IsSuspected），或生成异常日志
|
||||
|
||||
步骤 2：人工复核可疑数据
|
||||
规则：所有标记为 “可疑” 的匹配结果，必须经过人工复核后再更新到数据库；
|
||||
落地：生成 “可疑数据清单”（包含Guid、Content、PersonName），供运营 / 编辑人员校验，修正错误后重新填充。
|
||||
|
||||
|
||||
收集错误案例：记录每次匹配错误的场景（如 “银月的对话匹配给韩立”），分析错误原因；
|
||||
迭代特征字典：将错误案例转化为新的特征规则（如新增 “乌云”→银月，“元婴”→韩立）；
|
||||
A/B 测试：对优化后的规则做小范围测试，对比匹配准确率，确认有效后全量上线。
|
||||
*/
|
||||
@@ -1,7 +1,6 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.DependencyInjection.Task;
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using CloudBuilder.Topshelf.Task.Novel.Process;
|
||||
using CloudBuilder.Topshelf.Utility;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task
|
||||
@@ -29,22 +28,24 @@ namespace CloudBuilder.Topshelf.Task
|
||||
|
||||
text = "银发老者神色一肃的向金越禅师问道。";
|
||||
//text = "鬼灵门门主一边吩咐着，一边冷冷打量着悬浮在身前的画轴，见其并没有任何异样发生，目光闪动几下。";
|
||||
text = "韩立闻言，眉头不经意的一皱，想了想后，还是摇摇头说道：";
|
||||
List<string> sentiments = hanlp.GetTokFine(text);
|
||||
text = "这是一个小城，说是小城其实只是一个大点的镇子，名字也叫青牛镇，只有那些住在附近山沟里、没啥见识的土人，才“青牛城”“青牛城”的叫个不停。这是干了十几年门丁的张二心里话。";
|
||||
//List<string> sentiments = hanlp.GetTokFine(text);
|
||||
|
||||
sentiments = hanlp.GetTokCoarse(text);
|
||||
//sentiments = hanlp.GetTokCoarse(text);
|
||||
|
||||
string depRoot = hanlp.GetDepRoot(text);
|
||||
//string depRoot = hanlp.GetDepRoot(text);
|
||||
|
||||
List<NamedEntity> ners = hanlp.GetHanlpNer(text);
|
||||
//List<NamedEntity> ners = hanlp.GetHanlpNer(text);
|
||||
|
||||
List<List<MeaningRepresentationParsingEntity>> srls = hanlp.GetHanlpSrl(text);
|
||||
//List<List<MeaningRepresentationParsingEntity>> srls = hanlp.GetHanlpSrl(text);
|
||||
|
||||
List<HanlpConstituencyNode> cons = hanlp.GetHanlpCon(text);
|
||||
//List<HanlpConstituencyNode> cons = hanlp.GetHanlpCon(text);
|
||||
|
||||
SpeakerAnalysisHelper speakerFeatureLearner = new SpeakerAnalysisHelper();
|
||||
|
||||
speakerFeatureLearner.GetPerson(depRoot, ners, srls, cons);
|
||||
var person = speakerFeatureLearner.GetPerson(hanlp, text);
|
||||
|
||||
text = "韩立闻言，眉头不经意的一皱，想了想后，还是摇摇头说道：";
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
||||
+1
-1
@@ -12,7 +12,7 @@ using static System.Runtime.InteropServices.JavaScript.JSType;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.TTS
|
||||
{
|
||||
//task:TtsTask voice:Sherpa2 save_path:D:\MAFAI\cc.wav text:乌云忽然凝聚变形，瞬间化为了一个身高寸许的黑绿婴儿出来，并口吐人言的说道。
|
||||
//task:TtsTask voice:Sherpa2 save_path:D:\\Net8\\FileServer\\Backup\\DmsFile\\voice\\20260528112739.mp3 text:二愣子睁大着双眼，直直望着茅草和烂泥糊成的黑屋顶，身上盖着的旧棉被，已呈深黄色，看不出原来的本来面目，还若有若无的散发着淡淡的霉味。
|
||||
public class TtsTask : IScheduleTask
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
Reference in new issue
Block a user