240 lines
9.6 KiB
C#
240 lines
9.6 KiB
C#
using CloudBuilder.AI.Data;
|
||
using CloudBuilder.AI.Entity;
|
||
using CloudBuilder.AI.SearchCriteria;
|
||
using CloudBuilder.AI.Service;
|
||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||
using CloudBuilder.Core.DependencyInjection.Task;
|
||
using CloudBuilder.Core.Service;
|
||
using CloudBuilder.Security.Entity;
|
||
using CloudBuilder.Topshelf.Python;
|
||
using CloudBuilder.Topshelf.Task.Novel.Process;
|
||
using Microsoft.Extensions.DependencyInjection;
|
||
using System;
|
||
using System.Collections.Generic;
|
||
using System.Linq;
|
||
using System.Text;
|
||
using System.Threading.Tasks;
|
||
using static System.Net.Mime.MediaTypeNames;
|
||
|
||
namespace CloudBuilder.Topshelf.Task.Novel
|
||
{
|
||
//task:SpeakerTask param_name:param_value
|
||
public class SpeakerTask : IScheduleTask
|
||
{
|
||
private readonly IApplicationService service;
|
||
|
||
public SpeakerTask(IApplicationService service)
|
||
{
|
||
this.service = service;
|
||
}
|
||
|
||
public void Run(Dictionary<string, string> bodyDict)
|
||
{
|
||
AiSentenceSearchCriteria c = new AiSentenceSearchCriteria();
|
||
c.limit = 999999;
|
||
List<SearchFilter> filters = new List<SearchFilter>();
|
||
|
||
SearchFilter filter = new SearchFilter();
|
||
filter.Column = AiSentenceViewEntity.DB_CHAPTER_GUID;
|
||
filter.Type = SearchFilterTypePolicy.TEXT;
|
||
filter.Operator = SearchFilterOperatorPolicy.CONTAIN;
|
||
filter.Value = "cf34136e-d30a-4901-9109-626e412eaf57";
|
||
filters.Add(filter);
|
||
|
||
c.Filter = filters.ToArray();
|
||
|
||
AiSentenceViewResultData data = service.ServiceProvider.GetService<IAiSentenceService>().FindBySearchCriteria(c);
|
||
|
||
if (data == null || data.Datas == null || data.Datas.Count() == 0) return;
|
||
|
||
AiSentenceViewEntity[] sentenceViews = data.Datas.OrderBy(x => x.ParagraphIndex).ThenBy(x => x.DialogueIndc).ToArray();
|
||
|
||
if (sentenceViews == null || sentenceViews.Length == 0) return;
|
||
|
||
PythonCommand python = new PythonCommand(service);
|
||
|
||
Hanlp2 hanlp = new Hanlp2(python);
|
||
|
||
List<NamedSummary> all = GetNamedSummary(hanlp, data.Datas, 0, data.Datas.Length);
|
||
|
||
string text = string.Empty;
|
||
//@"乌云忽然凝聚变形，瞬间化为了一个身高寸许的黑绿婴儿出来，并口吐人言的说道。";
|
||
//text = "艳美少妇打了韩立";
|
||
//text = "银月嫣然一笑后，敛冲元婴一拜的说道。这元婴看起来除了比韩立主元婴小了一些外，模样神态全都一般无二的样子。";
|
||
|
||
//List<string> sentiments = hanlp.GetSentiment(text);
|
||
|
||
SpeakerFeatureLearner speakerFeatureLearner = new SpeakerFeatureLearner();
|
||
BuildPersonNameRules(all.ToArray(), speakerFeatureLearner);
|
||
|
||
AiSentenceViewEntity[] sentenceTemps;
|
||
int paragraphIndex = 1;
|
||
foreach (var sentence in sentenceViews)
|
||
{
|
||
sentence.CreatedBy = sentence.UpdatedBy = string.Empty;
|
||
if (string.IsNullOrEmpty(sentence.Content)) continue;
|
||
|
||
text = sentence.Content;
|
||
|
||
List<NamedEntity> ners = hanlp.GetHanlpNer(text);
|
||
List<List<MeaningRepresentationParsingEntity>> srls = hanlp.GetHanlpSrl(text);
|
||
List<HanlpConstituencyNode> cons = hanlp.GetHanlpCon(text);
|
||
|
||
speakerFeatureLearner.AutoLearnFromText("", ners, srls, cons);
|
||
|
||
//如果不存在人名
|
||
if (ners == null || !ners.Where(x => x.Type.Contains("PERSON")).Any())
|
||
{
|
||
if (sentence.DialogueIndc == YesNoPolicy.NO)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
|
||
foreach (var p in sentenceViews.Where(x => x.ParagraphIndex == sentence.ParagraphIndex).ToArray())
|
||
{
|
||
if (string.IsNullOrEmpty(p.PersonName)) continue;
|
||
|
||
sentence.PersonName = p.PersonName;
|
||
}
|
||
}
|
||
|
||
string[] persons = ners.Where(x => x.Type.Contains("PERSON")).Select(x => x.Text).Distinct().ToArray();
|
||
if (persons == null && persons.Count() == 0) continue;
|
||
if (persons != null && persons.Count() == 1)
|
||
{
|
||
sentence.PersonName = speakerFeatureLearner.MatchSpeaker(persons.FirstOrDefault());
|
||
continue;
|
||
}
|
||
//如果是两个，取ARG0
|
||
sentence.PersonName = speakerFeatureLearner.MatchSpeaker(persons.FirstOrDefault());
|
||
}
|
||
|
||
List<NamedSummary> indexRange = GetNamedSummaryBefore(hanlp, data.Datas, 8);
|
||
|
||
|
||
|
||
//3字词根据次数划分到2字词内
|
||
python.Dispose();
|
||
}
|
||
|
||
public List<NamedSummary> GetNamedSummaryBefore(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int paragraphIndex)
|
||
{
|
||
return GetNamedSummary(hanlp, datas, paragraphIndex - 3 > 0 ? paragraphIndex - 3 : 0, paragraphIndex);
|
||
}
|
||
|
||
public List<NamedSummary> GetNamedSummaryAfter(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int paragraphIndex)
|
||
{
|
||
return GetNamedSummary(hanlp, datas, paragraphIndex, paragraphIndex + 3);
|
||
}
|
||
|
||
public List<NamedSummary> GetNamedSummary(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int startIndex, int endIndex)
|
||
{
|
||
List<NamedSummary> nameds = new List<NamedSummary>();
|
||
NamedSummary summary;
|
||
List<NamedEntity> temps;
|
||
AiSentenceViewEntity sentence;
|
||
for (int i = 0; i < datas.Length; i++)
|
||
{
|
||
sentence = datas[i];
|
||
if (sentence.ParagraphIndex < startIndex) continue;
|
||
if (sentence.ParagraphIndex > endIndex) continue;
|
||
|
||
temps = hanlp.GetHanlpNer(sentence.Content);
|
||
if (temps == null || temps.Count() == 0) continue;
|
||
|
||
foreach (var tp in temps)
|
||
{
|
||
if (tp.Text.Length == 1) continue;
|
||
if (tp.Type != "PERSON") continue;
|
||
if (nameds.Select(x => x.Person).Contains(tp.Text))
|
||
{
|
||
summary = nameds.Where(x => x.Person == tp.Text).FirstOrDefault();
|
||
summary.Count++;
|
||
continue;
|
||
}
|
||
summary = new NamedSummary();
|
||
summary.Person = tp.Text;
|
||
summary.Count = 1;
|
||
nameds.Add(summary);
|
||
}
|
||
}
|
||
|
||
return nameds;
|
||
}
|
||
|
||
/// <summary>
|
||
/// 处理 NamedSummary 数组：按包含关系分组，以次数多的为主名称调用 AddRule
|
||
/// </summary>
|
||
/// <param name="namedSummaries">人名统计数组</param>
|
||
public void BuildPersonNameRules(NamedSummary[] namedSummaries, SpeakerFeatureLearner speakerFeatureLearner)
|
||
{
|
||
// 空值校验
|
||
if (namedSummaries == null || !namedSummaries.Any())
|
||
{
|
||
Console.WriteLine("NamedSummary 数组为空，无需处理");
|
||
return;
|
||
}
|
||
|
||
// 1. 提取去重的人名-次数字典（确保每个人名只统计一次总次数）
|
||
var nameCountDict = namedSummaries
|
||
.Where(ns => !string.IsNullOrWhiteSpace(ns.Person))
|
||
.GroupBy(ns => ns.Person.Trim())
|
||
.ToDictionary(g => g.Key, g => g.Sum(ns => ns.Count));
|
||
|
||
if (nameCountDict.Count < 2)
|
||
{
|
||
Console.WriteLine("有效人名数量不足，无需建立映射");
|
||
return;
|
||
}
|
||
|
||
// 2. 标记已处理的人名，避免重复配对
|
||
var processedNames = new HashSet<string>();
|
||
|
||
// 3. 遍历查找包含关系的人名对，按次数确定主从
|
||
foreach (var (name1, count1) in nameCountDict)
|
||
{
|
||
if (processedNames.Contains(name1)) continue;
|
||
|
||
// 查找与当前人名存在包含关系的其他人名
|
||
var matchedNames = nameCountDict
|
||
.Where(kv => !processedNames.Contains(kv.Key)
|
||
&& kv.Key != name1
|
||
&& (kv.Key.Contains(name1) || name1.Contains(kv.Key))) // 互相包含都算匹配
|
||
.ToList();
|
||
|
||
if (matchedNames.Count == 0)
|
||
{
|
||
processedNames.Add(name1);
|
||
continue; // 无匹配的包含关系人名，跳过
|
||
}
|
||
|
||
// 4. 合并当前人名和匹配人名，按次数选主名称
|
||
foreach (var (name2, count2) in matchedNames)
|
||
{
|
||
// 确定主/次要名称：次数多的是主，次数相同则选长度长的（可选规则）
|
||
string mainName = count1 > count2 ? name1 : (count2 > count1 ? name2 : (name1.Length > name2.Length ? name1 : name2));
|
||
string minorName = mainName == name1 ? name2 : name1;
|
||
|
||
// 5. 调用 AddRule(次要名称, 主名称)
|
||
speakerFeatureLearner.AddRule(mainName, minorName);
|
||
Console.WriteLine($"已调用 AddRule：AddRule(\"{mainName}\", \"{minorName}\")");
|
||
|
||
// 标记为已处理，避免重复配对
|
||
processedNames.Add(name1);
|
||
processedNames.Add(name2);
|
||
}
|
||
}
|
||
}
|
||
|
||
public class NamedSummary
|
||
{
|
||
public int Count { get; set; }
|
||
|
||
public string Person { get; set; }
|
||
|
||
public decimal MinConfidence { get; set; }
|
||
}
|
||
}
|
||
}
|