ow
This commit is contained in:
commit
597b88d075
37 files changed
+3943
No files matched your search
@@ -0,0 +1,136 @@
|
||||
using CloudBuilder.AI.Data;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.Policy;
|
||||
using CloudBuilder.AI.Service;
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.DependencyInjection.Task;
|
||||
using CloudBuilder.Topshelf.Task.Novel.Process;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Transactions;
|
||||
using static System.Runtime.InteropServices.JavaScript.JSType;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel
|
||||
{
|
||||
//task:ImportBookTask encoding:GB2312 pattern: @"第([^】]+)章" input:D:\NET8\AI\仙侠\凡人修仙传_忘语.txt
|
||||
//input可以是文件名:D:\NET8\AI\仙侠\书名_作者.txt
|
||||
//input也可以是目錄名:D:\NET8\AI\仙侠,必须是小说类型为结尾
|
||||
//output輸出目錄
|
||||
//pattern可能自定義分割表達式
|
||||
public class ImportBookTask : IScheduleTask
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
public ImportBookTask(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public void Run(Dictionary<string, string> bodyDict)
|
||||
{
|
||||
if (!bodyDict.ContainsKey("input") || string.IsNullOrEmpty(bodyDict["input"])) return;
|
||||
|
||||
string input = bodyDict["input"];
|
||||
|
||||
string pattern = @"第([^】]+)章";
|
||||
if (bodyDict.ContainsKey("pattern") && !string.IsNullOrEmpty(bodyDict["pattern"]))
|
||||
pattern = bodyDict["pattern"];
|
||||
|
||||
string encoding = "UTF-8";
|
||||
|
||||
if (bodyDict.ContainsKey("encoding") && !string.IsNullOrEmpty(bodyDict["encoding"])) encoding = bodyDict["encoding"];
|
||||
|
||||
List<string> files = PackFiles(input);
|
||||
|
||||
foreach (string file in files)
|
||||
{
|
||||
ProcessBook(file, encoding, pattern);
|
||||
}
|
||||
}
|
||||
|
||||
public void ProcessBook(string file, string encoding, string pattern)
|
||||
{
|
||||
AiBookData data = new AiBookData();
|
||||
AiBookEntity book = new AiBookEntity();
|
||||
book.Guid = Guid.NewGuid().ToString();
|
||||
string[] temps = Path.GetFileNameWithoutExtension(file).Split('_');
|
||||
if (temps == null || temps.Length != 2)
|
||||
{
|
||||
Console.WriteLine("文件格式要求:书名_作者.txt");
|
||||
return;
|
||||
}
|
||||
|
||||
book.Title = temps[0];
|
||||
book.Author = temps[1];
|
||||
|
||||
temps = file.Split('\\');
|
||||
book.BookType = temps[temps.Length - 2];
|
||||
|
||||
//book.FilePath = file;
|
||||
book.Status = BookStatusPolicy.SUBMIT;//S-待提交
|
||||
|
||||
ChapterSplitter chapterSplitter = new ChapterSplitter();
|
||||
data.AiBook = book;
|
||||
|
||||
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
|
||||
|
||||
string context = File.ReadAllText(file, Encoding.GetEncoding(encoding));
|
||||
|
||||
List<AiChapterEntity> aiChapterEntities = chapterSplitter.SplitNovelIntoChapters(context, pattern);
|
||||
|
||||
if (aiChapterEntities == null || aiChapterEntities.Count() == 0) return;
|
||||
|
||||
aiChapterEntities.ToLookup(x => x.BookGuid = book.Guid);
|
||||
aiChapterEntities.ToLookup(x => x.Status = ChapterStatusPolicy.PROCESS);
|
||||
|
||||
data.AiChapters = aiChapterEntities.ToArray();
|
||||
|
||||
var transactionOptions = new TransactionOptions
|
||||
{
|
||||
IsolationLevel = IsolationLevel.ReadCommitted,
|
||||
Timeout = TimeSpan.FromSeconds(60 * 10)
|
||||
};
|
||||
|
||||
using (TransactionScope _transactionScope = new TransactionScope(
|
||||
TransactionScopeOption.Required,
|
||||
transactionOptions,
|
||||
TransactionScopeAsyncFlowOption.Enabled))
|
||||
{
|
||||
if (data.AiBook != null)
|
||||
service.ServiceProvider.GetService<IAiBookService>().Insert(data);
|
||||
|
||||
_transactionScope.Complete();
|
||||
}
|
||||
}
|
||||
|
||||
private List<string> PackFiles(string input)
|
||||
{
|
||||
List<string> files = new List<string>();
|
||||
if (Path.HasExtension(input) && !string.IsNullOrEmpty(input))
|
||||
{
|
||||
if (File.Exists(input))
|
||||
{
|
||||
files.Add(input);
|
||||
}
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(input))
|
||||
{
|
||||
if (Directory.Exists(input))
|
||||
{
|
||||
string[] tempFiles = Directory.GetFiles(input);
|
||||
if (tempFiles != null && tempFiles.Length > 0)
|
||||
{
|
||||
files.AddRange(tempFiles);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
using CloudBuilder.AI.Data;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.Service;
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.DependencyInjection.Task;
|
||||
using CloudBuilder.Topshelf.Task.Novel.Process;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using System.Transactions;
|
||||
using static System.Runtime.InteropServices.JavaScript.JSType;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel
|
||||
{
|
||||
//task:ParagraphSplitterTask book_title:凡人修仙传
|
||||
public class ParagraphSplitterTask : IScheduleTask
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
public ParagraphSplitterTask(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public void Run(Dictionary<string, string> bodyDict)
|
||||
{
|
||||
if (!bodyDict.ContainsKey("book_title") || string.IsNullOrEmpty(bodyDict["book_title"])) return;
|
||||
|
||||
string title = bodyDict["book_title"];
|
||||
|
||||
AiChapterEntity[] aiChapters = service.ServiceProvider.GetService<IAiChapterService>().FindByTitle(title);
|
||||
|
||||
if (aiChapters == null || aiChapters.Length == 0) return;
|
||||
|
||||
ParagraphSplitter paragraphSplitter = new ParagraphSplitter();
|
||||
List<AiParagraphEntity> aiParagraphs = new List<AiParagraphEntity>();
|
||||
List<AiSentenceEntity> aiSentences = new List<AiSentenceEntity>();
|
||||
AiBookData data;
|
||||
foreach (var cpt in aiChapters)
|
||||
{
|
||||
data = paragraphSplitter.SplitChapterIntoParagraphs(cpt);
|
||||
if (data.AiParagraphs == null || data.AiParagraphs.Count() == 0) continue;
|
||||
|
||||
aiParagraphs.AddRange(data.AiParagraphs);
|
||||
}
|
||||
|
||||
if (aiParagraphs == null || aiParagraphs.Count() == 0) return;
|
||||
|
||||
SentenceSplitter sentenceSplitter = new SentenceSplitter();
|
||||
AiSentenceEntity aiSentence;
|
||||
|
||||
foreach (var ph in aiParagraphs)
|
||||
{
|
||||
if (ph.DialogueIndc != YesNoPolicy.YES)
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = ph.Content;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.NO;
|
||||
aiSentence.SentenceIndex = 1;
|
||||
aiSentence.ParagraphGuid = ph.Guid;
|
||||
aiSentence.Status = "G";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
aiSentences.Add(aiSentence);
|
||||
continue;
|
||||
}
|
||||
|
||||
var s = sentenceSplitter.SplitByConfigurableQuotes(ph);
|
||||
if (s == null || s.Count() == 0) continue;
|
||||
|
||||
aiSentences.AddRange(s);
|
||||
}
|
||||
|
||||
var transactionOptions = new TransactionOptions
|
||||
{
|
||||
IsolationLevel = IsolationLevel.ReadCommitted,
|
||||
Timeout = TimeSpan.FromSeconds(60 * 10)
|
||||
};
|
||||
|
||||
using (TransactionScope _transactionScope = new TransactionScope(
|
||||
TransactionScopeOption.Required,
|
||||
transactionOptions,
|
||||
TransactionScopeAsyncFlowOption.Enabled))
|
||||
{
|
||||
service.ServiceProvider.GetService<IAiParagraphService>().Inserts(aiParagraphs.ToArray());
|
||||
service.ServiceProvider.GetService<IAiSentenceService>().Inserts(aiSentences.ToArray());
|
||||
|
||||
_transactionScope.Complete();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// 章节分割器
|
||||
/// 不同的小说有自己的分割方法,可以传入参数
|
||||
/// </summary>
|
||||
public class ChapterSplitter
|
||||
{
|
||||
public ChapterSplitter()
|
||||
{
|
||||
|
||||
}
|
||||
/// <summary>
|
||||
/// 按章节分割小说文本
|
||||
/// </summary>
|
||||
public List<AiChapterEntity> SplitNovelIntoChapters(string novelText, string chapterPatterns)
|
||||
{
|
||||
var chapters = new List<AiChapterEntity>();
|
||||
var lines = novelText.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries);
|
||||
var currentAiChapterEntity = new StringBuilder();
|
||||
var currentAiChapterEntityTitle = "未命名章节";
|
||||
var chapterCount = 1;
|
||||
|
||||
foreach (var line in lines)
|
||||
{
|
||||
// 检查是否是章节标题
|
||||
if (IsChapterTitle(line, chapterPatterns))
|
||||
{
|
||||
// 如果当前章节有内容,则保存
|
||||
if (currentAiChapterEntity.Length > 0)
|
||||
{
|
||||
chapters.Add(new AiChapterEntity
|
||||
{
|
||||
Title = currentAiChapterEntityTitle,
|
||||
Guid = Guid.NewGuid().ToString(),
|
||||
Content = currentAiChapterEntity.ToString(),
|
||||
ChapterIndex = chapterCount
|
||||
});
|
||||
currentAiChapterEntity.Clear();
|
||||
chapterCount++;
|
||||
}
|
||||
|
||||
// 更新章节标题
|
||||
currentAiChapterEntityTitle = line.Trim();
|
||||
// 添加章节标题
|
||||
currentAiChapterEntity.AppendLine(line.Trim());
|
||||
// 添加一个空行作为分隔,确保章节标题和正文内容分离
|
||||
currentAiChapterEntity.AppendLine();
|
||||
}
|
||||
else
|
||||
{
|
||||
// 添加到当前章节,确保每行独立
|
||||
currentAiChapterEntity.AppendLine(line);
|
||||
}
|
||||
}
|
||||
|
||||
// 添加最后一个章节
|
||||
if (currentAiChapterEntity.Length > 0)
|
||||
{
|
||||
chapters.Add(new AiChapterEntity
|
||||
{
|
||||
Title = currentAiChapterEntityTitle,
|
||||
Content = currentAiChapterEntity.ToString(),
|
||||
Guid = Guid.NewGuid().ToString(),
|
||||
ChapterIndex = chapterCount
|
||||
});
|
||||
}
|
||||
|
||||
return chapters;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 检查是否为章节标题
|
||||
/// </summary>
|
||||
private bool IsChapterTitle(string line, string cptPattern)
|
||||
{
|
||||
// 去除首尾空白字符
|
||||
var trimmedLine = line.Trim();
|
||||
|
||||
// 常见的章节标题模式
|
||||
var chapterPatterns = new[]
|
||||
{
|
||||
@"^第\s*[\u4e00-\u9fa50-9]+\s*章", // 第X章, 第XX章等
|
||||
@"^第\s*[0-9]+\s*章", // 第1章, 第2章等
|
||||
};
|
||||
|
||||
if (!string.IsNullOrEmpty(cptPattern))
|
||||
{
|
||||
Array.Resize(ref chapterPatterns, chapterPatterns.Length + 1);
|
||||
chapterPatterns[chapterPatterns.Length - 1] = cptPattern;
|
||||
}
|
||||
|
||||
foreach (var pattern in chapterPatterns)
|
||||
{
|
||||
if (Regex.IsMatch(trimmedLine, pattern, RegexOptions.IgnoreCase))
|
||||
{
|
||||
//第一章内容说明了什么,请描述一下:
|
||||
if (trimmedLine.Length > 20) return false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 保存章节到单独的文件
|
||||
/// </summary>
|
||||
public void SaveChaptersToFiles(List<AiChapterEntity> chapters, string outputDirectory)
|
||||
{
|
||||
// 确保输出目录存在
|
||||
if (!Directory.Exists(outputDirectory))
|
||||
{
|
||||
Directory.CreateDirectory(outputDirectory);
|
||||
}
|
||||
|
||||
foreach (var chapter in chapters)
|
||||
{
|
||||
// 使用章节标题作为文件名,清理非法字符
|
||||
var fileName = CleanFileName(chapter.Title) + ".txt";
|
||||
var filePath = Path.Combine(outputDirectory, fileName);
|
||||
File.WriteAllText(filePath, chapter.Content, Encoding.UTF8);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 清理文件名中的非法字符
|
||||
/// </summary>
|
||||
private string CleanFileName(string fileName)
|
||||
{
|
||||
var invalidChars = Path.GetInvalidFileNameChars();
|
||||
foreach (var invalidChar in invalidChars)
|
||||
{
|
||||
fileName = fileName.Replace(invalidChar, '_');
|
||||
}
|
||||
|
||||
// 限制文件名长度
|
||||
if (fileName.Length > 100)
|
||||
{
|
||||
fileName = fileName.Substring(0, 100);
|
||||
}
|
||||
|
||||
return fileName;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using CloudBuilder.AI.Data;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel.Process
|
||||
{
|
||||
public class ParagraphSplitter
|
||||
{
|
||||
public ParagraphSplitter()
|
||||
{
|
||||
}
|
||||
|
||||
public AiBookData SplitChapterIntoParagraphs(AiChapterEntity aiChapter)
|
||||
{
|
||||
|
||||
AiBookData data = new AiBookData();
|
||||
|
||||
var paragraphs = new List<AiParagraphEntity>();
|
||||
var sentences = new List<AiSentenceEntity>();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(aiChapter.Content)) return data;
|
||||
|
||||
// 按换行拆分段落(过滤空段落)
|
||||
var paragraphTexts = aiChapter.Content.Split("\r\n")
|
||||
.Select(p => p.Trim())
|
||||
.Where(p => !string.IsNullOrWhiteSpace(p))
|
||||
.ToList();
|
||||
|
||||
SentenceSplitter sentenceSplitter = new SentenceSplitter();
|
||||
|
||||
AiParagraphEntity aiParagraph;
|
||||
int index = 1;
|
||||
foreach (var text in paragraphTexts)
|
||||
{
|
||||
if (!ValidText(text)) continue;
|
||||
|
||||
aiParagraph = new AiParagraphEntity();
|
||||
aiParagraph.Content = text;
|
||||
aiParagraph.ChapterGuid = aiChapter.Guid;
|
||||
aiParagraph.Guid = Guid.NewGuid().ToString();
|
||||
aiParagraph.ParagraphIndex = index++;
|
||||
aiParagraph.DialogueIndc = sentenceSplitter.IsDialogue(text) ? YesNoPolicy.YES : YesNoPolicy.NO;
|
||||
paragraphs.Add(aiParagraph);
|
||||
}
|
||||
|
||||
data.AiParagraphs = paragraphs.ToArray();
|
||||
data.AiSentences = sentences.ToArray();
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
private bool ValidText(string text)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(text.Trim())) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
using CloudBuilder.AI.Entity;
|
||||
using Microsoft.EntityFrameworkCore.Metadata.Internal;
|
||||
using Microsoft.Extensions.FileSystemGlobbing;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel.Process
|
||||
{
|
||||
/// <summary>
|
||||
/// 智能句子分割器
|
||||
/// </summary>
|
||||
public class SentenceSplitter
|
||||
{
|
||||
// 成对引号配置数组(可自由扩展,格式:[左引号, 右引号])
|
||||
private readonly List<(string LeftQuote, string RightQuote)> _quotePairs;
|
||||
// 动态生成的正则表达式(匹配所有配置的引号类型)
|
||||
private Regex _dynamicQuotePattern;
|
||||
// 引号内必须包含的结尾标点集合
|
||||
private readonly string _endPunctuations = "?!。,;:、……~·";
|
||||
|
||||
/// <summary>
|
||||
/// 默认构造函数:初始化常用成对引号(中文、英文、直角引号)
|
||||
/// </summary>
|
||||
public SentenceSplitter()
|
||||
{
|
||||
_quotePairs = new List<(string, string)>
|
||||
{
|
||||
("“", "”"), // 中文弯引号
|
||||
("\"", "\""), // 英文双引号(需转义)
|
||||
("〝", "〞"), // 中文直角引号
|
||||
("「", "」") // 中文方头引号(可选扩展)
|
||||
};
|
||||
InitDynamicRegex();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 自定义构造函数:允许外部传入任意成对引号配置
|
||||
/// </summary>
|
||||
/// <param name="customQuotePairs">自定义成对引号列表</param>
|
||||
public SentenceSplitter(List<(string LeftQuote, string RightQuote)> customQuotePairs)
|
||||
{
|
||||
_quotePairs = customQuotePairs ?? new List<(string, string)>();
|
||||
InitDynamicRegex();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 初始化动态正则:根据配置的成对引号生成匹配规则
|
||||
/// </summary>
|
||||
private void InitDynamicRegex()
|
||||
{
|
||||
if (_quotePairs.Count == 0)
|
||||
{
|
||||
// 空配置时生成空正则,避免匹配任何内容
|
||||
_dynamicQuotePattern = new Regex(@"^$", RegexOptions.Compiled);
|
||||
return;
|
||||
}
|
||||
|
||||
// 步骤1:转义所有引号(避免正则特殊字符冲突),拼接左/右引号匹配组
|
||||
string escapedLeftQuotes = string.Join("|", _quotePairs.Select(p => Regex.Escape(p.LeftQuote)));
|
||||
string escapedRightQuotes = string.Join("|", _quotePairs.Select(p => Regex.Escape(p.RightQuote)));
|
||||
// 步骤2:转义标点集合,避免正则解析错误
|
||||
string escapedPunctuations = Regex.Escape(_endPunctuations);
|
||||
|
||||
// 步骤3:构建动态正则表达式
|
||||
// 分组说明:
|
||||
// ?<left>:匹配配置的任意左引号
|
||||
// ?<content>:匹配引号内的内容(必须以指定标点结尾)
|
||||
// ?<right>:匹配配置的任意右引号
|
||||
string regexPattern = $@"(?<left>{escapedLeftQuotes})(?<content>.+?[{escapedPunctuations}])(?<right>{escapedRightQuotes})";
|
||||
_dynamicQuotePattern = new Regex(regexPattern, RegexOptions.Compiled | RegexOptions.Singleline);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 核心分割方法:按配置的成对引号分割文本,结果无引号字符
|
||||
/// </summary>
|
||||
/// <param name="input">待分割的文本(如:“其他标点符号,如?”紫茜的秀目 或 "其他标点符号,如?"紫茜的秀目)</param>
|
||||
/// <returns>分割后的纯文本列表</returns>
|
||||
public List<AiSentenceEntity> SplitByConfigurableQuotes(AiParagraphEntity aiParagraphEntity)
|
||||
{
|
||||
AiSentenceEntity aiSentence;
|
||||
int sentenceIndex = 1;
|
||||
// 空值/空白校验
|
||||
if (string.IsNullOrWhiteSpace(aiParagraphEntity.Content))
|
||||
{
|
||||
return new List<AiSentenceEntity>();
|
||||
}
|
||||
|
||||
List<AiSentenceEntity> result = new List<AiSentenceEntity>();
|
||||
string remainingText = aiParagraphEntity.Content.Trim();
|
||||
int lastMatchEnd = 0;
|
||||
|
||||
// 遍历所有匹配的「引号+内容+标点+引号」文本块
|
||||
foreach (Match match in _dynamicQuotePattern.Matches(remainingText))
|
||||
{
|
||||
// 1. 提取匹配的左/右引号,校验是否为成对配置
|
||||
string matchedLeft = match.Groups["left"].Value;
|
||||
string matchedRight = match.Groups["right"].Value;
|
||||
bool isPairMatched = _quotePairs.Any(p => p.LeftQuote == matchedLeft && p.RightQuote == matchedRight);
|
||||
if (!isPairMatched)
|
||||
{
|
||||
continue; // 非成对引号,跳过该匹配(避免“xxx"这类混合匹配)
|
||||
}
|
||||
|
||||
// 2. 提取引号前的文本(过滤空值)
|
||||
string prefixText = remainingText.Substring(lastMatchEnd, match.Index - lastMatchEnd).Trim();
|
||||
if (!string.IsNullOrEmpty(prefixText))
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = prefixText;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.NO;
|
||||
aiSentence.SentenceIndex = sentenceIndex++;
|
||||
aiSentence.ParagraphGuid = aiParagraphEntity.Guid;
|
||||
aiSentence.Status = "G";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
result.Add(aiSentence);
|
||||
}
|
||||
|
||||
// 3. 提取引号内的纯内容(无引号字符)
|
||||
string quotedContent = match.Groups["content"].Value.Trim();
|
||||
if (!string.IsNullOrEmpty(quotedContent))
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = quotedContent;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.YES;
|
||||
aiSentence.SentenceIndex = sentenceIndex++;
|
||||
aiSentence.ParagraphGuid = aiParagraphEntity.Guid;
|
||||
aiSentence.Status = "P";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
result.Add(aiSentence);
|
||||
}
|
||||
|
||||
// 更新最后匹配位置
|
||||
lastMatchEnd = match.Index + match.Length;
|
||||
}
|
||||
|
||||
// 4. 提取最后一个匹配项后的剩余文本
|
||||
string suffixText = remainingText.Substring(lastMatchEnd).Trim();
|
||||
if (!string.IsNullOrEmpty(suffixText))
|
||||
{
|
||||
aiSentence = new AiSentenceEntity();
|
||||
aiSentence.Guid = Guid.NewGuid().ToString();
|
||||
aiSentence.Content = suffixText;
|
||||
aiSentence.DialogueIndc = YesNoPolicy.NO;
|
||||
aiSentence.SentenceIndex = sentenceIndex++;
|
||||
aiSentence.ParagraphGuid = aiParagraphEntity.Guid;
|
||||
aiSentence.Status = "G";
|
||||
//D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】
|
||||
result.Add(aiSentence);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 新增:判断文本是否包含对话内容
|
||||
/// </summary>
|
||||
/// <param name="text">待判断的文本(句子/段落)</param>
|
||||
/// <returns>true=包含对话,false=纯旁白/空文本</returns>
|
||||
public bool IsDialogue(string text)
|
||||
{
|
||||
// 1. 处理空文本/NULL,直接返回false
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
return false;
|
||||
|
||||
// 2. 文本预处理:去空格、统一全角/半角引号
|
||||
string cleanedText = text.Trim();
|
||||
// 替换全角空格为半角,避免干扰判断
|
||||
cleanedText = Regex.Replace(cleanedText, @" ", " ");
|
||||
// 统一引号格式(中文弯引号、直引号 → 标准双引号)
|
||||
cleanedText = cleanedText.Replace("“", "\"")
|
||||
.Replace("”", "\"")
|
||||
.Replace("〝", "\"")
|
||||
.Replace("〞", "\"")
|
||||
.Replace("「", "\"")
|
||||
.Replace("」", "\"")
|
||||
.Replace("‘", "'")
|
||||
.Replace("’", "'");
|
||||
|
||||
// 3. 核心判断规则:
|
||||
// - 包含成对/单个双引号(排除仅含引号无内容的情况)
|
||||
// - 包含冒号+引号(如:"XXX说:"XXX"" 场景)
|
||||
var hasQuote = Regex.IsMatch(cleanedText, @""".+?""") || // 成对双引号(含内容)
|
||||
Regex.IsMatch(cleanedText, @":\s*"""); // 冒号后接引号(说话人+对话)
|
||||
var hasSingleQuote = Regex.IsMatch(cleanedText, @"'[^']+'"); // 成对单引号(英文对话)
|
||||
|
||||
// 4. 排除仅含引号符号的无效情况(如 ""、''、“”)
|
||||
var onlyQuote = Regex.IsMatch(cleanedText, @"^\s*[""''“”]{1,2}\s*$");
|
||||
|
||||
// 最终判断:包含有效引号且不是仅含引号 → 判定为对话
|
||||
return (hasQuote || hasSingleQuote) && !onlyQuote;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel.Process
|
||||
{
|
||||
public class SpeakerAnalysisHelper
|
||||
{
|
||||
public string GetPerson(string depRoot, List<NamedEntity> ners, List<List<MeaningRepresentationParsingEntity>> srls, List<HanlpConstituencyNode> cons)
|
||||
{
|
||||
string arg0 = string.Empty;
|
||||
|
||||
if (!string.IsNullOrEmpty(depRoot) && srls != null && srls.Count() > 0)
|
||||
{
|
||||
foreach (var srl in srls)
|
||||
{
|
||||
if (srl.Where(x => x.Text == depRoot).Any())
|
||||
{
|
||||
foreach (var mrp in srl)
|
||||
{
|
||||
if (mrp.Type.ToUpper() != "ARG0") continue;
|
||||
|
||||
if (ners != null && ners.Where(x => mrp.Text.Contains(x.Text)).Any() &&
|
||||
ners.Where(x => mrp.Text.Contains(x.Text)).FirstOrDefault().Text.Length > 1)
|
||||
return ners.Where(x => mrp.Text.Contains(x.Text)).FirstOrDefault().Text;
|
||||
|
||||
return mrp.Text.Length > 6 ? mrp.Text.Substring(mrp.Text.Length - 6, 6) : mrp.Text;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var mrp in srl)
|
||||
{
|
||||
if (mrp.Type.ToUpper() != "ARG0") continue;
|
||||
arg0 = mrp.Text.Length > 6 ? mrp.Text.Substring(mrp.Text.Length - 6, 6) : mrp.Text;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (ners != null && ners.Count() > 0)
|
||||
return ners.FirstOrDefault().Text;
|
||||
|
||||
return arg0;
|
||||
}
|
||||
|
||||
public AiParagraphEntity[] AiParagraphs { get; set; }
|
||||
|
||||
public AiSentenceEntity[] AiSentences { get; set; }
|
||||
|
||||
public AiChapterPersonEntity[] AiChapterPersons { get; set; }
|
||||
}
|
||||
}
|
||||
/*
|
||||
“文本解析→人物提取→上下文匹配→结果输出”
|
||||
|
||||
graph TD
|
||||
A[对话行] --> B{规则1:文本特征匹配};
|
||||
B -->|匹配成功| C[直接赋值说话者];
|
||||
B -->|匹配失败| D{规则2:紧邻旁白匹配};
|
||||
D -->|匹配成功| C;
|
||||
D -->|匹配失败| E{规则3:段落内最近人物};
|
||||
E -->|匹配成功| C;
|
||||
E -->|匹配失败| F[赋值默认值+标记异常];
|
||||
|
||||
规则 1:文本特征精准匹配(最高优先级)
|
||||
基于对话内容中的 “专属特征” 直接匹配说话者,避免上下文干扰:
|
||||
示例规则:
|
||||
对话内容特征 匹配的说话者
|
||||
包含 “第二元婴” 韩立
|
||||
包含 “银月所化” 银月
|
||||
包含 “本座 / 本尊” 反派角色
|
||||
|
||||
落地:维护一个 “特征 - 人物” 映射字典,匹配时优先使用:
|
||||
规则 2:紧邻旁白匹配(次高优先级)只找对话行 “紧邻的前 1-2 行旁白”,而非整个段落的旁白,避免跨越多个人物导致匹配错误:
|
||||
规则:仅检查对话行的前 1 行、前 2 行旁白(最多前 3 行),超过则停止;
|
||||
落地:修改向前查找的逻辑,限制查找范围
|
||||
|
||||
规则 3:排除无效人物(避免干扰)从旁白中提取人物时,过滤掉 “非核心人物”(如 “店小二”“路人”),只保留主要人物:
|
||||
落地:维护 “核心人物白名单”,提取人物时只保留白名单内的名称
|
||||
|
||||
3. 增加异常处理和人工校验(兜底保障)
|
||||
即使逻辑再完善,也会有边缘场景匹配错误,需通过 “异常标记 + 人工复核” 兜底:
|
||||
|
||||
步骤 1:标记可疑匹配结果对以下情况标记为 “可疑”,存入日志或单独字段:
|
||||
匹配到的人物在旁白中出现次数≥2(比如同一段落有韩立和银月,无法确定);
|
||||
对话行前后 3 行都没有找到人物,最终赋值为 “未知人物”;
|
||||
对话内容无任何特征,且段落内人物数量≥3;
|
||||
落地:为实体增加扩展字段(如IsSuspected),或生成异常日志
|
||||
|
||||
步骤 2:人工复核可疑数据
|
||||
规则:所有标记为 “可疑” 的匹配结果,必须经过人工复核后再更新到数据库;
|
||||
落地:生成 “可疑数据清单”(包含Guid、Content、PersonName),供运营 / 编辑人员校验,修正错误后重新填充。
|
||||
|
||||
|
||||
收集错误案例:记录每次匹配错误的场景(如 “银月的对话匹配给韩立”),分析错误原因;
|
||||
迭代特征字典:将错误案例转化为新的特征规则(如新增 “乌云”→银月,“元婴”→韩立);
|
||||
A/B 测试:对优化后的规则做小范围测试,对比匹配准确率,确认有效后全量上线。
|
||||
*/
|
||||
@@ -0,0 +1,148 @@
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using Microsoft.VisualBasic;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Reflection.Metadata;
|
||||
using System.Runtime.Intrinsics.Arm;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel.Process
|
||||
{
|
||||
public class SpeakerFeatureLearner
|
||||
{
|
||||
private readonly List<SpeakerRule> _rules;
|
||||
public SpeakerFeatureLearner()
|
||||
{
|
||||
_rules = new List<SpeakerRule>();
|
||||
InitializeDefaultRules();
|
||||
}
|
||||
|
||||
private void InitializeDefaultRules()
|
||||
{
|
||||
//AddRule("银月", "少妇");
|
||||
}
|
||||
public void AutoLearnFromText(string text)
|
||||
{
|
||||
|
||||
}
|
||||
public void AutoLearnFromText(string depRoot, List<NamedEntity> ners, List<List<MeaningRepresentationParsingEntity>> srls, List<HanlpConstituencyNode> cons)
|
||||
{
|
||||
|
||||
|
||||
if (ners == null || !ners.Where(x => x.Type.Trim().ToUpper().Contains("PERSON")).Any()) return;
|
||||
|
||||
HanlpConstituencyNode[] nodes;
|
||||
HanlpConstituencyNode nodef;
|
||||
string feature = null;
|
||||
foreach (NamedEntity namedEntity in ners.Where(x => x.Type.Trim().ToUpper().Contains("PERSON")).ToArray())
|
||||
{
|
||||
nodes = HanlpResultHelper.GetNodeByLabel(cons, namedEntity.Text);
|
||||
if (nodes == null || nodes.Length == 0) continue;
|
||||
|
||||
//如果存在,找到父节点上的NP
|
||||
foreach (HanlpConstituencyNode node in nodes)
|
||||
{
|
||||
nodef = HanlpResultHelper.GetNodeTopLabelById(cons, node.FatherId, "NP");
|
||||
if (nodef == null) continue;
|
||||
feature = HanlpResultHelper.GetNodeBottomLabelById(cons, nodef.ItemId, "NP");
|
||||
if (string.IsNullOrEmpty(feature)) continue;
|
||||
if (namedEntity.Text == feature) continue;
|
||||
|
||||
AddRule(namedEntity.Text, feature);
|
||||
}
|
||||
}
|
||||
|
||||
//自学创建特征
|
||||
//NR->NP->NP->IP
|
||||
//NN->NP(称呼、职业)
|
||||
|
||||
//完整汉语句法树库核心标签对照表(含 TOP/ NP / IP / CLP)
|
||||
//按 “核心层级 / 短语类型” 分类整理,覆盖 90 % 以上高频标注:
|
||||
//1.顶层 / 句子级标签
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//TOP Top - level Phrase 顶层短语 句法树的根节点,包含整个句子的最高层级结构 所有句子的根节点均标注为 TOP
|
||||
//IP Inflectional Phrase / Simple Clause 简单从句 句子级核心短语(含主谓结构),是 TOP 的直接子节点 “我吃饭” 整体标注为 IP
|
||||
//CP Complement Clause 补语从句 带补语标记(的 / 吗 / 呢)的从句 / 复句成分 “我买的书” 中 “我买的” 标注为 CP
|
||||
//2.名词类短语
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//NP Noun Phrase 名词短语 以名词为核心的短语(可含定语 / 量词) “苹果”“我的手机”“三本书”
|
||||
//DNP XP+DEG Phrase “的” 字短语 由 “XP + 的” 构成的名词性短语 “红色的”“我喜欢的”
|
||||
//NNP Proper Noun Phrase 专有名词短语 人名 / 地名 / 机构名等专有名词构成的短语 “北京大学”“张三”“北京市”
|
||||
//3.动词类短语
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//VP Verb Phrase 动词短语 以动词为核心的短语(可含宾语 / 状语) “吃饭”“快速跑步”“看电影”
|
||||
//VV Verb Phrase(Verb Core) 动词核心短语 纯动词构成的短语(无附加成分) “跑”“思考”“购买”
|
||||
//VCP Verb Complement Phrase 动补短语 动词 +补语构成的短语 “吃完”“跑快”“看明白”
|
||||
//4.数量 / 量词类短语
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//CLP Classifier Phrase 量词短语 以量词为核心的短语(数词 + 量词 / 指示词 + 量词) “三个”“这本”“那些”
|
||||
//QP Quantifier Phrase 量化短语 以数量为核心的短语(涵盖数词 / 量词 / 数量整体) “三”“若干”“三本”“许多”
|
||||
//DP Determiner Phrase 指示词短语 以指示词(这 / 那 / 此)为核心的短语 “这个”“那些”“此问题”
|
||||
//5.介词 / 方位类短语
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//PP Prepositional Phrase 介词短语 以介词为核心的短语 “在学校”“关于这件事”“把书”
|
||||
//LCP Locative Complement Phrase 方位补语短语 含方位词的短语 “桌子上”“房间里”“马路边”
|
||||
//LOC Locative Phrase 方位短语 纯方位名词构成的短语(LCP 的简化版) “上面”“左边”“内部”
|
||||
//6.形容词 / 副词类短语
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//AP Adjective Phrase 形容词短语 以形容词为核心的短语 “漂亮”“非常开心”“红色的”
|
||||
//ADVP Adverb Phrase 副词短语 以副词为核心的短语 “快速地”“很”“非常”
|
||||
//7.其他功能类短语
|
||||
//标签 英文全称 中文名称 定义 示例
|
||||
//INTJ Interjection Phrase 感叹语短语 感叹词 / 语气词构成的短语 “啊”“哦”“哎呀”
|
||||
//CONJP Conjunction Phrase 连词短语 连词构成的短语 “和”“但是”“因为”
|
||||
//PRN Parenthetical Phrase 插入语短语 插入句子的附加成分 “换句话说”“总的来说”
|
||||
//总结
|
||||
//CLP 核心是量词短语,以量词为核心(如 “三本”“这本”),是数量表达的核心标注标签;
|
||||
//TOP / IP 是句子层级标签(根节点 / 核心从句),NP / VP 是核心短语标签(名词 / 动词为核心);
|
||||
//不同工具(CTB / HanLP / 北大树库)对标签的细分略有差异(如 CLP 是否归为 QP),但核心含义一致,优先以宾州中文树库(CTB)规范为准。
|
||||
}
|
||||
|
||||
public void AddRule(string speaker, params string[] keywords)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(speaker))
|
||||
throw new ArgumentException("说话者名称不能为空", nameof(speaker));
|
||||
|
||||
if (keywords == null || keywords.Length == 0)
|
||||
throw new ArgumentException("至少需要提供一个关键词", nameof(keywords));
|
||||
|
||||
_rules.Add(new SpeakerRule(speaker, keywords));
|
||||
}
|
||||
|
||||
public string MatchSpeaker(string dialogue)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(dialogue))
|
||||
return null;
|
||||
|
||||
foreach (var rule in _rules)
|
||||
{
|
||||
if (rule.Match(dialogue))
|
||||
{
|
||||
return rule.Speaker;
|
||||
}
|
||||
}
|
||||
|
||||
return dialogue;
|
||||
}
|
||||
}
|
||||
|
||||
public class SpeakerRule
|
||||
{
|
||||
public string Speaker { get; }
|
||||
public string[] Keywords { get; }
|
||||
|
||||
public SpeakerRule(string speaker, string[] keywords)
|
||||
{
|
||||
Speaker = speaker;
|
||||
Keywords = keywords;
|
||||
}
|
||||
|
||||
public bool Match(string dialogue)
|
||||
{
|
||||
return Keywords.Any(keyword =>
|
||||
dialogue.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) >= 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
using CloudBuilder.AI.Data;
|
||||
using CloudBuilder.AI.Entity;
|
||||
using CloudBuilder.AI.SearchCriteria;
|
||||
using CloudBuilder.AI.Service;
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.DependencyInjection.Task;
|
||||
using CloudBuilder.Core.Service;
|
||||
using CloudBuilder.Security.Entity;
|
||||
using CloudBuilder.Topshelf.Python;
|
||||
using CloudBuilder.Topshelf.Task.Novel.Process;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using static System.Net.Mime.MediaTypeNames;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel
|
||||
{
|
||||
//task:SpeakerTask param_name:param_value
|
||||
public class SpeakerTask : IScheduleTask
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
public SpeakerTask(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public void Run(Dictionary<string, string> bodyDict)
|
||||
{
|
||||
AiSentenceSearchCriteria c = new AiSentenceSearchCriteria();
|
||||
c.limit = 999999;
|
||||
List<SearchFilter> filters = new List<SearchFilter>();
|
||||
|
||||
SearchFilter filter = new SearchFilter();
|
||||
filter.Column = AiSentenceViewEntity.DB_CHAPTER_GUID;
|
||||
filter.Type = SearchFilterTypePolicy.TEXT;
|
||||
filter.Operator = SearchFilterOperatorPolicy.CONTAIN;
|
||||
filter.Value = "cf34136e-d30a-4901-9109-626e412eaf57";
|
||||
filters.Add(filter);
|
||||
|
||||
c.Filter = filters.ToArray();
|
||||
|
||||
AiSentenceViewResultData data = service.ServiceProvider.GetService<IAiSentenceService>().FindBySearchCriteria(c);
|
||||
|
||||
if (data == null || data.Datas == null || data.Datas.Count() == 0) return;
|
||||
|
||||
AiSentenceViewEntity[] sentenceViews = data.Datas.OrderBy(x => x.ParagraphIndex).ThenBy(x => x.DialogueIndc).ToArray();
|
||||
|
||||
if (sentenceViews == null || sentenceViews.Length == 0) return;
|
||||
|
||||
PythonCommand python = new PythonCommand(service);
|
||||
|
||||
Hanlp2 hanlp = new Hanlp2(python);
|
||||
|
||||
List<NamedSummary> all = GetNamedSummary(hanlp, data.Datas, 0, data.Datas.Length);
|
||||
|
||||
string text = string.Empty;
|
||||
//@"乌云忽然凝聚变形,瞬间化为了一个身高寸许的黑绿婴儿出来,并口吐人言的说道。";
|
||||
//text = "艳美少妇打了韩立";
|
||||
//text = "银月嫣然一笑后,敛冲元婴一拜的说道。这元婴看起来除了比韩立主元婴小了一些外,模样神态全都一般无二的样子。";
|
||||
|
||||
//List<string> sentiments = hanlp.GetSentiment(text);
|
||||
|
||||
SpeakerFeatureLearner speakerFeatureLearner = new SpeakerFeatureLearner();
|
||||
BuildPersonNameRules(all.ToArray(), speakerFeatureLearner);
|
||||
|
||||
AiSentenceViewEntity[] sentenceTemps;
|
||||
int paragraphIndex = 1;
|
||||
foreach (var sentence in sentenceViews)
|
||||
{
|
||||
sentence.CreatedBy = sentence.UpdatedBy = string.Empty;
|
||||
if (string.IsNullOrEmpty(sentence.Content)) continue;
|
||||
|
||||
text = sentence.Content;
|
||||
|
||||
List<NamedEntity> ners = hanlp.GetHanlpNer(text);
|
||||
List<List<MeaningRepresentationParsingEntity>> srls = hanlp.GetHanlpSrl(text);
|
||||
List<HanlpConstituencyNode> cons = hanlp.GetHanlpCon(text);
|
||||
|
||||
speakerFeatureLearner.AutoLearnFromText("", ners, srls, cons);
|
||||
|
||||
//如果不存在人名
|
||||
if (ners == null || !ners.Where(x => x.Type.Contains("PERSON")).Any())
|
||||
{
|
||||
if (sentence.DialogueIndc == YesNoPolicy.NO)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
foreach (var p in sentenceViews.Where(x => x.ParagraphIndex == sentence.ParagraphIndex).ToArray())
|
||||
{
|
||||
if (string.IsNullOrEmpty(p.PersonName)) continue;
|
||||
|
||||
sentence.PersonName = p.PersonName;
|
||||
}
|
||||
}
|
||||
|
||||
string[] persons = ners.Where(x => x.Type.Contains("PERSON")).Select(x => x.Text).Distinct().ToArray();
|
||||
if (persons == null && persons.Count() == 0) continue;
|
||||
if (persons != null && persons.Count() == 1)
|
||||
{
|
||||
sentence.PersonName = speakerFeatureLearner.MatchSpeaker(persons.FirstOrDefault());
|
||||
continue;
|
||||
}
|
||||
//如果是两个,取ARG0
|
||||
sentence.PersonName = speakerFeatureLearner.MatchSpeaker(persons.FirstOrDefault());
|
||||
}
|
||||
|
||||
List<NamedSummary> indexRange = GetNamedSummaryBefore(hanlp, data.Datas, 8);
|
||||
|
||||
|
||||
|
||||
//3字词根据次数划分到2字词内
|
||||
python.Dispose();
|
||||
}
|
||||
|
||||
public List<NamedSummary> GetNamedSummaryBefore(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int paragraphIndex)
|
||||
{
|
||||
return GetNamedSummary(hanlp, datas, paragraphIndex - 3 > 0 ? paragraphIndex - 3 : 0, paragraphIndex);
|
||||
}
|
||||
|
||||
public List<NamedSummary> GetNamedSummaryAfter(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int paragraphIndex)
|
||||
{
|
||||
return GetNamedSummary(hanlp, datas, paragraphIndex, paragraphIndex + 3);
|
||||
}
|
||||
|
||||
public List<NamedSummary> GetNamedSummary(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int startIndex, int endIndex)
|
||||
{
|
||||
List<NamedSummary> nameds = new List<NamedSummary>();
|
||||
NamedSummary summary;
|
||||
List<NamedEntity> temps;
|
||||
AiSentenceViewEntity sentence;
|
||||
for (int i = 0; i < datas.Length; i++)
|
||||
{
|
||||
sentence = datas[i];
|
||||
if (sentence.ParagraphIndex < startIndex) continue;
|
||||
if (sentence.ParagraphIndex > endIndex) continue;
|
||||
|
||||
temps = hanlp.GetHanlpNer(sentence.Content);
|
||||
if (temps == null || temps.Count() == 0) continue;
|
||||
|
||||
foreach (var tp in temps)
|
||||
{
|
||||
if (tp.Text.Length == 1) continue;
|
||||
if (tp.Type != "PERSON") continue;
|
||||
if (nameds.Select(x => x.Person).Contains(tp.Text))
|
||||
{
|
||||
summary = nameds.Where(x => x.Person == tp.Text).FirstOrDefault();
|
||||
summary.Count++;
|
||||
continue;
|
||||
}
|
||||
summary = new NamedSummary();
|
||||
summary.Person = tp.Text;
|
||||
summary.Count = 1;
|
||||
nameds.Add(summary);
|
||||
}
|
||||
}
|
||||
|
||||
return nameds;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 处理 NamedSummary 数组:按包含关系分组,以次数多的为主名称调用 AddRule
|
||||
/// </summary>
|
||||
/// <param name="namedSummaries">人名统计数组</param>
|
||||
public void BuildPersonNameRules(NamedSummary[] namedSummaries, SpeakerFeatureLearner speakerFeatureLearner)
|
||||
{
|
||||
// 空值校验
|
||||
if (namedSummaries == null || !namedSummaries.Any())
|
||||
{
|
||||
Console.WriteLine("NamedSummary 数组为空,无需处理");
|
||||
return;
|
||||
}
|
||||
|
||||
// 1. 提取去重的人名-次数字典(确保每个人名只统计一次总次数)
|
||||
var nameCountDict = namedSummaries
|
||||
.Where(ns => !string.IsNullOrWhiteSpace(ns.Person))
|
||||
.GroupBy(ns => ns.Person.Trim())
|
||||
.ToDictionary(g => g.Key, g => g.Sum(ns => ns.Count));
|
||||
|
||||
if (nameCountDict.Count < 2)
|
||||
{
|
||||
Console.WriteLine("有效人名数量不足,无需建立映射");
|
||||
return;
|
||||
}
|
||||
|
||||
// 2. 标记已处理的人名,避免重复配对
|
||||
var processedNames = new HashSet<string>();
|
||||
|
||||
// 3. 遍历查找包含关系的人名对,按次数确定主从
|
||||
foreach (var (name1, count1) in nameCountDict)
|
||||
{
|
||||
if (processedNames.Contains(name1)) continue;
|
||||
|
||||
// 查找与当前人名存在包含关系的其他人名
|
||||
var matchedNames = nameCountDict
|
||||
.Where(kv => !processedNames.Contains(kv.Key)
|
||||
&& kv.Key != name1
|
||||
&& (kv.Key.Contains(name1) || name1.Contains(kv.Key))) // 互相包含都算匹配
|
||||
.ToList();
|
||||
|
||||
if (matchedNames.Count == 0)
|
||||
{
|
||||
processedNames.Add(name1);
|
||||
continue; // 无匹配的包含关系人名,跳过
|
||||
}
|
||||
|
||||
// 4. 合并当前人名和匹配人名,按次数选主名称
|
||||
foreach (var (name2, count2) in matchedNames)
|
||||
{
|
||||
// 确定主/次要名称:次数多的是主,次数相同则选长度长的(可选规则)
|
||||
string mainName = count1 > count2 ? name1 : (count2 > count1 ? name2 : (name1.Length > name2.Length ? name1 : name2));
|
||||
string minorName = mainName == name1 ? name2 : name1;
|
||||
|
||||
// 5. 调用 AddRule(次要名称, 主名称)
|
||||
speakerFeatureLearner.AddRule(mainName, minorName);
|
||||
Console.WriteLine($"已调用 AddRule:AddRule(\"{mainName}\", \"{minorName}\")");
|
||||
|
||||
// 标记为已处理,避免重复配对
|
||||
processedNames.Add(name1);
|
||||
processedNames.Add(name2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public class NamedSummary
|
||||
{
|
||||
public int Count { get; set; }
|
||||
|
||||
public string Person { get; set; }
|
||||
|
||||
public decimal MinConfidence { get; set; }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
using CloudBuilder.Core.DatabaseAccessor.Entity;
|
||||
using CloudBuilder.Core.DependencyInjection.Task;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace CloudBuilder.Topshelf.Task.Novel
|
||||
{
|
||||
//task:SplitFileChapterTask encoding:GB2312 pattern: @"第([^】]+)章" input:D:\NET8\AI\frxxz.txt output:D:\NET8\AI\NovelSplitter
|
||||
//input可以是文件名,也可以是目錄名
|
||||
//output輸出目錄
|
||||
//pattern可能自定義分割表達式
|
||||
public class SplitFileChapterTask : IScheduleTask
|
||||
{
|
||||
private readonly IApplicationService service;
|
||||
|
||||
public SplitFileChapterTask(IApplicationService service)
|
||||
{
|
||||
this.service = service;
|
||||
}
|
||||
|
||||
public void Run(Dictionary<string, string> bodyDict)
|
||||
{
|
||||
if (!bodyDict.ContainsKey("input") || string.IsNullOrEmpty(bodyDict["input"])) return;
|
||||
|
||||
string input = bodyDict["input"];
|
||||
string output = AppDomain.CurrentDomain.BaseDirectory;
|
||||
|
||||
//如果是文件名,輸出到同一目錄下
|
||||
if (File.Exists(input))
|
||||
{
|
||||
output = Path.GetDirectoryName(input) + "\\";
|
||||
}
|
||||
else if (Directory.Exists(input))
|
||||
{
|
||||
output = input;
|
||||
}
|
||||
|
||||
if (bodyDict.ContainsKey("output") && !string.IsNullOrEmpty(bodyDict["output"]))
|
||||
{
|
||||
output = bodyDict["output"];
|
||||
//如果目錄不存在,創建目錄
|
||||
if (!Directory.Exists(output)) Directory.CreateDirectory(output);
|
||||
}
|
||||
|
||||
|
||||
string pattern = @"第([^】]+)章";
|
||||
if (bodyDict.ContainsKey("pattern") && !string.IsNullOrEmpty(bodyDict["pattern"]))
|
||||
pattern = bodyDict["pattern"];
|
||||
|
||||
string encoding = "UTF-8";
|
||||
|
||||
if (bodyDict.ContainsKey("encoding") && !string.IsNullOrEmpty(bodyDict["encoding"])) encoding = bodyDict["encoding"];
|
||||
|
||||
List<string> files = PackFiles(input);
|
||||
|
||||
foreach (string file in files)
|
||||
{
|
||||
SaveFile(encoding, file, output, pattern);
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveFile(string encoding, string fileName, string output, string pattern)
|
||||
{
|
||||
System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
|
||||
|
||||
string[] lines = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding));
|
||||
|
||||
string title = string.Empty;
|
||||
|
||||
int chapterIndex = 1;
|
||||
StringBuilder chapter = new StringBuilder();
|
||||
foreach (string line in lines)
|
||||
{
|
||||
Match match = Regex.Match(line, pattern);
|
||||
if (match.Success)
|
||||
{
|
||||
// 提取第一个分组(即括号内的内容)
|
||||
//string value = match.Groups[1].Value;
|
||||
|
||||
//第一次找到是空的,下次找到新的保存之前的
|
||||
if (!string.IsNullOrEmpty(title) && chapter.Length > 0)
|
||||
{
|
||||
File.WriteAllText(Path.Combine(output, Path.GetFileNameWithoutExtension(fileName) + string.Format("_{0}_{1}", chapterIndex++,title) + Path.GetExtension(fileName)), chapter.ToString(), Encoding.UTF8);
|
||||
|
||||
chapter = new StringBuilder();
|
||||
}
|
||||
|
||||
title = line;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(title))
|
||||
{
|
||||
chapter.AppendLine(line);
|
||||
}
|
||||
}
|
||||
//最后一章
|
||||
if (!string.IsNullOrEmpty(title) && chapter.Length > 0)
|
||||
{
|
||||
File.WriteAllText(Path.Combine(output, Path.GetFileNameWithoutExtension(fileName) + string.Format("_{0}_{1}", chapterIndex++, title) + Path.GetExtension(fileName)), chapter.ToString(), Encoding.UTF8);
|
||||
}
|
||||
}
|
||||
|
||||
private List<string> PackFiles(string input)
|
||||
{
|
||||
List<string> files = new List<string>();
|
||||
if (Path.HasExtension(input) && !string.IsNullOrEmpty(input))
|
||||
{
|
||||
if (File.Exists(input))
|
||||
{
|
||||
files.Add(input);
|
||||
}
|
||||
}
|
||||
else if (!string.IsNullOrEmpty(input))
|
||||
{
|
||||
if (Directory.Exists(input))
|
||||
{
|
||||
string[] tempFiles = Directory.GetFiles(input);
|
||||
if (tempFiles != null && tempFiles.Length > 0)
|
||||
{
|
||||
files.AddRange(tempFiles);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
1.创建一本书
|
||||
[title]
|
||||
[author]
|
||||
[description]
|
||||
[book_type]
|
||||
[file_path]
|
||||
[status]
|
||||
创建成功[status]>【待上传】,如果同步上传了文件[file_path]不空,[status]>【待提交】(此时已写入章节)
|
||||
[guid]
|
||||
[book_guid]
|
||||
[chapter_index]
|
||||
[title]
|
||||
[content]
|
||||
[status]
|
||||
[status]>【处理中】
|
||||
【待确认人物关系】【等待分配角色】【生成中】【完成】
|
||||
2.上传文件后【待提交】,用户检查章节分割无误后等待提交
|
||||
3.按【提交】后,写入段落及细分是否是对话
|
||||
段落换行来分割,如有对话规则,再划分对话
|
||||
Reference in new issue
Block a user