commit 597b88d075398e04177155fc14ae198203c1078b Author: owenchen Date: Thu May 28 15:49:25 2026 +0800 ow diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb70c0a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +################################################################################ +# 這個 .gitignore 檔案由 Microsoft(R) Visual Studio 自動建立。 +################################################################################ + +/.vs +/bin/Debug/net8.0 +/obj diff --git a/CloudBuilder.Topshelf.csproj b/CloudBuilder.Topshelf.csproj new file mode 100644 index 0000000..1c642a8 --- /dev/null +++ b/CloudBuilder.Topshelf.csproj @@ -0,0 +1,93 @@ + + + + Exe + net8.0 + enable + enable + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Lib\CloudBuilder.AI.dll + + + Lib\CloudBuilder.AI.Service.dll + + + Lib\CloudBuilder.Core.dll + + + Lib\CloudBuilder.Request.dll + + + Lib\CloudBuilder.Security.dll + + + Lib\CloudBuilder.Security.Service.dll + + + + diff --git a/CloudBuilder.Topshelf.sln b/CloudBuilder.Topshelf.sln new file mode 100644 index 0000000..649aae5 --- /dev/null +++ b/CloudBuilder.Topshelf.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.8.34316.72 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "CloudBuilder.Topshelf", "CloudBuilder.Topshelf.csproj", "{63EB47F6-A616-4C64-B850-85ED294E2524}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {63EB47F6-A616-4C64-B850-85ED294E2524}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {63EB47F6-A616-4C64-B850-85ED294E2524}.Debug|Any CPU.Build.0 = Debug|Any CPU + {63EB47F6-A616-4C64-B850-85ED294E2524}.Release|Any CPU.ActiveCfg = Release|Any CPU + {63EB47F6-A616-4C64-B850-85ED294E2524}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {24F7770A-0647-4A47-B147-80AFD34ACB87} + EndGlobalSection +EndGlobal diff --git a/Lib/CloudBuilder.AI.Service.dll b/Lib/CloudBuilder.AI.Service.dll new file mode 100644 index 0000000..114cd3e Binary files /dev/null and b/Lib/CloudBuilder.AI.Service.dll differ diff --git a/Lib/CloudBuilder.AI.dll b/Lib/CloudBuilder.AI.dll new file mode 100644 index 0000000..af01fb1 Binary files /dev/null and b/Lib/CloudBuilder.AI.dll differ diff --git a/Lib/CloudBuilder.Core.dll b/Lib/CloudBuilder.Core.dll new file mode 100644 index 0000000..ae05401 Binary files /dev/null and b/Lib/CloudBuilder.Core.dll differ diff --git a/Lib/CloudBuilder.Request.dll b/Lib/CloudBuilder.Request.dll new file mode 100644 index 0000000..6fd3977 Binary files /dev/null and b/Lib/CloudBuilder.Request.dll differ diff --git a/Lib/CloudBuilder.Security.Service.dll b/Lib/CloudBuilder.Security.Service.dll new file mode 100644 index 0000000..dc2f302 Binary files /dev/null and b/Lib/CloudBuilder.Security.Service.dll differ diff --git a/Lib/CloudBuilder.Security.dll b/Lib/CloudBuilder.Security.dll new file mode 100644 index 0000000..8021048 Binary files /dev/null and b/Lib/CloudBuilder.Security.dll differ diff --git a/Program.cs b/Program.cs new file mode 100644 index 0000000..1b1bfd5 --- /dev/null +++ b/Program.cs @@ -0,0 +1,156 @@ +using CloudBuilder; +using CloudBuilder.Core.Authorization; +using CloudBuilder.Core.DatabaseAccessor; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Form; +using CloudBuilder.Core.DependencyInjection.Request; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Core.Policy; +using CloudBuilder.Core.Service; +using CloudBuilder.Request.Security; +using CloudBuilder.Security.Data; +using CloudBuilder.Security.Policy; +using CloudBuilder.Topshelf.Task; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using System.Reflection; +using System.Text; + +public partial class Program +{ + /// + /// request:Y 要求调用webApi参数 + /// + /// + static void Main(string[] args) + { + try + { + // 设置控制台编码为UTF-8，解决中文乱码问题 + Console.OutputEncoding = Encoding.UTF8; + Console.InputEncoding = Encoding.UTF8; + + string taskName = null; + Dictionary bodyDict = new Dictionary(); + + + if (args != null && args.Length > 0) + { + var services = new ServiceCollection(); + + services.Initialize(); + + var serviceProvider = services.BuildServiceProvider(); + + ApplicationServiceContext applicationServiceContext = serviceProvider.GetService()!; + + string key = null; + string[] temp = null; + + foreach (string s in args) + { + temp = s.Split(new char[] { ':' }, 2, StringSplitOptions.RemoveEmptyEntries); + if (temp.Length <= 1) continue; + if (temp[1] == null || temp[1].Trim() == string.Empty) continue; + + key = temp[0].Trim(); + + if (key == "task") + { + taskName = temp[1];// config.TaskMappings.Where(x => x.Task == temp[1]).FirstOrDefault(); + if (taskName == null) continue; + + continue; + } + + if (bodyDict.ContainsKey(key)) continue; + + bodyDict.Add(key, temp[1]); + } + + if (string.IsNullOrEmpty(taskName)) return; + + try + { + + IApplicationService applicationService = serviceProvider.GetService(); + applicationService.UserId = applicationService.Configuration["IApplicationService:UserId"]; + //如果有传入用户ID，则使用传入的用户ID + if (bodyDict.ContainsKey("user_id")) + { + applicationService.UserId = bodyDict["user_id"]; + } + + IEnumerable tasks = serviceProvider.GetServices(); + + foreach (var task in tasks) + { + if (task.GetType().Name.ToLower() != taskName.ToLower()) continue; + + if (bodyDict.ContainsKey("request") && bodyDict["request"].ToUpper().Trim() == "Y") Login(applicationService); + + task.Run(bodyDict); + } + + } + catch (Exception ex) + { + Console.WriteLine($"[{ex.Message}]"); + } + finally + { + Environment.Exit(0); + } + } + } + catch (Exception ex) + { + Console.WriteLine(ex.ToString()); + } + } + + static void Login(IApplicationService applicationService) + { + #region 系统用户登录 + try + { + LoginData loginData = new LoginData(); + //从容器获取请求对象 + AuthRequest authRequest = ServiceHelper.GetTypeService(); + //获取RSA公钥 + string pubKey = authRequest.GetRsaPublicKey(); + //加密用户密码 + string psw = applicationService.Configuration["CloudBuilder:SecurityUser:UserPassword"]!; + loginData.Guid = psw.Substring(0, 36); + var rsa = new RSA(pubKey, true); + loginData.UserMobile = applicationService.Configuration["CloudBuilder:SecurityUser:UserMobile"]!; + loginData.RsaUserPassword = rsa.Encode(psw); + //设置当前文化 + loginData.Culture = CulturePolicy.ZH_CN; + + LoginProfile loginProfile; + + loginProfile = authRequest.Login(loginData); + + IContext loginContext = ServiceHelper.GetInterfaceService()!; + //登录成功后设置登录上下文 + loginContext.Requestor = loginProfile.Username; + loginContext.SetPermissionInfo(loginProfile.PermissionInfo); + loginContext.SetToken(loginProfile.Token.AccessToken); + loginContext.LoginDate = DateTime.Now; + } + catch (Exception ex) + { + throw ex; + } + + //ISecurityMasterService securityMasterService = service.ServiceProvider.GetService(); + + //SecurityMasterEntityKey key = new SecurityMasterEntityKey(); + //key.MasterCodeItem = "default_work_shift"; + //key.MasterCategory = "CloudBuilder.Hr.Policy.Master"; + //key.MasterSubcategory = "StaffWorkShiftPolicy"; + //SecurityMasterEntity securityMaster = securityMasterService.FindByKey(key); + #endregion + } +} \ No newline at end of file diff --git a/Properties/launchSettings.json b/Properties/launchSettings.json new file mode 100644 index 0000000..cbfd429 --- /dev/null +++ b/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "CloudBuilder.Topshelf": { + "commandName": "Project", + "commandLineArgs": "task:TtsTask voice:Sherpa2 save_path:D:\\\\Net8\\\\FileServer\\\\Backup\\\\DmsFile\\\\voice\\\\20260528112739.mp3 text:二愣子睁大着双眼，直直望着茅草和烂泥糊成的黑屋顶，身上盖着的旧棉被，已呈深黄色，看不出原来的本来面目，还若有若无的散发着淡淡的霉味。" + } + } +} \ No newline at end of file diff --git a/Python/Hanlp2.cs b/Python/Hanlp2.cs new file mode 100644 index 0000000..103e8fa --- /dev/null +++ b/Python/Hanlp2.cs @@ -0,0 +1,167 @@ +using Python.Runtime; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Python +{ + public class Hanlp2 + { + public readonly PythonCommand python; + + public Hanlp2(PythonCommand python) + { + this.python = python; + string file = python.ApplicationService.Configuration["Python:PythonFile"]!; + + if (!File.Exists(file)) return; + + string init = File.ReadAllText(file, Encoding.UTF8); + + using (Py.GIL()) + { + python.Exec(init); + } + } + + public List GetHanlpNer(string text) + { + try + { + //必须定义方法的方式使用 + using (Py.GIL()) + { + string json = python.GetValueExec($"processor.get_ner_json('{text}')"); + // 反序列化 + if (json == "[]") return null; + return JsonSerializer.Deserialize>(json); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + return null; + } + + public List> GetHanlpSrl(string text) + { + try + { + //必须定义方法的方式使用 + using (Py.GIL()) + { + string json = python.GetValueExec($"processor.get_srl_json('{text}')"); + var options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true // 忽略属性名大小写 + }; + // 反序列化 + return JsonSerializer.Deserialize>>(json, options); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + return null; + } + + public string GetDepRoot(string text) + { + try + { + //必须定义方法的方式使用 + using (Py.GIL()) + { + string json = python.GetValueExec($"processor.get_root('{text}')"); + var options = new JsonSerializerOptions + { + PropertyNameCaseInsensitive = true // 忽略属性名大小写 + }; + // 反序列化 + return json; + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + return null; + } + + + + public List GetHanlpCon(string text) + { + if (string.IsNullOrEmpty(text) || text.Length < 3) return null; + + try + { + //必须定义方法的方式使用 + using (Py.GIL()) + { + string json = python.GetValueExec($"processor.get_con_json('{text}')"); + + if (json == "[]") return null; + + return HanlpConstituencyNode.FromJsonString(json); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + return null; + } + + public List GetTokFine(string text) + { + try + { + //必须定义方法的方式使用 + using (Py.GIL()) + { + string json = python.GetValueExec($"processor.get_tok_fine('{text}')"); + // 反序列化 + if (json == "[]") return null; + return JsonSerializer.Deserialize>(json); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + return null; + } + + + public List GetTokCoarse(string text) + { + try + { + //必须定义方法的方式使用 + using (Py.GIL()) + { + string json = python.GetValueExec($"processor.get_tok_coarse('{text}')"); + // 反序列化 + if (json == "[]") return null; + return JsonSerializer.Deserialize>(json); + } + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + return null; + } + + public string Get(string arg) + { + return python.GetValueExec(arg); + } + } +} diff --git a/Python/NamedEntity.cs b/Python/NamedEntity.cs new file mode 100644 index 0000000..e63b7d7 --- /dev/null +++ b/Python/NamedEntity.cs @@ -0,0 +1,954 @@ +using Microsoft.Extensions.FileSystemGlobbing; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Python +{ + public class SentenceInfo + { + public List Subjects { get; set; } = new(); + public List Predicates { get; set; } = new(); + public List Objects { get; set; } = new(); + public List PersonRoles { get; set; } = new(); + } + + public class NamedEntity + { + public string Text { get; set; } // 實體文本 (如 "王小明") + public string Type { get; set; } // 實體類型 (如 "PERSON") + public int Start { get; set; } // 起始位置 (索引) + public int End { get; set; } // 結束位置 (索引) + + public override string ToString() => $"{Text} ({Type}) [{Start}-{End}]"; + } + + public class MeaningRepresentationParsingEntity + { + public string Text { get; set; } + public string Type { get; set; } + public int Start { get; set; } + public int End { get; set; } + } + + public class HanlpConstituencyNode + { + /// + /// 节点ID + /// + [JsonProperty("ItemId")] + public int ItemId { get; set; } + + /// + /// 父节点ID + /// + [JsonProperty("FatherId")] + public int FatherId { get; set; } + + /// + /// 节点标签（如TOP、IP、NP、VP等） + /// + [JsonProperty("Label")] + public string Label { get; set; } + + /// + /// 节点层级 + /// + [JsonProperty("Level")] + public int Level { get; set; } + + /// + /// 子节点数量 + /// + [JsonProperty("Children")] + public int Children { get; set; } + + /// + /// 从JSON字符串解析节点列表 + /// + /// Python HanLP返回的JSON字符串 + /// 节点列表 + public static List FromJsonString(string jsonString) + { + if (string.IsNullOrEmpty(jsonString)) + return new List(); + + try + { + return JsonConvert.DeserializeObject>(jsonString); + } + catch (Exception ex) + { + Console.WriteLine($"JSON解析错误: {ex.Message}"); + return new List(); + } + } + + /// + /// 将节点列表转换为JSON字符串 + /// + /// 节点列表 + /// JSON字符串 + public static string ToJsonString(List nodes) + { + if (nodes == null) + return "[]"; + + return JsonConvert.SerializeObject(nodes, + Formatting.Indented, + new JsonSerializerSettings { StringEscapeHandling = StringEscapeHandling.Default }); + } + + /// + /// 重写ToString方法，返回节点的详细信息 + /// + /// 节点信息字符串 + public override string ToString() + { + return $"ItemId: {ItemId}, FatherId: {FatherId}, Label: {Label}, Level: {Level}, Children: {Children}"; + } + } + + /// + /// HanLP处理结果工具类 + /// 用于与Python HanLP服务交互 + /// + public class HanlpResultHelper + { + /// + /// 解析句法分析结果 + /// + /// Python程序输出的JSON字符串 + /// 节点列表 + public static List ParseConstituencyResult(string pythonOutput) + { + return HanlpConstituencyNode.FromJsonString(pythonOutput); + } + + /// + /// 根据父节点ID查找子节点 + /// + /// 所有节点 + /// 父节点ID + /// 子节点列表 + public static List GetChildrenNodes(List nodes, int fatherId) + { + return nodes?.FindAll(node => node.FatherId == fatherId) ?? new List(); + } + + /// + /// 根据ID查找节点 + /// + /// 所有节点 + /// 节点ID + /// 节点对象，如果未找到则返回null + public static HanlpConstituencyNode GetNodeById(List nodes, int itemId) + { + return nodes?.Find(node => node.ItemId == itemId); + } + + /// + /// 根据Label查找节点 + /// + /// 所有节点 + /// 节点ID + /// 节点对象，如果未找到则返回null + public static HanlpConstituencyNode[] GetNodeByLabel(List nodes, string label) + { + return nodes?.Where(node => node.Label == label).ToArray(); + } + + /// + /// 根据ID查找父节点 + /// + /// 所有节点 + /// 节点ID + /// 节点对象，如果未找到则返回null + public static HanlpConstituencyNode GetNodeFatherById(List nodes, int itemId) + { + HanlpConstituencyNode self = GetNodeById(nodes, itemId); + return nodes?.Find(node => node.ItemId == self.FatherId); + } + + /// + /// 根据ID查找等于label父节点 + /// + /// 所有节点 + /// 节点ID + /// 节点对象，如果未找到则返回null + public static HanlpConstituencyNode GetNodeTopLabelById(List nodes, int itemId, string label) + { + HanlpConstituencyNode nd = GetNodeFatherById(nodes, itemId); + + if (label == nd.Label && nd.Children > 0) return nd; + + while (nd.FatherId > 0) + { + return GetNodeTopLabelById(nodes, nd.ItemId, label); + } + + return null; + } + + /// + /// 根据ID查找等于label子节点 + /// + /// 所有节点 + /// 节点ID + /// 节点对象，如果未找到则返回null + public static string GetNodeBottomLabelById(List nodes, int itemId, string label) + { + HanlpConstituencyNode[] nds = nodes?.Where(node => node.FatherId == itemId && node.Label == label && node.Children > 0).ToArray(); + if (nds == null || nds.Length == 0) return null; + HanlpConstituencyNode father; + List list = new List(); + foreach (HanlpConstituencyNode nd in nds) + { + List sons = GetChildrenNodes(nodes, nd.ItemId); + if (sons == null || sons.Count() == 0) continue; + return string.Join("", sons.Select(x => x.Label).ToArray()); + } + + return null; + } + + /// + /// 根据ID查找所有兄弟节点（同父且不同ID的节点） + /// + /// 所有节点列表 + /// 目标节点ID + /// 兄弟节点列表，如果无兄弟节点/节点不存在则返回空列表 + public static List GetNodeBrothersById(List nodes, int itemId) + { + // 空值保护：如果节点列表为空，直接返回空列表 + if (nodes == null || nodes.Count == 0) + { + return new List(); + } + + // 第一步：找到目标节点（根据ID） + var targetNode = nodes.Find(node => node.ItemId == itemId); + // 如果目标节点不存在，返回空列表 + if (targetNode == null) + { + return new List(); + } + + // 第二步：获取目标节点的父ID，筛选所有同父且ID不等于目标节点的节点 + int fatherId = targetNode.FatherId; + var brotherNodes = nodes.FindAll(node => + node.FatherId == fatherId && // 同父节点 + node.ItemId != itemId // 排除自身 + ); + + // 返回兄弟节点列表（无兄弟则返回空列表） + return brotherNodes ?? new List(); + } + } +} + + +/* +import os +import json +import logging + +# 配置日志 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') +logger = logging.getLogger(__name__) + +try: + import hanlp + from hanlp.components.mtl.multi_task_learning import MultiTaskLearning + from hanlp.components.mtl.tasks.tok.tag_tok import TaggingTokenization + from hanlp.components.mtl.tasks.ner.tag_ner import TaggingNamedEntityRecognition + + # 尝试导入tqdm，如果不存在则使用替代 + try: + from tqdm import tqdm + HAS_TQDM = True + except ImportError: + logger.warning("tqdm未安装，将不显示进度条") + HAS_TQDM = False + + # 创建一个简单的tqdm替代 + class tqdm: + def __init__(self, iterable, desc=None): + self.iterable = iterable + self.desc = desc + def __iter__(self): + return iter(self.iterable) +except ImportError as e: + logger.error(f"导入HanLP失败: {e}") + +class HanLPProcessor: + """HanLP自然语言处理工具类""" + + def __init__(self, + hanlp_home='D:/Python38/hanlp/', + hf_home='D:/Python38/models/', + hf_endpoint='https://hf-mirror.com', + hanlp_url='https://ftp.hankcs.com/hanlp/', + surnames_path='D:/Python38/hanlp/dictionary/surnames.txt', + titles_path='D:/Python38/hanlp/dictionary/titles.txt', + words_path='D:/Python38/hanlp/dictionary/words.txt', + salutations_path='D:/Python38/hanlp/dictionary/salutations.txt'): + # 初始化ID计数器和父ID跟踪 + self.item_id_counter = 1 + self.current_father_id = 0 + """ + 初始化HanLP处理器 + + 参数: + hanlp_home: HanLP资源缓存目录 + hf_home: HuggingFace模型缓存目录 + hf_endpoint: HuggingFace下载端点 + hanlp_url: HanLP下载URL + surnames_path: 姓氏词典路径 + titles_path: 头衔词典路径 + words_path: 专有名词词典路径 + salutations_path: 称呼词典路径 + """ + # 存储配置信息 + self.hanlp_home = hanlp_home + self.hf_home = hf_home + self.hf_endpoint = hf_endpoint + self.hanlp_url = hanlp_url + self.surnames_path = surnames_path + self.titles_path = titles_path + self.words_path = words_path + self.salutations_path = salutations_path + + # 初始化组件 + self.mtl = None + self.tok = None + self.ner = None + + # 设置环境变量 + self._set_environment_variables() + + # 加载模型和词典 + self._load_model() + self._load_dictionaries() + + def _set_environment_variables(self): + """设置必要的环境变量""" + try: + os.environ['HANLP_HOME'] = self.hanlp_home + os.environ['HF_HOME'] = self.hf_home + os.environ['HF_ENDPOINT'] = self.hf_endpoint + os.environ['HANLP_URL'] = self.hanlp_url + logger.info("环境变量设置成功") + except Exception as e: + logger.error(f"设置环境变量失败: {e}") + + def _load_dict_from_file(self, file_path, default_tag='S-PERSON'): + """从文件加载词典""" + dict_tags = {} + try: + with open(file_path, 'r', encoding='utf-8') as f: + for word in f: + word = word.strip() + if word: # 跳过空行 + dict_tags[(word,)] = (default_tag,) # 保持元组形式 + except Exception as e: + print(f"加载词典文件{file_path}失败: {e}") + return dict_tags + + def _load_names(self, surnames_path, titles_path): + """加载姓氏和头衔，生成姓名组合""" + try: + with open(surnames_path, 'r', encoding='utf-8') as f: + surnames = [line.strip() for line in f if line.strip()] + + with open(titles_path, 'r', encoding='utf-8') as f: + titles = [line.strip() for line in f if line.strip()] + + # 使用生成器表达式（节省内存） + full_names = (f'{surname}{title}' for surname in surnames for title in titles) + return full_names + except Exception as e: + print(f"加载姓名文件失败: {e}") + return [] + + def _load_model(self): + """加载HanLP模型""" + try: + # CLOSE是自然语义标注的闭源语料库，BASE是中号模型，ZH中文 + logger.info("开始加载HanLP模型...") + self.mtl = hanlp.load(hanlp.pretrained.mtl.CLOSE_TOK_POS_NER_SRL_DEP_SDP_CON_ELECTRA_SMALL_ZH) + self.tok = self.mtl['tok/coarse'] + self.tok.dict_force = self.tok.dict_combine = None + self.ner = self.mtl['ner/msra'] + logger.info("模型加载成功") + except Exception as e: + logger.error(f"加载模型失败: {e}") + self.mtl = None + self.tok = None + self.ner = None + + def _load_dictionaries(self): + """加载各种词典""" + try: + # 加载称呼词典 + self.ner.dict_tags = self._load_dict_from_file(self.salutations_path) + + # 读取专有名词词典 + with open(self.words_path, 'r', encoding='utf-8') as f: + names = [line.strip() for line in f if line.strip()] + + # 生成所有姓名组合 + full_names = self._load_names(self.surnames_path, self.titles_path) + + # 构建强制分词词典（使用进度条） + dict_force = {} + for name in tqdm(full_names, desc='读取人名词典'): + dict_force[name] = [name] + + for name in names: + dict_force[name] = [name] + + self.tok.dict_force = dict_force + except Exception as e: + print(f"加载词典失败: {e}") + + def get_ner_json(self, text): + """识别文本中的实体并返回JSON格式""" + if not text or not isinstance(text, str): + logger.warning("无效的输入文本") + return json.dumps([]) + + try: + if not self.mtl: + raise RuntimeError("模型未加载") + + result = self.mtl(text, tasks=['tok/coarse', 'ner/msra'], skip_tasks=['tok/fine']) + + # 检查结果格式 + if 'ner/msra' not in result or not isinstance(result['ner/msra'], (list, tuple)): + logger.warning("模型返回结果格式异常") + return json.dumps([]) + + json_data = [] + for item in result['ner/msra']: + # 确保item是正确的格式 + if isinstance(item, (list, tuple)) and len(item) >= 4: + json_data.append({ + 'Text': str(item[0]), + 'Type': str(item[1]), + 'Start': int(item[2]), + 'End': int(item[3]) + }) + + return json.dumps(json_data, ensure_ascii=False) + except Exception as e: + logger.error(f"实体识别失败: {e}") + return json.dumps([]) + + def get_srl_json(self, text): + """识别文本中的人物角色并返回JSON格式""" + if not text or not isinstance(text, str): + logger.warning("无效的输入文本") + return json.dumps([]) + + try: + if not self.mtl: + raise RuntimeError("模型未加载") + + results = self.mtl(text, tasks=['tok/coarse', 'srl'], skip_tasks=['tok/fine']) + result = [] + + # 检查结果格式 + if 'srl' not in results: + logger.warning("模型返回结果中未找到SRL数据") + return json.dumps([]) + + data = results['srl'] + if not data: + return json.dumps(result) + + for sentence in data: + if not isinstance(sentence, (list, tuple)): + continue + + sentence_data = [] + for item in sentence: + if not isinstance(item, (list, tuple)): + continue + + # 确保至少有2个元素 + if len(item) < 2: + continue + + entry = { + 'Text': str(item[0]), + 'Type': str(item[1]), + 'Start': int(item[2]) if len(item) > 2 else -1, + 'End': int(item[3]) if len(item) > 3 else -1 + } + sentence_data.append(entry) + + if sentence_data: # 只添加非空的句子数据 + result.append(sentence_data) + + return json.dumps(result, ensure_ascii=False, indent=2) + except Exception as e: + logger.error(f"人物角色识别失败: {e}") + return json.dumps([]) + + def get_con(self, text): + """解析文本的句法结构并返回结果""" + if not text or not isinstance(text, str): + logger.warning("无效的输入文本") + return None + + try: + if not self.mtl: + raise RuntimeError("模型未加载") + + logger.info("开始句法分析...") + result = self.mtl(text, tasks=['con']) + logger.info("句法分析完成") + return result + except Exception as e: + logger.error(f"句法分析失败: {e}") + return None + + def get_all(self,text): + result = self.mtl(text) + return result; + + def get_con_json(self, text): + """解析文本的句法结构并返回JSON格式字符串，便于C#对象处理""" + # 重置ID计数器和父ID，确保每次调用都从1开始 + self.item_id_counter = 0 + self.current_father_id = 0 + + # 初始化JSON数组用于存储所有解析结果元素 + result_array = [] + # 保存结果数组的引用，供packLabel方法使用 + self.result_array = result_array + + if not text or not isinstance(text, str): + logger.warning("无效的输入文本") + return json.dumps([]) + + if not self.mtl: + raise RuntimeError("模型未加载") + + logger.info("开始句法分析...") + results = self.mtl(text, tasks=['con']) + logger.info("句法分析完成") + + # 检查结果格式 + if 'con' not in results: + logger.warning("模型返回结果中未找到CON数据") + return json.dumps([]) + + #从results中截取"con": [截取字符串]} + # 将results转为字符串并提取con字段值 + results_str = str(results) + # logger.info(results_str) + # 只取出top的内容 + new_results = self.get_con_top('"con": [',results_str) + level=0 + + self.item_id_counter += 1 + + # 调用pack_childen处理所有节点，所有生成的元素将通过packLabel方法添加到result_array + try: + self.pack_childen(new_results, level) + except Exception as e: + logger.warning(f"pack_childen处理时出错: {e}") + + # 清理引用，避免内存泄漏 + delattr(self, 'result_array') + + # 将结果数组转换为JSON字符串返回，便于C#交互 + return json.dumps(result_array, ensure_ascii=False) + + def pack_childen(self, text, level, father_id=None): + # 如果没有提供father_id，则使用当前类的current_father_id + if father_id is None: + father_id = self.current_father_id + + childen_json = [] + json_temp = self.split(text, ',') + # 去除前后的[] + childen = self.substring_before_after_one(json_temp[1]) + # 分隔元素 + childens = self.split(childen, ',') + label = json_temp[0] + + if json_temp[0] == '_': + # "_", ["二愣子"] - 提取列表中的值 + # 当标签为"_"时，提取childen中的值并直接返回值字符串 + label = self.substring_before_after_one(childen) + # 保存当前的current_father_id + current_father = self.current_father_id + # 设置当前元素的父ID + self.current_father_id = father_id + # 创建节点 + top_json = self.packLabel(label, level, 0) + # 恢复原来的current_father_id + self.current_father_id = current_father + # logger.info(top_json) + return + + # 保存当前的current_father_id + current_father = self.current_father_id + # 设置当前元素的父ID + self.current_father_id = father_id + # 创建当前节点 + top_json = self.packLabel(label, level, len(childens)) + # 获取当前节点的ID作为子节点的父ID + current_node_id = top_json['ItemId'] + # 恢复原来的current_father_id + self.current_father_id = current_father + + # logger.info(top_json) + + # 递归处理子节点，传递当前节点的ID作为父ID + for i, item in enumerate(childens): + childen_temp = self.substring_before_after_one(item) + childen_json.append(self.pack_childen(childen_temp, level + 1, current_node_id)) + + + def get_task(self, text, task): + ''' + 执行指定的任务并跳过fine分词 + + 参数: + text: 输入文本 + task: 要执行的任务名称 + + 返回: + 任务执行结果 + ''' + if not text or not isinstance(text, str): + logger.warning('无效的输入文本') + return None + + try: + if not self.mtl: + raise RuntimeError('模型未加载') + + # 跳过tok/fine任务 + result = self.mtl(text, tasks=[task]) + return result + except Exception as e: + logger.error(f'执行任务{task}失败: {e}') + return None + + def get_con_top(self,from_text,text): + if '"con": [' in text: + start = text.find(from_text) + len(from_text) + # 寻找对应的结束括号 + end = start + bracket_count = 1 + while end < len(text) and bracket_count > 0: + if text[end] == '[': + bracket_count += 1 + elif text[end] == ']': + bracket_count -= 1 + end += 1 + new_results = text[start:end-1] # -1 是为了去掉最后的']' + else: + new_results = '' + return new_results + + def substring_before_after_one(self, text): + if not text or not isinstance(text, str): + return text + if len(text) < 2: + return text + return text[1:-1] + + def split(self, text, char): + if not text: + return [] + + result = [] + current = [] + in_quotes = False # 是否在引号内 + bracket_level = 0 # 括号嵌套级别 + + i = 0 + while i < len(text): + c = text[i] + + # 处理引号 + if c == '"': + in_quotes = not in_quotes + current.append(c) + + # 处理括号 + elif c == '[' and not in_quotes: + bracket_level += 1 + current.append(c) + elif c == ']' and not in_quotes: + if bracket_level > 0: + bracket_level -= 1 + current.append(c) + + # 遇到分隔符且不在括号内和引号内 + elif c == char and bracket_level == 0 and not in_quotes: + # 添加当前部分到结果 + part = ''.join(current).strip() + # 清理引号（如果有） + if part.startswith('"') and part.endswith('"'): + part = part[1:-1] + result.append(part) + current = [] + + # 处理转义字符 + elif c == '\\' and i + 1 < len(text): + current.append(c) + current.append(text[i + 1]) + i += 1 + + else: + current.append(c) + + i += 1 + + # 处理最后一个元素 + if current: + part = ''.join(current).strip() + # 清理引号（如果有） + if part.startswith('"') and part.endswith('"'): + part = part[1:-1] + result.append(part) + + return result + + def packLabel(self, label, level, children): + # 保存当前ID作为返回值 + current_id = self.item_id_counter + + # 创建包含ID和父ID的字典 + result = { + "ItemId": current_id, + "FatherId": self.current_father_id, + "Label": label, + "Level": level, + "Children": children + } + + # 将生成的元素添加到result_array（如果存在） + if hasattr(self, 'result_array'): + self.result_array.append(result) + + # 更新计数器 + self.item_id_counter += 1 + + return result + + def extract_underscore_value(self, text): + """ + 从"_", ["值"]格式的文本中提取值，并直接返回字符串值 + + Args: + text: 格式为"_", ["值"]的字符串 + + Returns: + 提取的字符串值 + """ + # 分割文本以获取值部分 + parts = self.split(text, ',') + if len(parts) < 2: + return "" + + # 获取第二个部分（包含值的列表） + value_part = parts[1] + # 去除前后的[]和可能的引号 + value = self.substring_before_after_one(value_part).strip('"') + + # 直接返回提取的值 + return value + +processor=HanLPProcessor() + +# 测试split函数 +def test_split_function(): + # 测试用例1: 简单的逗号分隔 + text1 = ' "TOP",[["IP", [["NP", [["_", ["二愣子"]]]], ["VP", [["VP", [["_", ["姓"]], ["NP", [["_", ["韩"]]]]]], ["VP", [["_", ["名"]], ["NP", [["_", ["立"]]]]]]]], ["_", ["。"]]]]]' + result1 = processor.split(text1, ',') + print("测试用例1结果:") + for i, item in enumerate(result1): + print(f" {i}: {item}") + + text1='二愣子姓韩名立。' + print(processor.get_con_json(text1)) + # 测试用例2: 括号内包含逗号 + # text2 = '"NP",[["NN",["_", "张三"]],["NN",["_", "李四"]]]' + # result2 = processor.split(text2, ',') + # print("\n测试用例2结果:") + +def test_extract_underscore_value(): + """ + 测试新添加的extract_underscore_value函数 + """ + print("\n测试extract_underscore_value函数:") + + # 测试用例1: 基本格式 + test_text1 = '"_", ["二愣子"]' + result1 = processor.extract_underscore_value(test_text1) + print(f"输入: {test_text1}") + print(f"输出: {result1}") + + # 测试用例2: 其他值 + test_text2 = '"_", ["张三"]' + result2 = processor.extract_underscore_value(test_text2) + print(f"\n输入: {test_text2}") + print(f"输出: {result2}") + + # 测试用例3: 空值情况 + test_text3 = '"_", []' + result3 = processor.extract_underscore_value(test_text3) + print(f"\n输入: {test_text3}") + print(f"输出: {result3}") + +# 测试修复后的pack_childen方法对"_", ["二愣子"]格式的处理 +def test_pack_childen_with_underscore(): + """ + 测试pack_childen方法对"_", ["值"]格式的处理 + """ + print("\n测试pack_childen方法对'_', ['二愣子']格式的处理:") + # 测试用例：直接调用pack_childen处理"_", ["二愣子"]格式 + test_input = '"_", ["二愣子"]' + print(f"输入: {test_input}") + try: + # 调用pack_childen方法，使用level=0 + result = processor.pack_childen(test_input, 0) + # 由于pack_childen在处理"_"标签时会直接返回，我们需要查看logger输出 + print("测试完成，请检查日志输出") + except Exception as e: + print(f"测试出错: {e}") + # for i, item in enumerate(result2): + # print(f" {i}: {item}") + +# 如果直接运行此文件，则执行测试 +def test_id_generation(): + """ + 测试ItemId和FatherId的生成逻辑 + """ + print("开始测试ID生成...") + # 创建处理器实例 + processor = HanLPProcessor() + + # 测试基本的packLabel方法 + print("\n测试基本的packLabel方法:") + result1 = processor.packLabel("TOP", 0, 1) + print(result1) + + # 测试生成第二个节点，应该有正确的父子关系 + print("\n测试生成第二个节点:") + # 对于第二个节点，我们手动设置它的父ID为第一个节点的ID + processor.current_father_id = result1['ItemId'] + result2 = processor.packLabel("TOP", 0, 1) + print(result2) + + # 测试pack_childen方法 + print("\n测试pack_childen方法:") + # 重置处理器的ID计数器 + processor.item_id_counter = 1 + processor.current_father_id = 0 + + # 使用一个简单的测试字符串来模拟解析结果，确保格式正确 + test_input = "TOP, [IP, [NP, [NN, 你]], [VP, [VV, 好]]]" + try: + processor.pack_childen(test_input, 0) + print("pack_childen测试成功") + except Exception as e: + print(f"pack_childen测试失败: {e}") + # 为了演示，我们可以直接测试packLabel的组合 + print("\n直接测试ID和父子关系:") + processor.item_id_counter = 1 + processor.current_father_id = 0 + + # 模拟用户要求的输出格式 + result1 = processor.packLabel("TOP", 0, 1) + print(f"{{'ItemId':{result1['ItemId']},'FatherId':{result1['FatherId']},'Label': '{result1['Label']}', 'Level': {result1['Level']}, 'Children': {result1['Children']}}}") + + # 第二个节点的父ID是第一个节点的ID + processor.current_father_id = result1['ItemId'] + result2 = processor.packLabel("TOP", 0, 1) + print(f"{{'ItemId':{result2['ItemId']},'FatherId':{result2['FatherId']},'Label': '{result2['Label']}', 'Level': {result2['Level']}, 'Children': {result2['Children']}}}") + + print("\nID生成测试完成") + +def test_con_json(): + """ + 测试get_con_json方法返回的JSON格式字符串 + """ + print("\n开始测试get_con_json方法...") + processor = HanLPProcessor() + + # 测试空输入 + empty_result = processor.get_con_json("") + print(f"空输入测试: {empty_result}") + print(f"空输入结果类型: {type(empty_result)}") + # 验证可以解析为空数组 + try: + parsed_empty = json.loads(empty_result) + print(f"空输入解析结果: {parsed_empty}, 类型: {type(parsed_empty)}") + except json.JSONDecodeError as e: + print(f"空输入JSON解析失败: {e}") + + # 测试None输入 + none_result = processor.get_con_json(None) + print(f"None输入测试: {none_result}") + print(f"None输入结果类型: {type(none_result)}") + # 验证可以解析为空数组 + try: + parsed_none = json.loads(none_result) + print(f"None输入解析结果: {parsed_none}, 类型: {type(parsed_none)}") + except json.JSONDecodeError as e: + print(f"None输入JSON解析失败: {e}") + + # 尝试测试有效输入（注意：由于可能没有加载模型，这里可能会抛出异常） + try: + # 尝试简单的中文文本 + text = "二愣子姓韩名立。" + print(f"测试有效输入: {text}") + result = processor.get_con_json(text) + print(f"返回结果类型: {type(result)}") + print(f"返回结果字符串长度: {len(result)}") + + # 验证返回的是字符串 + assert isinstance(result, str), "返回结果应该是字符串类型" + print("✓ 验证通过：返回结果是字符串类型") + + # 尝试解析JSON字符串 + try: + parsed_result = json.loads(result) + print(f"✓ JSON解析成功，解析后类型: {type(parsed_result)}") + print(f"解析后数组长度: {len(parsed_result)}") + + # 如果解析成功且有元素，显示部分内容 + if parsed_result: + print(f"解析后第一个元素: {parsed_result[0]}") + if len(parsed_result) > 1: + print(f"解析后第二个元素: {parsed_result[1]}") + except json.JSONDecodeError as e: + print(f"✗ JSON解析失败: {e}") + + print(f"完整返回结果: {result}") + except Exception as e: + print(f"有效输入测试出错（可能是模型未加载）: {e}") + + print("get_con_json方法测试完成") + +if __name__ == "__main__": + # test_split_function() + # test_id_generation() + # test_extract_underscore_value() + # test_pack_childen_with_underscore() + test_con_json() + # processor=HanLPProcessor() + +*/ \ No newline at end of file diff --git a/Python/PythonCommand.cs b/Python/PythonCommand.cs new file mode 100644 index 0000000..ac1f3b9 --- /dev/null +++ b/Python/PythonCommand.cs @@ -0,0 +1,88 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using Python.Runtime; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Python +{ + public class PythonCommand + { + private bool _disposed = false; // 释放标记 + public dynamic PythonCreateScope { get; set; } + public IApplicationService ApplicationService; + public PythonCommand(IApplicationService applicationService) + { + ApplicationService = applicationService; + + Runtime.PythonDLL = applicationService.Configuration["Python:PythonCommand"]; + PythonEngine.PythonHome = Path.GetDirectoryName(Runtime.PythonDLL); + PythonEngine.Initialize(); + + PythonCreateScope = Py.CreateScope(); + } + + public void Exec(string arg) + { + using (Py.GIL()) + { + PythonCreateScope.Exec(arg);//"from pyhanlp import HanLP" + } + } + + public string GetValueExec(string arg) + { + using (Py.GIL()) + { + PythonCreateScope.Exec(string.Format("s_value={0}", arg)); + + var value = PythonCreateScope.Eval("s_value"); + + return value.ToString(); + } + } + + public string[] GetValuesExec(string arg) + { + using (Py.GIL()) + { + PythonCreateScope.Exec(string.Format("s_list={0}", arg)); + + var value = PythonCreateScope.Eval("s_list"); + + string[] st = value.As(); + + return st; + } + } + + // 核心：释放 Python.NET 资源 + public void Dispose() + { + Dispose(true); + GC.SuppressFinalize(this); // 阻止析构函数重复执行 + } + + protected virtual void Dispose(bool disposing) + { + if (_disposed) return; + + // 1. 释放 Python 作用域 + if (PythonCreateScope != null) + { + PythonCreateScope.Dispose(); + PythonCreateScope = null; + } + + // 2. 关闭 Python 运行时（关键！） + if (PythonEngine.IsInitialized) + { + PythonEngine.Shutdown(); // 销毁 Python 解释器 + } + + _disposed = true; + } + } +} diff --git a/Task/AI/Character.md b/Task/AI/Character.md new file mode 100644 index 0000000..4730313 --- /dev/null +++ b/Task/AI/Character.md @@ -0,0 +1,25 @@ + static void Main() + { + // 1. 生成 U+4E00 ~ U+9FA5 完整汉字合集 + List cjkBasicChars = new List(); + for (int code = 0x4E00; code <= 0x9FA5; code++) + { + cjkBasicChars.Add((char)code); + } + + // 验证数量（应输出 20976） + Console.WriteLine($"基础区汉字总数：{cjkBasicChars.Count}"); + + // 2. 输出前 10 个汉字（验证生成结果） + Console.WriteLine("前 10 个汉字："); + for (int i = 0; i < 10; i++) + { + Console.Write(cjkBasicChars[i] + " "); + } + // 输出：一 丁 丂 七 丄 丅 丆 万 丈 三 + + // 3. 导出为文本文件（UTF-8 编码，便于查看/使用） + string allChars = new string(cjkBasicChars.ToArray()); + File.WriteAllText("CJK_Basic_Chars.txt", allChars, System.Text.Encoding.UTF8); + Console.WriteLine("\n已导出完整汉字集到 CJK_Basic_Chars.txt"); + } \ No newline at end of file diff --git a/Task/AI/CharacterInsertTask.cs b/Task/AI/CharacterInsertTask.cs new file mode 100644 index 0000000..fefac25 --- /dev/null +++ b/Task/AI/CharacterInsertTask.cs @@ -0,0 +1,65 @@ +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Transactions; +using static System.Runtime.InteropServices.JavaScript.JSType; + +namespace CloudBuilder.Topshelf.Task.AI +{ + //task:CharacterInsertTask encoding:GB2312 file_path:D:\NET8\AI\Novel.txt + public class CharacterInsertTask : IScheduleTask + { + private readonly IApplicationService service; + + public CharacterInsertTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + IAiCharacterService aiCharacterService = service.ServiceProvider.GetService(); + + string encoding = "utf-8"; + + if (bodyDict.ContainsKey("encoding")) + encoding = bodyDict["encoding"]; + else + Console.WriteLine("文件字符集"+ encoding); + + string fileName = null; + if (bodyDict.ContainsKey("encoding")) + fileName = bodyDict["file_path"]; + else + Console.WriteLine("文件路径不存在"); + + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + string content = File.ReadAllText(fileName, Encoding.GetEncoding(encoding)); + + var transactionOptions = new TransactionOptions + { + IsolationLevel = IsolationLevel.ReadCommitted, + Timeout = TimeSpan.FromSeconds(60 * 10) + }; + + using (TransactionScope _transactionScope = new TransactionScope( + TransactionScopeOption.Required, + transactionOptions, + TransactionScopeAsyncFlowOption.Enabled)) + { + aiCharacterService.Insert(content); + + _transactionScope.Complete(); + } + + + } + } +} diff --git a/Task/AI/CharacterTtsTask.cs b/Task/AI/CharacterTtsTask.cs new file mode 100644 index 0000000..5b12b0f --- /dev/null +++ b/Task/AI/CharacterTtsTask.cs @@ -0,0 +1,86 @@ +using CloudBuilder.AI.Entity; +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using EdgeTtsSharp; +using EdgeTtsSharp.NAudio; +using EdgeTtsSharp.Structures; +using Microsoft.EntityFrameworkCore.Query.Internal; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.AI +{ + //task:CharacterTtsTask save_path:D:\NET8\CloudBuilder.Topshelf\mp3 short_name:zh-CN-XiaoxiaoNeural + public class CharacterTtsTask : IScheduleTask + { + private readonly IApplicationService service; + + public CharacterTtsTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + IAiCharacterService aiService = service.ServiceProvider.GetService(); + AiCharacterEntity[] aiCharacters = aiService.FindAll(); + string shortName = "zh-CN-XiaoxiaoNeural"; + string savePath = string.Empty; + if (bodyDict.ContainsKey("short_name")) + shortName = bodyDict["short_name"]; + + if (bodyDict.ContainsKey("save_path")) + savePath = bodyDict["save_path"]; + + int totalLines = aiCharacters.Length; + int processedLines = 0; // 已处理行数计数器 + int totalValidWords = 0; // 可选：统计有效词总数，用于进度详情 + + foreach (var c in aiCharacters) + { + try + { + processedLines++; + TtsAsync(shortName, savePath, c).GetAwaiter().GetResult(); + + UpdateProgress(processedLines, totalLines, totalValidWords); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + + } + } + + public async System.Threading.Tasks.Task TtsAsync(string shortName, string savePath, AiCharacterEntity aiCharacter) + { + savePath = Path.Combine(savePath, string.Format("{0}.mp3", aiCharacter.Content)); + + if (File.Exists(savePath)) return; + + var voice = await EdgeTts.GetVoice(shortName); + await voice.SaveAudioToFile(aiCharacter.Content, savePath); + } + + private static void UpdateProgress(int processed, int total, int validWords) + { + double progressPercent = (double)processed / total * 100; + int progressBarLength = 50; // 进度条总长度 + int filledLength = (int)(progressPercent / 100 * progressBarLength); + + // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]） + string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]"; + + string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} "; + Console.Write($"\r{progressInfo}"); + Console.Out.Flush(); + } + } +} diff --git a/Task/AI/ChineseNameExtractorTask.cs b/Task/AI/ChineseNameExtractorTask.cs new file mode 100644 index 0000000..41f01c5 --- /dev/null +++ b/Task/AI/ChineseNameExtractorTask.cs @@ -0,0 +1,102 @@ +using CloudBuilder.AI.Entity; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Topshelf.Python; +using CloudBuilder.Topshelf.Task.Novel.Process; +using static System.Net.Mime.MediaTypeNames; + +namespace CloudBuilder.Topshelf.Task +{ + //task:ChineseNameExtractorTask guid:B000006 + public class ChineseNameExtractorTask : IScheduleTask + { + private readonly IApplicationService service; + + public ChineseNameExtractorTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + if (!bodyDict.ContainsKey("guid")) return; + + string guid = bodyDict["guid"]; + + PythonCommand python = new PythonCommand(service); + try + { + Hanlp2 hanlp = new Hanlp2(python); + + IRepository repository = service.GetRepository>(); + + IRepository repositoryAiSentenceEntity = service.GetRepository>(); + + string depRoot; + + List ners; + + List> srls; + + List cons = null; + + SpeakerAnalysisHelper speakerFeatureLearner = new SpeakerAnalysisHelper(); + + AiSentenceDialogueViewEntity[] ents = repository.DetachedEntities.Where(x => x.BookGuid == guid).OrderBy(x => x.ChapterIndex).ThenBy(x => x.ParagraphIndex).ThenBy(x => x.DialogueIndc).ToArray(); + AiSentenceEntity aiSentenceEntity; + string person = null; + int paragraphIndex = 0; + + int totalLines = ents.Length; + int processedLines = 0; // 已处理行数计数器 + + foreach (var ent in ents) + { + processedLines++; + + if (!string.IsNullOrEmpty(person) && paragraphIndex == ent.ParagraphIndex) + { + paragraphIndex = 0; + aiSentenceEntity = repositoryAiSentenceEntity.Entities.Where(x => x.Guid == ent.Guid).FirstOrDefault(); + aiSentenceEntity.PersonName = person; + + repositoryAiSentenceEntity.UpdateNow(aiSentenceEntity); + continue; + } + + depRoot = hanlp.GetDepRoot(ent.Content); + ners = hanlp.GetHanlpNer(ent.Content); + srls = hanlp.GetHanlpSrl(ent.Content); + + person = speakerFeatureLearner.GetPerson(depRoot, ners, srls, cons); + paragraphIndex = ent.ParagraphIndex; + + UpdateProgress(processedLines, totalLines); + } + + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + finally + { + python.Dispose(); + } + } + + private static void UpdateProgress(int processed, int total) + { + double progressPercent = (double)processed / total * 100; + int progressBarLength = 50; // 进度条总长度 + int filledLength = (int)(progressPercent / 100 * progressBarLength); + + // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]） + string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]"; + + string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} "; + Console.Write($"\r{progressInfo}"); + Console.Out.Flush(); + } + } +} diff --git a/Task/AI/NovelCutWordTask.cs b/Task/AI/NovelCutWordTask.cs new file mode 100644 index 0000000..fb8a7ee --- /dev/null +++ b/Task/AI/NovelCutWordTask.cs @@ -0,0 +1,182 @@ +using CloudBuilder.AI.Entity; +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Topshelf.Python; +using CloudBuilder.Topshelf.Utility; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using System.Transactions; + +namespace CloudBuilder.Topshelf.Task.AI +{ + //task:NovelCutWordTask encoding:gb2312 file_path:D:\NET8\AI\12Novel.txt + public class NovelCutWordTask : IScheduleTask + { + private static readonly Regex _noNumberLetterRegex = new Regex(@"^(?!.*[0-9a-zA-Z]).+$", RegexOptions.Compiled); + private readonly Regex _hasPunctuationRegex = new Regex(@"[，。！？；：""''()（）【】《》、·￥…—\+\-*/<>＝=～@#￥%&*()_+{}|:<>?`\-=[\];\\,./{}~！@#￥%……&*（）——+|{}】‘；：”“’。，、？·｡＂＇｀｜〃〔〕〈〉「」『』．〖〗［］｛｝]", + RegexOptions.Compiled); + + private readonly IApplicationService service; + + public NovelCutWordTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + PythonCommand python = new PythonCommand(service); + + Hanlp2 hanlp = new Hanlp2(python); + + IAiWordService aiService = service.ServiceProvider.GetService(); + AiWordEntity[] aiWords = aiService.FindAll(); + + string encoding = "utf-8"; + + if (bodyDict.ContainsKey("encoding")) + encoding = bodyDict["encoding"]; + else + Console.WriteLine("文件字符集" + encoding); + + string fileName = null; + if (bodyDict.ContainsKey("encoding")) + fileName = bodyDict["file_path"]; + else + Console.WriteLine("文件路径不存在"); + + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + string[] lines = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding)); + + int totalLines = lines.Length; + int processedLines = 0; // 已处理行数计数器 + int totalValidWords = 0; // 可选：统计有效词总数，用于进度详情 + + Console.WriteLine($"开始处理文件：{fileName}"); + Console.WriteLine($"文件总行数：{totalLines}\n"); + + SegmenterHelper segmenterHelper = new SegmenterHelper(); + + IEnumerable words; + List contents = new List(); + + List existingContents = new List(); + + if (aiWords != null && aiWords.Length > 0) + existingContents = aiWords.Select(x => x.Content).ToList(); + + foreach (string line in lines) + { + processedLines++; // 已处理行数+1 + + if (string.IsNullOrEmpty(line.Trim())) + { + // 空行也计入进度，更新进度显示 + UpdateProgress(processedLines, totalLines, totalValidWords); + continue; + } + + if (string.IsNullOrEmpty(line.Trim())) continue; + + try + { + words = segmenterHelper.Cut(line, hanlp); + } + catch (Exception ex) + { + Console.Write($"\r{ex.Message}"); + continue; + } + + if (words == null || words.Count() == 0) continue; + + var validDatas = words + .Where(str => + !string.IsNullOrWhiteSpace(str) // 过滤空/空白字符串 + && str.Length > 1 // 过滤长度≤1的字符串 + && _noNumberLetterRegex.IsMatch(str) // 过滤含数字/字母的字符串 + && !str.StartsWith("第") // 过滤以“第”开头的字符串 + && !_hasPunctuationRegex.IsMatch(str) // 核心新增：过滤含任意标点的字符串 + ) + .Distinct(StringComparer.OrdinalIgnoreCase) // 数组内去重（不区分大小写） + .ToList(); + + + // 无有效数据则直接返回 + if (!validDatas.Any()) continue; + + var newDatas = validDatas + .Where(str => !existingContents.Contains(str)) + .ToList(); + + // 6. 批量插入（仅当有新数据时执行） + if (!newDatas.Any()) + { + UpdateProgress(processedLines, totalLines, totalValidWords); + continue; + } + + contents.AddRange(newDatas); + + contents = contents + .Where(str => !string.IsNullOrWhiteSpace(str)) // 过滤无效字符串 + .Distinct(StringComparer.OrdinalIgnoreCase) // 数组内去重（不区分大小写） + .ToList(); + + totalValidWords = contents.Count; // 更新有效词总数 + + // 3. 实时更新进度（每行处理完后刷新） + UpdateProgress(processedLines, totalLines, totalValidWords); + } + + var transactionOptions = new TransactionOptions + { + IsolationLevel = IsolationLevel.ReadCommitted, + Timeout = TimeSpan.FromSeconds(60 * 10) + }; + + try + { + python.Dispose(); + } + catch (Exception) + { + + } + + + if (contents.Count() == 0) return; + + using (TransactionScope _transactionScope = new TransactionScope( + TransactionScopeOption.Required, + transactionOptions, + TransactionScopeAsyncFlowOption.Enabled)) + { + aiService.Insert(contents.ToArray()); + + _transactionScope.Complete(); + } + } + + private static void UpdateProgress(int processed, int total, int validWords) + { + double progressPercent = (double)processed / total * 100; + int progressBarLength = 50; // 进度条总长度 + int filledLength = (int)(progressPercent / 100 * progressBarLength); + + // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]） + string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]"; + + string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} 行 | 有效词数：{validWords}"; + Console.Write($"\r{progressInfo}"); + Console.Out.Flush(); + } + } +} diff --git a/Task/AI/WordInsertTask.cs b/Task/AI/WordInsertTask.cs new file mode 100644 index 0000000..3efacf8 --- /dev/null +++ b/Task/AI/WordInsertTask.cs @@ -0,0 +1,64 @@ +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Transactions; + +namespace CloudBuilder.Topshelf.Task.AI +{ + //task:WordInsertTask encoding:utf-8 file_path:D:\NET8\AI\Novel.txt + public class WordInsertTask : IScheduleTask + { + private readonly IApplicationService service; + + public WordInsertTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + IAiWordService aiService = service.ServiceProvider.GetService(); + + string encoding = "utf-8"; + + if (bodyDict.ContainsKey("encoding")) + encoding = bodyDict["encoding"]; + else + Console.WriteLine("文件字符集" + encoding); + + string fileName = null; + if (bodyDict.ContainsKey("encoding")) + fileName = bodyDict["file_path"]; + else + Console.WriteLine("文件路径不存在"); + + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + string[] contents = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding)); + + var transactionOptions = new TransactionOptions + { + IsolationLevel = IsolationLevel.ReadCommitted, + Timeout = TimeSpan.FromSeconds(60 * 10) + }; + + using (TransactionScope _transactionScope = new TransactionScope( + TransactionScopeOption.Required, + transactionOptions, + TransactionScopeAsyncFlowOption.Enabled)) + { + aiService.Insert(contents); + + _transactionScope.Complete(); + } + + + } + } +} diff --git a/Task/Novel/ImportBookTask.cs b/Task/Novel/ImportBookTask.cs new file mode 100644 index 0000000..a4807e5 --- /dev/null +++ b/Task/Novel/ImportBookTask.cs @@ -0,0 +1,136 @@ +using CloudBuilder.AI.Data; +using CloudBuilder.AI.Entity; +using CloudBuilder.AI.Policy; +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Topshelf.Task.Novel.Process; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Transactions; +using static System.Runtime.InteropServices.JavaScript.JSType; + +namespace CloudBuilder.Topshelf.Task.Novel +{ + //task:ImportBookTask encoding:GB2312 pattern: @"第([^】]+)章" input:D:\NET8\AI\仙侠\凡人修仙传_忘语.txt + //input可以是文件名:D:\NET8\AI\仙侠\书名_作者.txt + //input也可以是目錄名:D:\NET8\AI\仙侠,必须是小说类型为结尾 + //output輸出目錄 + //pattern可能自定義分割表達式 + public class ImportBookTask : IScheduleTask + { + private readonly IApplicationService service; + + public ImportBookTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + if (!bodyDict.ContainsKey("input") || string.IsNullOrEmpty(bodyDict["input"])) return; + + string input = bodyDict["input"]; + + string pattern = @"第([^】]+)章"; + if (bodyDict.ContainsKey("pattern") && !string.IsNullOrEmpty(bodyDict["pattern"])) + pattern = bodyDict["pattern"]; + + string encoding = "UTF-8"; + + if (bodyDict.ContainsKey("encoding") && !string.IsNullOrEmpty(bodyDict["encoding"])) encoding = bodyDict["encoding"]; + + List files = PackFiles(input); + + foreach (string file in files) + { + ProcessBook(file, encoding, pattern); + } + } + + public void ProcessBook(string file, string encoding, string pattern) + { + AiBookData data = new AiBookData(); + AiBookEntity book = new AiBookEntity(); + book.Guid = Guid.NewGuid().ToString(); + string[] temps = Path.GetFileNameWithoutExtension(file).Split('_'); + if (temps == null || temps.Length != 2) + { + Console.WriteLine("文件格式要求：书名_作者.txt"); + return; + } + + book.Title = temps[0]; + book.Author = temps[1]; + + temps = file.Split('\\'); + book.BookType = temps[temps.Length - 2]; + + //book.FilePath = file; + book.Status = BookStatusPolicy.SUBMIT;//S-待提交 + + ChapterSplitter chapterSplitter = new ChapterSplitter(); + data.AiBook = book; + + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + string context = File.ReadAllText(file, Encoding.GetEncoding(encoding)); + + List aiChapterEntities = chapterSplitter.SplitNovelIntoChapters(context, pattern); + + if (aiChapterEntities == null || aiChapterEntities.Count() == 0) return; + + aiChapterEntities.ToLookup(x => x.BookGuid = book.Guid); + aiChapterEntities.ToLookup(x => x.Status = ChapterStatusPolicy.PROCESS); + + data.AiChapters = aiChapterEntities.ToArray(); + + var transactionOptions = new TransactionOptions + { + IsolationLevel = IsolationLevel.ReadCommitted, + Timeout = TimeSpan.FromSeconds(60 * 10) + }; + + using (TransactionScope _transactionScope = new TransactionScope( + TransactionScopeOption.Required, + transactionOptions, + TransactionScopeAsyncFlowOption.Enabled)) + { + if (data.AiBook != null) + service.ServiceProvider.GetService().Insert(data); + + _transactionScope.Complete(); + } + } + + private List PackFiles(string input) + { + List files = new List(); + if (Path.HasExtension(input) && !string.IsNullOrEmpty(input)) + { + if (File.Exists(input)) + { + files.Add(input); + } + } + else if (!string.IsNullOrEmpty(input)) + { + if (Directory.Exists(input)) + { + string[] tempFiles = Directory.GetFiles(input); + if (tempFiles != null && tempFiles.Length > 0) + { + files.AddRange(tempFiles); + } + } + } + + return files; + } + + } +} diff --git a/Task/Novel/ParagraphSplitterTask.cs b/Task/Novel/ParagraphSplitterTask.cs new file mode 100644 index 0000000..68a9629 --- /dev/null +++ b/Task/Novel/ParagraphSplitterTask.cs @@ -0,0 +1,96 @@ +using CloudBuilder.AI.Data; +using CloudBuilder.AI.Entity; +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Topshelf.Task.Novel.Process; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using System.Transactions; +using static System.Runtime.InteropServices.JavaScript.JSType; + +namespace CloudBuilder.Topshelf.Task.Novel +{ + //task:ParagraphSplitterTask book_title:凡人修仙传 + public class ParagraphSplitterTask : IScheduleTask + { + private readonly IApplicationService service; + + public ParagraphSplitterTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + if (!bodyDict.ContainsKey("book_title") || string.IsNullOrEmpty(bodyDict["book_title"])) return; + + string title = bodyDict["book_title"]; + + AiChapterEntity[] aiChapters = service.ServiceProvider.GetService().FindByTitle(title); + + if (aiChapters == null || aiChapters.Length == 0) return; + + ParagraphSplitter paragraphSplitter = new ParagraphSplitter(); + List aiParagraphs = new List(); + List aiSentences = new List(); + AiBookData data; + foreach (var cpt in aiChapters) + { + data = paragraphSplitter.SplitChapterIntoParagraphs(cpt); + if (data.AiParagraphs == null || data.AiParagraphs.Count() == 0) continue; + + aiParagraphs.AddRange(data.AiParagraphs); + } + + if (aiParagraphs == null || aiParagraphs.Count() == 0) return; + + SentenceSplitter sentenceSplitter = new SentenceSplitter(); + AiSentenceEntity aiSentence; + + foreach (var ph in aiParagraphs) + { + if (ph.DialogueIndc != YesNoPolicy.YES) + { + aiSentence = new AiSentenceEntity(); + aiSentence.Guid = Guid.NewGuid().ToString(); + aiSentence.Content = ph.Content; + aiSentence.DialogueIndc = YesNoPolicy.NO; + aiSentence.SentenceIndex = 1; + aiSentence.ParagraphGuid = ph.Guid; + aiSentence.Status = "G"; + //D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】 + aiSentences.Add(aiSentence); + continue; + } + + var s = sentenceSplitter.SplitByConfigurableQuotes(ph); + if (s == null || s.Count() == 0) continue; + + aiSentences.AddRange(s); + } + + var transactionOptions = new TransactionOptions + { + IsolationLevel = IsolationLevel.ReadCommitted, + Timeout = TimeSpan.FromSeconds(60 * 10) + }; + + using (TransactionScope _transactionScope = new TransactionScope( + TransactionScopeOption.Required, + transactionOptions, + TransactionScopeAsyncFlowOption.Enabled)) + { + service.ServiceProvider.GetService().Inserts(aiParagraphs.ToArray()); + service.ServiceProvider.GetService().Inserts(aiSentences.ToArray()); + + _transactionScope.Complete(); + } + } + } +} diff --git a/Task/Novel/Process/ChapterSplitter.cs b/Task/Novel/Process/ChapterSplitter.cs new file mode 100644 index 0000000..05470da --- /dev/null +++ b/Task/Novel/Process/ChapterSplitter.cs @@ -0,0 +1,156 @@ +using CloudBuilder.AI.Entity; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Topshelf.Python; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.Novel.Process +{ + /// + /// 章节分割器 + /// 不同的小说有自己的分割方法，可以传入参数 + /// + public class ChapterSplitter + { + public ChapterSplitter() + { + + } + /// + /// 按章节分割小说文本 + /// + public List SplitNovelIntoChapters(string novelText, string chapterPatterns) + { + var chapters = new List(); + var lines = novelText.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + var currentAiChapterEntity = new StringBuilder(); + var currentAiChapterEntityTitle = "未命名章节"; + var chapterCount = 1; + + foreach (var line in lines) + { + // 检查是否是章节标题 + if (IsChapterTitle(line, chapterPatterns)) + { + // 如果当前章节有内容，则保存 + if (currentAiChapterEntity.Length > 0) + { + chapters.Add(new AiChapterEntity + { + Title = currentAiChapterEntityTitle, + Guid = Guid.NewGuid().ToString(), + Content = currentAiChapterEntity.ToString(), + ChapterIndex = chapterCount + }); + currentAiChapterEntity.Clear(); + chapterCount++; + } + + // 更新章节标题 + currentAiChapterEntityTitle = line.Trim(); + // 添加章节标题 + currentAiChapterEntity.AppendLine(line.Trim()); + // 添加一个空行作为分隔，确保章节标题和正文内容分离 + currentAiChapterEntity.AppendLine(); + } + else + { + // 添加到当前章节，确保每行独立 + currentAiChapterEntity.AppendLine(line); + } + } + + // 添加最后一个章节 + if (currentAiChapterEntity.Length > 0) + { + chapters.Add(new AiChapterEntity + { + Title = currentAiChapterEntityTitle, + Content = currentAiChapterEntity.ToString(), + Guid = Guid.NewGuid().ToString(), + ChapterIndex = chapterCount + }); + } + + return chapters; + } + + /// + /// 检查是否为章节标题 + /// + private bool IsChapterTitle(string line, string cptPattern) + { + // 去除首尾空白字符 + var trimmedLine = line.Trim(); + + // 常见的章节标题模式 + var chapterPatterns = new[] + { + @"^第\s*[\u4e00-\u9fa50-9]+\s*章", // 第X章, 第XX章等 + @"^第\s*[0-9]+\s*章", // 第1章, 第2章等 + }; + + if (!string.IsNullOrEmpty(cptPattern)) + { + Array.Resize(ref chapterPatterns, chapterPatterns.Length + 1); + chapterPatterns[chapterPatterns.Length - 1] = cptPattern; + } + + foreach (var pattern in chapterPatterns) + { + if (Regex.IsMatch(trimmedLine, pattern, RegexOptions.IgnoreCase)) + { + //第一章内容说明了什么，请描述一下： + if (trimmedLine.Length > 20) return false; + return true; + } + } + + return false; + } + + /// + /// 保存章节到单独的文件 + /// + public void SaveChaptersToFiles(List chapters, string outputDirectory) + { + // 确保输出目录存在 + if (!Directory.Exists(outputDirectory)) + { + Directory.CreateDirectory(outputDirectory); + } + + foreach (var chapter in chapters) + { + // 使用章节标题作为文件名，清理非法字符 + var fileName = CleanFileName(chapter.Title) + ".txt"; + var filePath = Path.Combine(outputDirectory, fileName); + File.WriteAllText(filePath, chapter.Content, Encoding.UTF8); + } + } + + /// + /// 清理文件名中的非法字符 + /// + private string CleanFileName(string fileName) + { + var invalidChars = Path.GetInvalidFileNameChars(); + foreach (var invalidChar in invalidChars) + { + fileName = fileName.Replace(invalidChar, '_'); + } + + // 限制文件名长度 + if (fileName.Length > 100) + { + fileName = fileName.Substring(0, 100); + } + + return fileName; + } + } +} diff --git a/Task/Novel/Process/ParagraphSplitter.cs b/Task/Novel/Process/ParagraphSplitter.cs new file mode 100644 index 0000000..8e6ce15 --- /dev/null +++ b/Task/Novel/Process/ParagraphSplitter.cs @@ -0,0 +1,63 @@ +using CloudBuilder.AI.Data; +using CloudBuilder.AI.Entity; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.Novel.Process +{ + public class ParagraphSplitter + { + public ParagraphSplitter() + { + } + + public AiBookData SplitChapterIntoParagraphs(AiChapterEntity aiChapter) + { + + AiBookData data = new AiBookData(); + + var paragraphs = new List(); + var sentences = new List(); + + if (string.IsNullOrWhiteSpace(aiChapter.Content)) return data; + + // 按换行拆分段落（过滤空段落） + var paragraphTexts = aiChapter.Content.Split("\r\n") + .Select(p => p.Trim()) + .Where(p => !string.IsNullOrWhiteSpace(p)) + .ToList(); + + SentenceSplitter sentenceSplitter = new SentenceSplitter(); + + AiParagraphEntity aiParagraph; + int index = 1; + foreach (var text in paragraphTexts) + { + if (!ValidText(text)) continue; + + aiParagraph = new AiParagraphEntity(); + aiParagraph.Content = text; + aiParagraph.ChapterGuid = aiChapter.Guid; + aiParagraph.Guid = Guid.NewGuid().ToString(); + aiParagraph.ParagraphIndex = index++; + aiParagraph.DialogueIndc = sentenceSplitter.IsDialogue(text) ? YesNoPolicy.YES : YesNoPolicy.NO; + paragraphs.Add(aiParagraph); + } + + data.AiParagraphs = paragraphs.ToArray(); + data.AiSentences = sentences.ToArray(); + + return data; + } + + private bool ValidText(string text) + { + if (string.IsNullOrWhiteSpace(text.Trim())) return false; + + return true; + } + } +} diff --git a/Task/Novel/Process/SentenceSplitter.cs b/Task/Novel/Process/SentenceSplitter.cs new file mode 100644 index 0000000..fb48abd --- /dev/null +++ b/Task/Novel/Process/SentenceSplitter.cs @@ -0,0 +1,199 @@ +using CloudBuilder.AI.Entity; +using Microsoft.EntityFrameworkCore.Metadata.Internal; +using Microsoft.Extensions.FileSystemGlobbing; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.Novel.Process +{ + /// + /// 智能句子分割器 + /// + public class SentenceSplitter + { + // 成对引号配置数组（可自由扩展，格式：[左引号, 右引号]） + private readonly List<(string LeftQuote, string RightQuote)> _quotePairs; + // 动态生成的正则表达式（匹配所有配置的引号类型） + private Regex _dynamicQuotePattern; + // 引号内必须包含的结尾标点集合 + private readonly string _endPunctuations = "？！。，；：、……～·"; + + /// + /// 默认构造函数：初始化常用成对引号（中文、英文、直角引号） + /// + public SentenceSplitter() + { + _quotePairs = new List<(string, string)> + { + ("“", "”"), // 中文弯引号 + ("\"", "\""), // 英文双引号（需转义） + ("〝", "〞"), // 中文直角引号 + ("「", "」") // 中文方头引号（可选扩展） + }; + InitDynamicRegex(); + } + + /// + /// 自定义构造函数：允许外部传入任意成对引号配置 + /// + /// 自定义成对引号列表 + public SentenceSplitter(List<(string LeftQuote, string RightQuote)> customQuotePairs) + { + _quotePairs = customQuotePairs ?? new List<(string, string)>(); + InitDynamicRegex(); + } + + /// + /// 初始化动态正则：根据配置的成对引号生成匹配规则 + /// + private void InitDynamicRegex() + { + if (_quotePairs.Count == 0) + { + // 空配置时生成空正则，避免匹配任何内容 + _dynamicQuotePattern = new Regex(@"^$", RegexOptions.Compiled); + return; + } + + // 步骤1：转义所有引号（避免正则特殊字符冲突），拼接左/右引号匹配组 + string escapedLeftQuotes = string.Join("|", _quotePairs.Select(p => Regex.Escape(p.LeftQuote))); + string escapedRightQuotes = string.Join("|", _quotePairs.Select(p => Regex.Escape(p.RightQuote))); + // 步骤2：转义标点集合，避免正则解析错误 + string escapedPunctuations = Regex.Escape(_endPunctuations); + + // 步骤3：构建动态正则表达式 + // 分组说明： + // ?：匹配配置的任意左引号 + // ?：匹配引号内的内容（必须以指定标点结尾） + // ?：匹配配置的任意右引号 + string regexPattern = $@"(?{escapedLeftQuotes})(?.+?[{escapedPunctuations}])(?{escapedRightQuotes})"; + _dynamicQuotePattern = new Regex(regexPattern, RegexOptions.Compiled | RegexOptions.Singleline); + } + + /// + /// 核心分割方法：按配置的成对引号分割文本，结果无引号字符 + /// + /// 待分割的文本（如：“其他标点符号，如？”紫茜的秀目 或 "其他标点符号，如？"紫茜的秀目） + /// 分割后的纯文本列表 + public List SplitByConfigurableQuotes(AiParagraphEntity aiParagraphEntity) + { + AiSentenceEntity aiSentence; + int sentenceIndex = 1; + // 空值/空白校验 + if (string.IsNullOrWhiteSpace(aiParagraphEntity.Content)) + { + return new List(); + } + + List result = new List(); + string remainingText = aiParagraphEntity.Content.Trim(); + int lastMatchEnd = 0; + + // 遍历所有匹配的「引号+内容+标点+引号」文本块 + foreach (Match match in _dynamicQuotePattern.Matches(remainingText)) + { + // 1. 提取匹配的左/右引号，校验是否为成对配置 + string matchedLeft = match.Groups["left"].Value; + string matchedRight = match.Groups["right"].Value; + bool isPairMatched = _quotePairs.Any(p => p.LeftQuote == matchedLeft && p.RightQuote == matchedRight); + if (!isPairMatched) + { + continue; // 非成对引号，跳过该匹配（避免“xxx"这类混合匹配） + } + + // 2. 提取引号前的文本（过滤空值） + string prefixText = remainingText.Substring(lastMatchEnd, match.Index - lastMatchEnd).Trim(); + if (!string.IsNullOrEmpty(prefixText)) + { + aiSentence = new AiSentenceEntity(); + aiSentence.Guid = Guid.NewGuid().ToString(); + aiSentence.Content = prefixText; + aiSentence.DialogueIndc = YesNoPolicy.NO; + aiSentence.SentenceIndex = sentenceIndex++; + aiSentence.ParagraphGuid = aiParagraphEntity.Guid; + aiSentence.Status = "G"; + //D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】 + result.Add(aiSentence); + } + + // 3. 提取引号内的纯内容（无引号字符） + string quotedContent = match.Groups["content"].Value.Trim(); + if (!string.IsNullOrEmpty(quotedContent)) + { + aiSentence = new AiSentenceEntity(); + aiSentence.Guid = Guid.NewGuid().ToString(); + aiSentence.Content = quotedContent; + aiSentence.DialogueIndc = YesNoPolicy.YES; + aiSentence.SentenceIndex = sentenceIndex++; + aiSentence.ParagraphGuid = aiParagraphEntity.Guid; + aiSentence.Status = "P"; + //D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】 + result.Add(aiSentence); + } + + // 更新最后匹配位置 + lastMatchEnd = match.Index + match.Length; + } + + // 4. 提取最后一个匹配项后的剩余文本 + string suffixText = remainingText.Substring(lastMatchEnd).Trim(); + if (!string.IsNullOrEmpty(suffixText)) + { + aiSentence = new AiSentenceEntity(); + aiSentence.Guid = Guid.NewGuid().ToString(); + aiSentence.Content = suffixText; + aiSentence.DialogueIndc = YesNoPolicy.NO; + aiSentence.SentenceIndex = sentenceIndex++; + aiSentence.ParagraphGuid = aiParagraphEntity.Guid; + aiSentence.Status = "G"; + //D-【待确认对话】p-【待确认人物】A-【等待分配角色】G-【生成中】C-【完成】 + result.Add(aiSentence); + } + + return result; + } + + /// + /// 新增：判断文本是否包含对话内容 + /// + /// 待判断的文本（句子/段落） + /// true=包含对话，false=纯旁白/空文本 + public bool IsDialogue(string text) + { + // 1. 处理空文本/NULL，直接返回false + if (string.IsNullOrWhiteSpace(text)) + return false; + + // 2. 文本预处理：去空格、统一全角/半角引号 + string cleanedText = text.Trim(); + // 替换全角空格为半角，避免干扰判断 + cleanedText = Regex.Replace(cleanedText, @"　", " "); + // 统一引号格式（中文弯引号、直引号 → 标准双引号） + cleanedText = cleanedText.Replace("“", "\"") + .Replace("”", "\"") + .Replace("〝", "\"") + .Replace("〞", "\"") + .Replace("「", "\"") + .Replace("」", "\"") + .Replace("‘", "'") + .Replace("’", "'"); + + // 3. 核心判断规则： + // - 包含成对/单个双引号（排除仅含引号无内容的情况） + // - 包含冒号+引号（如："XXX说："XXX"" 场景） + var hasQuote = Regex.IsMatch(cleanedText, @""".+?""") || // 成对双引号（含内容） + Regex.IsMatch(cleanedText, @":\s*"""); // 冒号后接引号（说话人+对话） + var hasSingleQuote = Regex.IsMatch(cleanedText, @"'[^']+'"); // 成对单引号（英文对话） + + // 4. 排除仅含引号符号的无效情况（如 ""、''、“”） + var onlyQuote = Regex.IsMatch(cleanedText, @"^\s*[""''“”]{1,2}\s*$"); + + // 最终判断：包含有效引号且不是仅含引号 → 判定为对话 + return (hasQuote || hasSingleQuote) && !onlyQuote; + } + } +} diff --git a/Task/Novel/Process/SpeakerAnalysisHelper.cs b/Task/Novel/Process/SpeakerAnalysisHelper.cs new file mode 100644 index 0000000..4781509 --- /dev/null +++ b/Task/Novel/Process/SpeakerAnalysisHelper.cs @@ -0,0 +1,102 @@ +using CloudBuilder.AI.Entity; +using CloudBuilder.Topshelf.Python; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.Novel.Process +{ + public class SpeakerAnalysisHelper + { + public string GetPerson(string depRoot, List ners, List> srls, List cons) + { + string arg0 = string.Empty; + + if (!string.IsNullOrEmpty(depRoot) && srls != null && srls.Count() > 0) + { + foreach (var srl in srls) + { + if (srl.Where(x => x.Text == depRoot).Any()) + { + foreach (var mrp in srl) + { + if (mrp.Type.ToUpper() != "ARG0") continue; + + if (ners != null && ners.Where(x => mrp.Text.Contains(x.Text)).Any() && + ners.Where(x => mrp.Text.Contains(x.Text)).FirstOrDefault().Text.Length > 1) + return ners.Where(x => mrp.Text.Contains(x.Text)).FirstOrDefault().Text; + + return mrp.Text.Length > 6 ? mrp.Text.Substring(mrp.Text.Length - 6, 6) : mrp.Text; + } + } + + foreach (var mrp in srl) + { + if (mrp.Type.ToUpper() != "ARG0") continue; + arg0 = mrp.Text.Length > 6 ? mrp.Text.Substring(mrp.Text.Length - 6, 6) : mrp.Text; + } + } + } + + if (ners != null && ners.Count() > 0) + return ners.FirstOrDefault().Text; + + return arg0; + } + + public AiParagraphEntity[] AiParagraphs { get; set; } + + public AiSentenceEntity[] AiSentences { get; set; } + + public AiChapterPersonEntity[] AiChapterPersons { get; set; } + } +} +/* +“文本解析→人物提取→上下文匹配→结果输出” + +graph TD +A[对话行] --> B{规则1：文本特征匹配}; +B -->|匹配成功| C[直接赋值说话者]; +B -->|匹配失败| D{规则2：紧邻旁白匹配}; +D -->|匹配成功| C; +D -->|匹配失败| E{规则3：段落内最近人物}; +E -->|匹配成功| C; +E -->|匹配失败| F[赋值默认值+标记异常]; + +规则 1：文本特征精准匹配（最高优先级） +基于对话内容中的 “专属特征” 直接匹配说话者，避免上下文干扰： +示例规则： +对话内容特征 匹配的说话者 +包含 “第二元婴” 韩立 +包含 “银月所化” 银月 +包含 “本座 / 本尊” 反派角色 + +落地：维护一个 “特征 - 人物” 映射字典，匹配时优先使用： +规则 2：紧邻旁白匹配（次高优先级）只找对话行 “紧邻的前 1-2 行旁白”，而非整个段落的旁白，避免跨越多个人物导致匹配错误： +规则：仅检查对话行的前 1 行、前 2 行旁白（最多前 3 行），超过则停止； +落地：修改向前查找的逻辑，限制查找范围 + +规则 3：排除无效人物（避免干扰）从旁白中提取人物时，过滤掉 “非核心人物”（如 “店小二”“路人”），只保留主要人物： +落地：维护 “核心人物白名单”，提取人物时只保留白名单内的名称 + +3. 增加异常处理和人工校验（兜底保障） +即使逻辑再完善，也会有边缘场景匹配错误，需通过 “异常标记 + 人工复核” 兜底： + +步骤 1：标记可疑匹配结果对以下情况标记为 “可疑”，存入日志或单独字段： +匹配到的人物在旁白中出现次数≥2（比如同一段落有韩立和银月，无法确定）； +对话行前后 3 行都没有找到人物，最终赋值为 “未知人物”； +对话内容无任何特征，且段落内人物数量≥3； +落地：为实体增加扩展字段（如IsSuspected），或生成异常日志 + +步骤 2：人工复核可疑数据 +规则：所有标记为 “可疑” 的匹配结果，必须经过人工复核后再更新到数据库； +落地：生成 “可疑数据清单”（包含Guid、Content、PersonName），供运营 / 编辑人员校验，修正错误后重新填充。 + + +收集错误案例：记录每次匹配错误的场景（如 “银月的对话匹配给韩立”），分析错误原因； +迭代特征字典：将错误案例转化为新的特征规则（如新增 “乌云”→银月，“元婴”→韩立）； +A/B 测试：对优化后的规则做小范围测试，对比匹配准确率，确认有效后全量上线。 +*/ \ No newline at end of file diff --git a/Task/Novel/Process/SpeakerFeatureLearner.cs b/Task/Novel/Process/SpeakerFeatureLearner.cs new file mode 100644 index 0000000..bc09645 --- /dev/null +++ b/Task/Novel/Process/SpeakerFeatureLearner.cs @@ -0,0 +1,148 @@ +using CloudBuilder.Topshelf.Python; +using Microsoft.VisualBasic; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection.Metadata; +using System.Runtime.Intrinsics.Arm; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.Novel.Process +{ + public class SpeakerFeatureLearner + { + private readonly List _rules; + public SpeakerFeatureLearner() + { + _rules = new List(); + InitializeDefaultRules(); + } + + private void InitializeDefaultRules() + { + //AddRule("银月", "少妇"); + } + public void AutoLearnFromText(string text) + { + + } + public void AutoLearnFromText(string depRoot, List ners, List> srls, List cons) + { + + + if (ners == null || !ners.Where(x => x.Type.Trim().ToUpper().Contains("PERSON")).Any()) return; + + HanlpConstituencyNode[] nodes; + HanlpConstituencyNode nodef; + string feature = null; + foreach (NamedEntity namedEntity in ners.Where(x => x.Type.Trim().ToUpper().Contains("PERSON")).ToArray()) + { + nodes = HanlpResultHelper.GetNodeByLabel(cons, namedEntity.Text); + if (nodes == null || nodes.Length == 0) continue; + + //如果存在，找到父节点上的NP + foreach (HanlpConstituencyNode node in nodes) + { + nodef = HanlpResultHelper.GetNodeTopLabelById(cons, node.FatherId, "NP"); + if (nodef == null) continue; + feature = HanlpResultHelper.GetNodeBottomLabelById(cons, nodef.ItemId, "NP"); + if (string.IsNullOrEmpty(feature)) continue; + if (namedEntity.Text == feature) continue; + + AddRule(namedEntity.Text, feature); + } + } + + //自学创建特征 + //NR->NP->NP->IP + //NN->NP(称呼、职业) + + //完整汉语句法树库核心标签对照表（含 TOP/ NP / IP / CLP） + //按 “核心层级 / 短语类型” 分类整理，覆盖 90 % 以上高频标注： + //1.顶层 / 句子级标签 + //标签 英文全称 中文名称 定义 示例 + //TOP Top - level Phrase 顶层短语 句法树的根节点，包含整个句子的最高层级结构 所有句子的根节点均标注为 TOP + //IP Inflectional Phrase / Simple Clause 简单从句 句子级核心短语（含主谓结构），是 TOP 的直接子节点 “我吃饭” 整体标注为 IP + //CP Complement Clause 补语从句 带补语标记（的 / 吗 / 呢）的从句 / 复句成分 “我买的书” 中 “我买的” 标注为 CP + //2.名词类短语 + //标签 英文全称 中文名称 定义 示例 + //NP Noun Phrase 名词短语 以名词为核心的短语（可含定语 / 量词） “苹果”“我的手机”“三本书” + //DNP XP+DEG Phrase “的” 字短语 由 “XP + 的” 构成的名词性短语 “红色的”“我喜欢的” + //NNP Proper Noun Phrase 专有名词短语 人名 / 地名 / 机构名等专有名词构成的短语 “北京大学”“张三”“北京市” + //3.动词类短语 + //标签 英文全称 中文名称 定义 示例 + //VP Verb Phrase 动词短语 以动词为核心的短语（可含宾语 / 状语） “吃饭”“快速跑步”“看电影” + //VV Verb Phrase(Verb Core) 动词核心短语 纯动词构成的短语（无附加成分） “跑”“思考”“购买” + //VCP Verb Complement Phrase 动补短语 动词 +补语构成的短语 “吃完”“跑快”“看明白” + //4.数量 / 量词类短语 + //标签 英文全称 中文名称 定义 示例 + //CLP Classifier Phrase 量词短语 以量词为核心的短语（数词 + 量词 / 指示词 + 量词） “三个”“这本”“那些” + //QP Quantifier Phrase 量化短语 以数量为核心的短语（涵盖数词 / 量词 / 数量整体） “三”“若干”“三本”“许多” + //DP Determiner Phrase 指示词短语 以指示词（这 / 那 / 此）为核心的短语 “这个”“那些”“此问题” + //5.介词 / 方位类短语 + //标签 英文全称 中文名称 定义 示例 + //PP Prepositional Phrase 介词短语 以介词为核心的短语 “在学校”“关于这件事”“把书” + //LCP Locative Complement Phrase 方位补语短语 含方位词的短语 “桌子上”“房间里”“马路边” + //LOC Locative Phrase 方位短语 纯方位名词构成的短语（LCP 的简化版） “上面”“左边”“内部” + //6.形容词 / 副词类短语 + //标签 英文全称 中文名称 定义 示例 + //AP Adjective Phrase 形容词短语 以形容词为核心的短语 “漂亮”“非常开心”“红色的” + //ADVP Adverb Phrase 副词短语 以副词为核心的短语 “快速地”“很”“非常” + //7.其他功能类短语 + //标签 英文全称 中文名称 定义 示例 + //INTJ Interjection Phrase 感叹语短语 感叹词 / 语气词构成的短语 “啊”“哦”“哎呀” + //CONJP Conjunction Phrase 连词短语 连词构成的短语 “和”“但是”“因为” + //PRN Parenthetical Phrase 插入语短语 插入句子的附加成分 “换句话说”“总的来说” + //总结 + //CLP 核心是量词短语，以量词为核心（如 “三本”“这本”），是数量表达的核心标注标签； + //TOP / IP 是句子层级标签（根节点 / 核心从句），NP / VP 是核心短语标签（名词 / 动词为核心）； + //不同工具（CTB / HanLP / 北大树库）对标签的细分略有差异（如 CLP 是否归为 QP），但核心含义一致，优先以宾州中文树库（CTB）规范为准。 + } + + public void AddRule(string speaker, params string[] keywords) + { + if (string.IsNullOrWhiteSpace(speaker)) + throw new ArgumentException("说话者名称不能为空", nameof(speaker)); + + if (keywords == null || keywords.Length == 0) + throw new ArgumentException("至少需要提供一个关键词", nameof(keywords)); + + _rules.Add(new SpeakerRule(speaker, keywords)); + } + + public string MatchSpeaker(string dialogue) + { + if (string.IsNullOrWhiteSpace(dialogue)) + return null; + + foreach (var rule in _rules) + { + if (rule.Match(dialogue)) + { + return rule.Speaker; + } + } + + return dialogue; + } + } + + public class SpeakerRule + { + public string Speaker { get; } + public string[] Keywords { get; } + + public SpeakerRule(string speaker, string[] keywords) + { + Speaker = speaker; + Keywords = keywords; + } + + public bool Match(string dialogue) + { + return Keywords.Any(keyword => + dialogue.IndexOf(keyword, StringComparison.OrdinalIgnoreCase) >= 0); + } + } +} diff --git a/Task/Novel/SpeakerTask.cs b/Task/Novel/SpeakerTask.cs new file mode 100644 index 0000000..246c2b0 --- /dev/null +++ b/Task/Novel/SpeakerTask.cs @@ -0,0 +1,239 @@ +using CloudBuilder.AI.Data; +using CloudBuilder.AI.Entity; +using CloudBuilder.AI.SearchCriteria; +using CloudBuilder.AI.Service; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Core.Service; +using CloudBuilder.Security.Entity; +using CloudBuilder.Topshelf.Python; +using CloudBuilder.Topshelf.Task.Novel.Process; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using static System.Net.Mime.MediaTypeNames; + +namespace CloudBuilder.Topshelf.Task.Novel +{ + //task:SpeakerTask param_name:param_value + public class SpeakerTask : IScheduleTask + { + private readonly IApplicationService service; + + public SpeakerTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + AiSentenceSearchCriteria c = new AiSentenceSearchCriteria(); + c.limit = 999999; + List filters = new List(); + + SearchFilter filter = new SearchFilter(); + filter.Column = AiSentenceViewEntity.DB_CHAPTER_GUID; + filter.Type = SearchFilterTypePolicy.TEXT; + filter.Operator = SearchFilterOperatorPolicy.CONTAIN; + filter.Value = "cf34136e-d30a-4901-9109-626e412eaf57"; + filters.Add(filter); + + c.Filter = filters.ToArray(); + + AiSentenceViewResultData data = service.ServiceProvider.GetService().FindBySearchCriteria(c); + + if (data == null || data.Datas == null || data.Datas.Count() == 0) return; + + AiSentenceViewEntity[] sentenceViews = data.Datas.OrderBy(x => x.ParagraphIndex).ThenBy(x => x.DialogueIndc).ToArray(); + + if (sentenceViews == null || sentenceViews.Length == 0) return; + + PythonCommand python = new PythonCommand(service); + + Hanlp2 hanlp = new Hanlp2(python); + + List all = GetNamedSummary(hanlp, data.Datas, 0, data.Datas.Length); + + string text = string.Empty; + //@"乌云忽然凝聚变形，瞬间化为了一个身高寸许的黑绿婴儿出来，并口吐人言的说道。"; + //text = "艳美少妇打了韩立"; + //text = "银月嫣然一笑后，敛冲元婴一拜的说道。这元婴看起来除了比韩立主元婴小了一些外，模样神态全都一般无二的样子。"; + + //List sentiments = hanlp.GetSentiment(text); + + SpeakerFeatureLearner speakerFeatureLearner = new SpeakerFeatureLearner(); + BuildPersonNameRules(all.ToArray(), speakerFeatureLearner); + + AiSentenceViewEntity[] sentenceTemps; + int paragraphIndex = 1; + foreach (var sentence in sentenceViews) + { + sentence.CreatedBy = sentence.UpdatedBy = string.Empty; + if (string.IsNullOrEmpty(sentence.Content)) continue; + + text = sentence.Content; + + List ners = hanlp.GetHanlpNer(text); + List> srls = hanlp.GetHanlpSrl(text); + List cons = hanlp.GetHanlpCon(text); + + speakerFeatureLearner.AutoLearnFromText("", ners, srls, cons); + + //如果不存在人名 + if (ners == null || !ners.Where(x => x.Type.Contains("PERSON")).Any()) + { + if (sentence.DialogueIndc == YesNoPolicy.NO) + { + continue; + } + + + foreach (var p in sentenceViews.Where(x => x.ParagraphIndex == sentence.ParagraphIndex).ToArray()) + { + if (string.IsNullOrEmpty(p.PersonName)) continue; + + sentence.PersonName = p.PersonName; + } + } + + string[] persons = ners.Where(x => x.Type.Contains("PERSON")).Select(x => x.Text).Distinct().ToArray(); + if (persons == null && persons.Count() == 0) continue; + if (persons != null && persons.Count() == 1) + { + sentence.PersonName = speakerFeatureLearner.MatchSpeaker(persons.FirstOrDefault()); + continue; + } + //如果是两个，取ARG0 + sentence.PersonName = speakerFeatureLearner.MatchSpeaker(persons.FirstOrDefault()); + } + + List indexRange = GetNamedSummaryBefore(hanlp, data.Datas, 8); + + + + //3字词根据次数划分到2字词内 + python.Dispose(); + } + + public List GetNamedSummaryBefore(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int paragraphIndex) + { + return GetNamedSummary(hanlp, datas, paragraphIndex - 3 > 0 ? paragraphIndex - 3 : 0, paragraphIndex); + } + + public List GetNamedSummaryAfter(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int paragraphIndex) + { + return GetNamedSummary(hanlp, datas, paragraphIndex, paragraphIndex + 3); + } + + public List GetNamedSummary(Hanlp2 hanlp, AiSentenceViewEntity[] datas, int startIndex, int endIndex) + { + List nameds = new List(); + NamedSummary summary; + List temps; + AiSentenceViewEntity sentence; + for (int i = 0; i < datas.Length; i++) + { + sentence = datas[i]; + if (sentence.ParagraphIndex < startIndex) continue; + if (sentence.ParagraphIndex > endIndex) continue; + + temps = hanlp.GetHanlpNer(sentence.Content); + if (temps == null || temps.Count() == 0) continue; + + foreach (var tp in temps) + { + if (tp.Text.Length == 1) continue; + if (tp.Type != "PERSON") continue; + if (nameds.Select(x => x.Person).Contains(tp.Text)) + { + summary = nameds.Where(x => x.Person == tp.Text).FirstOrDefault(); + summary.Count++; + continue; + } + summary = new NamedSummary(); + summary.Person = tp.Text; + summary.Count = 1; + nameds.Add(summary); + } + } + + return nameds; + } + + /// + /// 处理 NamedSummary 数组：按包含关系分组，以次数多的为主名称调用 AddRule + /// + /// 人名统计数组 + public void BuildPersonNameRules(NamedSummary[] namedSummaries, SpeakerFeatureLearner speakerFeatureLearner) + { + // 空值校验 + if (namedSummaries == null || !namedSummaries.Any()) + { + Console.WriteLine("NamedSummary 数组为空，无需处理"); + return; + } + + // 1. 提取去重的人名-次数字典（确保每个人名只统计一次总次数） + var nameCountDict = namedSummaries + .Where(ns => !string.IsNullOrWhiteSpace(ns.Person)) + .GroupBy(ns => ns.Person.Trim()) + .ToDictionary(g => g.Key, g => g.Sum(ns => ns.Count)); + + if (nameCountDict.Count < 2) + { + Console.WriteLine("有效人名数量不足，无需建立映射"); + return; + } + + // 2. 标记已处理的人名，避免重复配对 + var processedNames = new HashSet(); + + // 3. 遍历查找包含关系的人名对，按次数确定主从 + foreach (var (name1, count1) in nameCountDict) + { + if (processedNames.Contains(name1)) continue; + + // 查找与当前人名存在包含关系的其他人名 + var matchedNames = nameCountDict + .Where(kv => !processedNames.Contains(kv.Key) + && kv.Key != name1 + && (kv.Key.Contains(name1) || name1.Contains(kv.Key))) // 互相包含都算匹配 + .ToList(); + + if (matchedNames.Count == 0) + { + processedNames.Add(name1); + continue; // 无匹配的包含关系人名，跳过 + } + + // 4. 合并当前人名和匹配人名，按次数选主名称 + foreach (var (name2, count2) in matchedNames) + { + // 确定主/次要名称：次数多的是主，次数相同则选长度长的（可选规则） + string mainName = count1 > count2 ? name1 : (count2 > count1 ? name2 : (name1.Length > name2.Length ? name1 : name2)); + string minorName = mainName == name1 ? name2 : name1; + + // 5. 调用 AddRule(次要名称, 主名称) + speakerFeatureLearner.AddRule(mainName, minorName); + Console.WriteLine($"已调用 AddRule：AddRule(\"{mainName}\", \"{minorName}\")"); + + // 标记为已处理，避免重复配对 + processedNames.Add(name1); + processedNames.Add(name2); + } + } + } + + public class NamedSummary + { + public int Count { get; set; } + + public string Person { get; set; } + + public decimal MinConfidence { get; set; } + } + } +} diff --git a/Task/Novel/SplitFileChapterTask.cs b/Task/Novel/SplitFileChapterTask.cs new file mode 100644 index 0000000..259fcf1 --- /dev/null +++ b/Task/Novel/SplitFileChapterTask.cs @@ -0,0 +1,133 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Task.Novel +{ + //task:SplitFileChapterTask encoding:GB2312 pattern: @"第([^】]+)章" input:D:\NET8\AI\frxxz.txt output:D:\NET8\AI\NovelSplitter + //input可以是文件名，也可以是目錄名 + //output輸出目錄 + //pattern可能自定義分割表達式 + public class SplitFileChapterTask : IScheduleTask + { + private readonly IApplicationService service; + + public SplitFileChapterTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + if (!bodyDict.ContainsKey("input") || string.IsNullOrEmpty(bodyDict["input"])) return; + + string input = bodyDict["input"]; + string output = AppDomain.CurrentDomain.BaseDirectory; + + //如果是文件名，輸出到同一目錄下 + if (File.Exists(input)) + { + output = Path.GetDirectoryName(input) + "\\"; + } + else if (Directory.Exists(input)) + { + output = input; + } + + if (bodyDict.ContainsKey("output") && !string.IsNullOrEmpty(bodyDict["output"])) + { + output = bodyDict["output"]; + //如果目錄不存在，創建目錄 + if (!Directory.Exists(output)) Directory.CreateDirectory(output); + } + + + string pattern = @"第([^】]+)章"; + if (bodyDict.ContainsKey("pattern") && !string.IsNullOrEmpty(bodyDict["pattern"])) + pattern = bodyDict["pattern"]; + + string encoding = "UTF-8"; + + if (bodyDict.ContainsKey("encoding") && !string.IsNullOrEmpty(bodyDict["encoding"])) encoding = bodyDict["encoding"]; + + List files = PackFiles(input); + + foreach (string file in files) + { + SaveFile(encoding, file, output, pattern); + } + } + + private void SaveFile(string encoding, string fileName, string output, string pattern) + { + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + string[] lines = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding)); + + string title = string.Empty; + + int chapterIndex = 1; + StringBuilder chapter = new StringBuilder(); + foreach (string line in lines) + { + Match match = Regex.Match(line, pattern); + if (match.Success) + { + // 提取第一个分组（即括号内的内容） + //string value = match.Groups[1].Value; + + //第一次找到是空的，下次找到新的保存之前的 + if (!string.IsNullOrEmpty(title) && chapter.Length > 0) + { + File.WriteAllText(Path.Combine(output, Path.GetFileNameWithoutExtension(fileName) + string.Format("_{0}_{1}", chapterIndex++,title) + Path.GetExtension(fileName)), chapter.ToString(), Encoding.UTF8); + + chapter = new StringBuilder(); + } + + title = line; + } + + if (!string.IsNullOrEmpty(title)) + { + chapter.AppendLine(line); + } + } + //最后一章 + if (!string.IsNullOrEmpty(title) && chapter.Length > 0) + { + File.WriteAllText(Path.Combine(output, Path.GetFileNameWithoutExtension(fileName) + string.Format("_{0}_{1}", chapterIndex++, title) + Path.GetExtension(fileName)), chapter.ToString(), Encoding.UTF8); + } + } + + private List PackFiles(string input) + { + List files = new List(); + if (Path.HasExtension(input) && !string.IsNullOrEmpty(input)) + { + if (File.Exists(input)) + { + files.Add(input); + } + } + else if (!string.IsNullOrEmpty(input)) + { + if (Directory.Exists(input)) + { + string[] tempFiles = Directory.GetFiles(input); + if (tempFiles != null && tempFiles.Length > 0) + { + files.AddRange(tempFiles); + } + } + } + + return files; + } + + } +} diff --git a/Task/Novel/readme.md b/Task/Novel/readme.md new file mode 100644 index 0000000..4274761 --- /dev/null +++ b/Task/Novel/readme.md @@ -0,0 +1,19 @@ +1.创建一本书 +[title] +[author] +[description] +[book_type] +[file_path] +[status] +创建成功[status]>【待上传】,如果同步上传了文件[file_path]不空,[status]>【待提交】（此时已写入章节） +[guid] +[book_guid] +[chapter_index] +[title] +[content] +[status] +[status]>【处理中】 +【待确认人物关系】【等待分配角色】【生成中】【完成】 +2.上传文件后【待提交】,用户检查章节分割无误后等待提交 +3.按【提交】后，写入段落及细分是否是对话 +段落换行来分割，如有对话规则，再划分对话 diff --git a/Task/Python/PythonRunTask.cs b/Task/Python/PythonRunTask.cs new file mode 100644 index 0000000..2d34b3f --- /dev/null +++ b/Task/Python/PythonRunTask.cs @@ -0,0 +1,59 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Topshelf.Python; +using CloudBuilder.Topshelf.Task.Novel.Process; +using CloudBuilder.Topshelf.Utility; + +namespace CloudBuilder.Topshelf.Task +{ + //task:PythonRunTask param_name:param_value + public class PythonRunTask : IScheduleTask + { + private readonly IApplicationService service; + + public PythonRunTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + PythonCommand python = new PythonCommand(service); + try + { + Hanlp2 hanlp = new Hanlp2(python); + + string text = @"金越大师，听说你前两日派人请这位韩道友聚一下，他却开始闭关了，可有此事的。"; + //text = "艳美少妇打了韩立"; + //text = "徐先生还具体帮助他确定了把画雄鹰、松鼠和麻雀作为主攻目标。"; + + text = "银发老者神色一肃的向金越禅师问道。"; + //text = "鬼灵门门主一边吩咐着，一边冷冷打量着悬浮在身前的画轴，见其并没有任何异样发生，目光闪动几下。"; + text = "韩立闻言，眉头不经意的一皱，想了想后，还是摇摇头说道："; + List sentiments = hanlp.GetTokFine(text); + + sentiments = hanlp.GetTokCoarse(text); + + string depRoot = hanlp.GetDepRoot(text); + + List ners = hanlp.GetHanlpNer(text); + + List> srls = hanlp.GetHanlpSrl(text); + + List cons = hanlp.GetHanlpCon(text); + + SpeakerAnalysisHelper speakerFeatureLearner = new SpeakerAnalysisHelper(); + + speakerFeatureLearner.GetPerson(depRoot, ners, srls, cons); + } + catch (Exception ex) + { + Console.WriteLine(ex.Message); + } + finally + { + python.Dispose(); + } + } + } +} diff --git a/Task/RunTask.cs b/Task/RunTask.cs new file mode 100644 index 0000000..7fbb64e --- /dev/null +++ b/Task/RunTask.cs @@ -0,0 +1,21 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; + +namespace CloudBuilder.Topshelf.Task +{ + //task:RunTask param_name:param_value + public class RunTask : IScheduleTask + { + private readonly IApplicationService service; + + public RunTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + + } + } +} diff --git a/Task/SplitFileSaveUTF8Task.cs b/Task/SplitFileSaveUTF8Task.cs new file mode 100644 index 0000000..79ffc42 --- /dev/null +++ b/Task/SplitFileSaveUTF8Task.cs @@ -0,0 +1,135 @@ +using CloudBuilder.Core.Authorization; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Core.Policy; +using Microsoft.AspNetCore.Http; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.Text; + + +namespace CloudBuilder.Topshelf.Task +{ + /// + /// 分隔文件并保存為UTF-8格式, + /// 【encoding】原文件格式必填, + /// 【split_size】分隔大小单位为M 【split_number】分隔文件數量,這兩個參數必填其中之一, + /// 【input】文件夹或者文件路径必填, + /// 【output】输出文件夹路径 + /// task:SplitFileSaveUTF8Task encoding:GB2312 split_size:1 input:D:\NET8\AIML文档\汉语大词典1994+订补+资料Txt output:D:\NET8\AIML文档\汉语大词典1994UTF8 + /// task:SplitFileSaveUTF8Task encoding:GB2312 split_number:1 input:D:\NET8\AIML文档\汉语大词典1994+订补+资料Txt output:D:\NET8\AIML文档\汉语大词典1994UTF8 + /// + public class SplitFileSaveUTF8Task : IScheduleTask + { + private readonly IApplicationService service; + + public SplitFileSaveUTF8Task(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + if (!bodyDict.ContainsKey("input")) return; + if (!bodyDict.ContainsKey("encoding")) return; + if (!bodyDict.ContainsKey("output")) return; + + int splitNumber = 0; + if (bodyDict.ContainsKey("split_number")) + splitNumber = Convert.ToInt32(bodyDict["split_number"]); + + string encoding = bodyDict["encoding"]; + string output = bodyDict["output"]; + string input = bodyDict["input"]; + + decimal splitSize = 0; + if (bodyDict.ContainsKey("split_size")) + splitSize = Convert.ToDecimal(bodyDict["split_size"]); + + List files = PackFiles(input); + + foreach (string file in files) + { + SaveFile(encoding, file, output, splitNumber, splitSize); + } + } + + private void SaveFile(string encoding, string fileName, string output, int splitNumber, decimal splitSize) + { + System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); + + string context = File.ReadAllText(fileName, Encoding.GetEncoding(encoding)); + + int maxlength = context.Length; + + int maxSplit = maxlength; + + //如果傳入的是分隔數，以分隔數為準 + if (splitNumber > 0) + { + maxSplit = (int)Math.Ceiling(maxlength * 1.0m / splitNumber); + } + + if (splitSize > 0 && splitNumber == 0) + { + FileInfo fileInfo = new FileInfo(fileName); + + splitNumber = (int)Math.Ceiling(fileInfo.Length / ((1024m * 1024m) * splitSize)); + + maxSplit = (int)Math.Ceiling(maxlength * 1.0m / splitNumber); + } + + + int reqLenth = maxSplit; + + string ct = string.Empty; + + for (int i = 0; i < splitNumber; i++) + { + if (i > 0 && splitNumber == i + 1) { reqLenth = maxlength % maxSplit; } + + if (splitNumber > 1) + { + ct = context.Substring(i * maxSplit, reqLenth); + } + else + { + ct = context; + } + + string outputPath = Path.Combine(output, + Path.GetFileNameWithoutExtension(fileName) + string.Format("_{0}", i+1) + + Path.GetExtension(fileName)); + + File.WriteAllText(outputPath, + ct, Encoding.UTF8); + } + } + + private List PackFiles(string input) + { + List files = new List(); + if (Path.HasExtension(input) && !string.IsNullOrEmpty(input)) + { + if (File.Exists(input)) + { + files.Add(input); + } + } + else if (!string.IsNullOrEmpty(input)) + { + if (Directory.Exists(input)) + { + string[] tempFiles = Directory.GetFiles(input); + if (tempFiles != null && tempFiles.Length > 0) + { + files.AddRange(tempFiles); + } + } + } + + return files; + } + + } +} diff --git a/Task/TTS/TtsTask.cs b/Task/TTS/TtsTask.cs new file mode 100644 index 0000000..6e562a1 --- /dev/null +++ b/Task/TTS/TtsTask.cs @@ -0,0 +1,42 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.DependencyInjection.Task; +using CloudBuilder.Topshelf.Utility; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; +using static System.Net.Mime.MediaTypeNames; +using static System.Runtime.InteropServices.JavaScript.JSType; + +namespace CloudBuilder.Topshelf.Task.TTS +{ + //task:TtsTask voice:Sherpa2 save_path:D:\MAFAI\cc.wav text:乌云忽然凝聚变形，瞬间化为了一个身高寸许的黑绿婴儿出来，并口吐人言的说道。 + public class TtsTask : IScheduleTask + { + private readonly IApplicationService service; + + public TtsTask(IApplicationService service) + { + this.service = service; + } + + public void Run(Dictionary bodyDict) + { + string save_path = string.Empty, text = string.Empty, voice = ChineseTtsVoice.Sherpa2.ToString(); + if (bodyDict.ContainsKey("voice")) + voice = bodyDict["voice"]; + + if (bodyDict.ContainsKey("save_path")) + save_path = bodyDict["save_path"]; + + if (bodyDict.ContainsKey("text")) + text = bodyDict["text"]; + + if (string.IsNullOrEmpty(voice) || string.IsNullOrEmpty(save_path) || string.IsNullOrEmpty(text)) return; + + SaveAudioToFileHelper.AudioToFile(EnumHelper.ToEnum(voice), text, save_path); + } + } +} diff --git a/Utility/SaveAudioToFileHelper.cs b/Utility/SaveAudioToFileHelper.cs new file mode 100644 index 0000000..6513b0a --- /dev/null +++ b/Utility/SaveAudioToFileHelper.cs @@ -0,0 +1,317 @@ +using SherpaOnnx; +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Text; +using System.Threading.Tasks; +using static System.Net.Mime.MediaTypeNames; + +namespace CloudBuilder.Topshelf.Utility +{ + public static class SaveAudioToFileHelper + { + private static OfflineTts fanchenTTs = null; + private static OfflineTts sherpaTTs = null; + private static OfflineTts theresaTTs = null; + private static OfflineTts meloTTs = null; + private static OfflineTts icefallTTs = null; + private static OfflineTts matchaTTs = null; + private static OfflineTts eulaTTs = null; + public static void InitConfig() + { + OfflineTtsConfig config; + OfflineTtsGeneratedAudio audio; + int numThreads = 4; + if (fanchenTTs == null) + { + config = new OfflineTtsConfig(); + config.Model.Vits.Model = "tts-models/vits-zh-hf-fanchen-C/vits-zh-hf-fanchen-C.onnx"; + config.Model.Vits.Tokens = "tts-models/vits-zh-hf-fanchen-C/tokens.txt"; + config.Model.Vits.Lexicon = "tts-models/vits-zh-hf-fanchen-C/lexicon.txt"; + config.RuleFars = "tts-models/vits-zh-hf-fanchen-C/number.fst,tts-models/vits-zh-hf-fanchen-C/phone.fst"; + config.Model.NumThreads = numThreads; + //187 + fanchenTTs = new OfflineTts(config); + + config = new OfflineTtsConfig(); + config.Model.Vits.Model = "tts-models/sherpa-onnx-vits-zh-ll/model.onnx"; + config.Model.Vits.Tokens = "tts-models/sherpa-onnx-vits-zh-ll/tokens.txt"; + config.Model.Vits.Lexicon = "tts-models/sherpa-onnx-vits-zh-ll/lexicon.txt"; + config.RuleFars = "tts-models/sherpa-onnx-vits-zh-ll/number.fst,tts-models/sherpa-onnx-vits-zh-ll/phone.fst"; + config.Model.NumThreads = numThreads; + //多人音质最好(0 - 4) + sherpaTTs = new OfflineTts(config); + + config = new OfflineTtsConfig(); + config.Model.Vits.Model = "tts-models/vits-zh-hf-theresa/theresa.onnx"; + config.Model.Vits.Tokens = "tts-models/vits-zh-hf-theresa/tokens.txt"; + config.Model.Vits.Lexicon = "tts-models/vits-zh-hf-theresa/lexicon.txt"; + config.RuleFars = "tts-models/vits-zh-hf-theresa/number.fst,tts-models/vits-zh-hf-theresa/phone.fst"; + config.Model.NumThreads = numThreads; + //804 + theresaTTs = new OfflineTts(config); + + config = new OfflineTtsConfig(); + config.Model.Vits.Model = "tts-models/vits-melo-tts-zh_en/model.onnx"; + config.Model.Vits.Tokens = "tts-models/vits-melo-tts-zh_en/tokens.txt"; + config.Model.Vits.Lexicon = "tts-models/vits-melo-tts-zh_en/lexicon.txt"; + config.RuleFars = "tts-models/vits-melo-tts-zh_en/number.fst,tts-models/vits-melo-tts-zh_en/phone.fst"; + config.Model.NumThreads = numThreads; + //1 + meloTTs = new OfflineTts(config); + + config = new OfflineTtsConfig(); + config.Model.Vits.Model = "tts-models/vits-icefall-zh-aishell3/model.onnx"; + config.Model.Vits.Tokens = "tts-models/vits-icefall-zh-aishell3/tokens.txt"; + config.Model.Vits.Lexicon = "tts-models/vits-icefall-zh-aishell3/lexicon.txt"; + config.RuleFars = "tts-models/vits-icefall-zh-aishell3/number.fst,tts-models/vits-icefall-zh-aishell3/phone.fst"; + config.Model.NumThreads = numThreads; + //中文、32 个不同说话人，最快 + icefallTTs = new OfflineTts(config); + + config = new OfflineTtsConfig(); + config.Model.Vits.Model = "tts-models/vits-zh-hf-eula/eula.onnx"; + config.Model.Vits.Tokens = "tts-models/vits-zh-hf-eula/tokens.txt"; + config.Model.Vits.Lexicon = "tts-models/vits-zh-hf-eula/lexicon.txt"; + config.RuleFars = "tts-models/vits-zh-hf-eula/number.fst,tts-models/vits-zh-hf-eula/phone.fst"; + config.Model.NumThreads = numThreads; + + eulaTTs = new OfflineTts(config); + + config = new OfflineTtsConfig(); + config.Model.Matcha.AcousticModel = "tts-models/matcha-icefall-zh-baker/model-steps-3.onnx"; + config.Model.Matcha.Vocoder = "tts-models/matcha-icefall-zh-baker/vocos-22khz-univ.onnx"; + config.Model.Matcha.Tokens = "tts-models/matcha-icefall-zh-baker/tokens.txt"; + config.Model.Matcha.Lexicon = "tts-models/matcha-icefall-zh-baker/lexicon.txt"; + config.RuleFars = "tts-models/matcha-icefall-zh-baker/number.fst,tts-models/matcha-icefall-zh-baker/phone.fst"; + config.Model.NumThreads = numThreads; + //1 + matchaTTs = new OfflineTts(config); + + + } + } + + + + public static void AudioToFile(ChineseTtsVoice actorName, string text, string saveFilePath) + { + InitConfig(); + int sid = 0; + OfflineTtsGeneratedAudio audio; + switch (actorName) + { + case ChineseTtsVoice.Fanchen10: + case ChineseTtsVoice.Fanchen110: + case ChineseTtsVoice.Fanchen6: + case ChineseTtsVoice.Fanchen15: + case ChineseTtsVoice.Fanchen2: + case ChineseTtsVoice.Fanchen20: + case ChineseTtsVoice.Fanchen14: + case ChineseTtsVoice.Fanchen36: + sid = actorName.GetSidId(); + audio = fanchenTTs.Generate(text, 1.0f, sid); + break; + case ChineseTtsVoice.Sherpa0: + case ChineseTtsVoice.Sherpa1: + case ChineseTtsVoice.Sherpa2: + case ChineseTtsVoice.Sherpa3: + case ChineseTtsVoice.Sherpa4: + sid = actorName.GetSidId(); + audio = sherpaTTs.Generate(text, 1.0f, sid); + break; + case ChineseTtsVoice.Melo: + sid = actorName.GetSidId(); + audio = meloTTs.Generate(text, 1.0f, sid); + break; + case ChineseTtsVoice.Eula: + sid = actorName.GetSidId(); + audio = eulaTTs.Generate(text, 1.0f, sid); + break; + case ChineseTtsVoice.Theresa: + sid = actorName.GetSidId(); + audio = theresaTTs.Generate(text, 1.0f, sid); + break; + case ChineseTtsVoice.Aaishe1: + case ChineseTtsVoice.Aaishe6: + case ChineseTtsVoice.Aaishe14: + case ChineseTtsVoice.Aaishe15: + case ChineseTtsVoice.Aaishe18: + case ChineseTtsVoice.Aaishe10: + sid = actorName.GetSidId(); + audio = icefallTTs.Generate(text, 1.0f, sid); + break; + default: + sid = actorName.GetSidId(); + audio = matchaTTs.Generate(text, 1.0f, sid); + break; + } + + // --- 4. 保存音频 --- + audio.SaveToWaveFile(saveFilePath); + } + } + /// + /// 微软Edge TTS中文语音音色枚举 + /// 对应官方Neural语音标识，覆盖港澳台及大陆各地区音色 + /// + public enum ChineseTtsVoice + { + /// + /// 普通话（中国大陆） 娇柔（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 喜佳（女性）-0")] + Sherpa0, + /// + /// 普通话（中国大陆）- 柔美（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 柔美（女性）-2")] + Sherpa2, + /// + /// 普通话（中国大陆）- 知性（男性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 知性（男性）-4")] + Sherpa4, + /// + /// 普通话（中国大陆）- 知青（男性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 知青（男性）-3")] + Sherpa3, + /// + /// 普通话（中国大陆）- 中青（男性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 中青（男性）-1")] + Sherpa1, + + + /// + /// 普通话（中国大陆）- 爱莎（女性）- 文件小速度快 + /// + [System.ComponentModel.Description("普通话（中国大陆） 爱莎（女性）-6")] + Aaishe6, + + /// + /// 普通话（中国大陆）- 爱莎（女性）- 文件小速度快 + /// + [System.ComponentModel.Description("普通话（中国大陆） 爱莎（女性）-14")] + Aaishe14, + + /// + /// 普通话（中国大陆）- 爱莎（女性）- 文件小速度快 + /// + [System.ComponentModel.Description("普通话（中国大陆） 爱莎（女性）-18")] + Aaishe18, + + /// + /// 普通话（中国大陆）- 云健（男性）- 文件小速度快 + /// + [System.ComponentModel.Description("普通话（中国大陆） 云健（男性）-10")] + Aaishe10, + + /// + /// 普通话（中国大陆）- 爱莎（女性）- 文件小速度快 + /// + [System.ComponentModel.Description("普通话（中国大陆） 爱莎（女性）-1")] + Aaishe1, + + /// + /// 普通话（中国大陆）- 云健（男性）- 文件小速度快 + /// + [System.ComponentModel.Description("普通话（中国大陆） 云健（男性）-15")] + Aaishe15, + + /// 普通话（中国大陆） 云阳（男性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 云阳（男性）-10")] + Fanchen10, + /// + /// 普通话（中国大陆） 云阳（男性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 云阳（男性）-110")] + Fanchen110, + /// 普通话（中国大陆） 喜佳（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 云夏（女性）-6")] + Fanchen6, + /// + /// 普通话（中国大陆） 云阳（男性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 云阳（男性）-15")] + Fanchen15, + /// + /// 普通话（中国大陆） 喜佳（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 喜佳（女性）-2")] + Fanchen2, + /// + /// 普通话（中国大陆） 喜佳（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 喜佳（女性）-14")] + Fanchen14, + /// + /// 普通话（中国大陆） 喜佳（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 喜佳（女性）-20")] + Fanchen20, + /// + /// 普通话（中国大陆） 喜佳（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 喜佳（女性）-36")] + Fanchen36, + + /// + /// 粤语（香港）- 女性（男性）- 文件大 + /// + [System.ComponentModel.Description("粤语（香港） 梅隆 （女性）-0")] + Melo, + + /// + /// 普通话（中国大陆）- 神精女（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 神精女（女性）-0")] + Eula, + + /// + /// 普通话（中国大陆）- 小神精女（女性） + /// + [System.ComponentModel.Description("普通话（中国大陆） 小神精女（女性）-27")] + Theresa, + } + + public static class ChineseTtsVoiceExtensions + { + /// + /// 获取枚举值对应的Description特性值 + /// + /// 语音枚举值 + /// 描述字符串（无特性时返回枚举名称） + public static int GetSidId(this ChineseTtsVoice voice) + { + // 获取枚举成员的特性 + var field = voice.GetType().GetField(voice.ToString()); + var attribute = field?.GetCustomAttribute(); + + // 有特性返回描述，无则返回枚举名 + return Convert.ToInt32(attribute?.Description.Split('-')[1] ?? "0"); + } + } + + public static class EnumHelper + { + // 枚举转字符串 + public static string ToEnumString(T value) where T : Enum + { + return value.ToString(); + } + + // 字符串转枚举 + public static T ToEnum(string str) where T : struct, Enum + { + return Enum.TryParse(str, out var result) ? result : default; + } + } +} + diff --git a/Utility/SegmenterHelper.cs b/Utility/SegmenterHelper.cs new file mode 100644 index 0000000..5ed7bfe --- /dev/null +++ b/Utility/SegmenterHelper.cs @@ -0,0 +1,18 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Topshelf.Python; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.Topshelf.Utility +{ + public class SegmenterHelper + { + public List Cut(string text, Hanlp2 hanlp) + { + return hanlp.GetTokFine(text); + } + } +} diff --git a/appsettings.json b/appsettings.json new file mode 100644 index 0000000..fd7f30a --- /dev/null +++ b/appsettings.json @@ -0,0 +1,38 @@ +{ + "CloudBuilder": { + "Culture": "zh-cn", + "DbContexts": { + "MasterDbContext": { + "DbContextTypeName": "MasterDbContext", + "ConnectionString": "Server=localhost\\SQLEXPRESS;Database=Furion;User=sa;Password=owen123456;MultipleActiveResultSets=True;TrustServerCertificate=true", + "DatabaseType": "SqlServer", + "LazyLoadingProxiesEnabled": true, + "AuditEntityEnabled": true, + "AutoMigrationEnabled": true + } + }, + "ServiceRequest": { + "BaseAddress": "http://localhost:5235", //通过Request是需要用户权限及服务器支持 + "DefaultRequestHeaders": { + "Accept": "application/json" + } + }, + "SecurityUser": { + "UserPassword": "1b457798-6d49-4b37-93a4-a6e105cce84b202cb962ac59075b964b07152d234b70", + "UserMobile": "13560851330" + } + }, + "Python": { + "PythonCommand": "D:\\Python38\\python38.dll", + "PythonFile": "D:\\Python38\\hanlp\\hanlp_processor.py", + "surnames_path": "D:/Python38/hanlp/dictionary/surnames.txt", + "titles_path": "D:/Python38/hanlp/dictionary/titles.txt", + "words_path": "D:/Python38/hanlp/dictionary/words.txt", + "salutations_path": "D:/Python38/hanlp/dictionary/salutations.txt", + "interlocutors_path": "D:/Python38/hanlp/dictionary/interlocutors.txt" + }, + "IApplicationService": { + "UserId": "000003" //通过Service是不需要用户权限而直接调用 + }, + "AllowedHosts": "*" +}