ow

2026-05-28 15:49:25 +08:00
commit 597b88d075
@@ -0,0 +1,25 @@
+    static void Main()
+    {
+        // 1. 生成 U+4E00 ~ U+9FA5 完整汉字合集
+        List<char> cjkBasicChars = new List<char>();
+        for (int code = 0x4E00; code <= 0x9FA5; code++)
+        {
+            cjkBasicChars.Add((char)code);
+        }
+
+        // 验证数量（应输出 20976）
+        Console.WriteLine($"基础区汉字总数：{cjkBasicChars.Count}");
+
+        // 2. 输出前 10 个汉字（验证生成结果）
+        Console.WriteLine("前 10 个汉字：");
+        for (int i = 0; i < 10; i++)
+        {
+            Console.Write(cjkBasicChars[i] + " ");
+        }
+        // 输出：一 丁 丂 七 丄 丅 丆 万 丈 三 
+
+        // 3. 导出为文本文件（UTF-8 编码，便于查看/使用）
+        string allChars = new string(cjkBasicChars.ToArray());
+        File.WriteAllText("CJK_Basic_Chars.txt", allChars, System.Text.Encoding.UTF8);
+        Console.WriteLine("\n已导出完整汉字集到 CJK_Basic_Chars.txt");
+    }
@@ -0,0 +1,65 @@
+using CloudBuilder.AI.Service;
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using CloudBuilder.Core.DependencyInjection.Task;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Transactions;
+using static System.Runtime.InteropServices.JavaScript.JSType;
+
+namespace CloudBuilder.Topshelf.Task.AI
+{
+    //task:CharacterInsertTask encoding:GB2312 file_path:D:\NET8\AI\Novel.txt
+    public class CharacterInsertTask : IScheduleTask
+    {
+        private readonly IApplicationService service;
+
+        public CharacterInsertTask(IApplicationService service)
+        {
+            this.service = service;
+        }
+
+        public void Run(Dictionary<string, string> bodyDict)
+        {
+            IAiCharacterService aiCharacterService = service.ServiceProvider.GetService<IAiCharacterService>();
+
+            string encoding = "utf-8";
+
+            if (bodyDict.ContainsKey("encoding"))
+                encoding = bodyDict["encoding"];
+            else
+                Console.WriteLine("文件字符集"+ encoding);
+
+                string fileName = null;
+            if (bodyDict.ContainsKey("encoding"))
+                fileName = bodyDict["file_path"];
+            else
+                Console.WriteLine("文件路径不存在");
+
+            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+
+            string content = File.ReadAllText(fileName, Encoding.GetEncoding(encoding));
+
+            var transactionOptions = new TransactionOptions
+            {
+                IsolationLevel = IsolationLevel.ReadCommitted,
+                Timeout = TimeSpan.FromSeconds(60 * 10)
+            };
+
+            using (TransactionScope _transactionScope = new TransactionScope(
+                TransactionScopeOption.Required,
+                transactionOptions,
+                TransactionScopeAsyncFlowOption.Enabled))
+            {
+                aiCharacterService.Insert(content);
+
+                _transactionScope.Complete();
+            }
+
+    
+        }
+    }
+}
@@ -0,0 +1,86 @@
+using CloudBuilder.AI.Entity;
+using CloudBuilder.AI.Service;
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using CloudBuilder.Core.DependencyInjection.Task;
+using EdgeTtsSharp;
+using EdgeTtsSharp.NAudio;
+using EdgeTtsSharp.Structures;
+using Microsoft.EntityFrameworkCore.Query.Internal;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CloudBuilder.Topshelf.Task.AI
+{
+    //task:CharacterTtsTask save_path:D:\NET8\CloudBuilder.Topshelf\mp3 short_name:zh-CN-XiaoxiaoNeural
+    public class CharacterTtsTask : IScheduleTask
+    {
+        private readonly IApplicationService service;
+
+        public CharacterTtsTask(IApplicationService service)
+        {
+            this.service = service;
+        }
+
+        public void Run(Dictionary<string, string> bodyDict)
+        {
+            IAiCharacterService aiService = service.ServiceProvider.GetService<IAiCharacterService>();
+            AiCharacterEntity[] aiCharacters = aiService.FindAll();
+            string shortName = "zh-CN-XiaoxiaoNeural";
+            string savePath = string.Empty;
+            if (bodyDict.ContainsKey("short_name"))
+                shortName = bodyDict["short_name"];
+
+            if (bodyDict.ContainsKey("save_path"))
+                savePath = bodyDict["save_path"];
+
+            int totalLines = aiCharacters.Length;
+            int processedLines = 0; // 已处理行数计数器
+            int totalValidWords = 0; // 可选：统计有效词总数，用于进度详情
+
+            foreach (var c in aiCharacters)
+            {
+                try
+                {
+                    processedLines++;
+                    TtsAsync(shortName, savePath, c).GetAwaiter().GetResult();
+
+                    UpdateProgress(processedLines, totalLines, totalValidWords);
+                }
+                catch (Exception ex)
+                {
+                    Console.WriteLine(ex.Message);
+                }
+
+            }
+        }
+
+        public async System.Threading.Tasks.Task TtsAsync(string shortName, string savePath, AiCharacterEntity aiCharacter)
+        {
+            savePath = Path.Combine(savePath, string.Format("{0}.mp3", aiCharacter.Content));
+
+            if (File.Exists(savePath)) return;
+
+            var voice = await EdgeTts.GetVoice(shortName);
+            await voice.SaveAudioToFile(aiCharacter.Content, savePath);
+        }
+
+        private static void UpdateProgress(int processed, int total, int validWords)
+        {
+            double progressPercent = (double)processed / total * 100;
+            int progressBarLength = 50; // 进度条总长度
+            int filledLength = (int)(progressPercent / 100 * progressBarLength);
+
+            // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]）
+            string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]";
+
+            string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} ";
+            Console.Write($"\r{progressInfo}");
+            Console.Out.Flush();
+        }
+    }
+}
@@ -0,0 +1,102 @@
+using CloudBuilder.AI.Entity;
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using CloudBuilder.Core.DependencyInjection.Task;
+using CloudBuilder.Topshelf.Python;
+using CloudBuilder.Topshelf.Task.Novel.Process;
+using static System.Net.Mime.MediaTypeNames;
+
+namespace CloudBuilder.Topshelf.Task
+{
+    //task:ChineseNameExtractorTask guid:B000006
+    public class ChineseNameExtractorTask : IScheduleTask
+    {
+        private readonly IApplicationService service;
+
+        public ChineseNameExtractorTask(IApplicationService service)
+        {
+            this.service = service;
+        }
+
+        public void Run(Dictionary<string, string> bodyDict)
+        {
+            if (!bodyDict.ContainsKey("guid")) return;
+
+            string guid = bodyDict["guid"];
+
+            PythonCommand python = new PythonCommand(service);
+            try
+            {
+                Hanlp2 hanlp = new Hanlp2(python);
+
+                IRepository<AiSentenceDialogueViewEntity> repository = service.GetRepository<IRepository<AiSentenceDialogueViewEntity>>();
+
+                IRepository<AiSentenceEntity> repositoryAiSentenceEntity = service.GetRepository<IRepository<AiSentenceEntity>>();
+
+                string depRoot;
+
+                List<NamedEntity> ners;
+
+                List<List<MeaningRepresentationParsingEntity>> srls;
+
+                List<HanlpConstituencyNode> cons = null;
+
+                SpeakerAnalysisHelper speakerFeatureLearner = new SpeakerAnalysisHelper();
+
+                AiSentenceDialogueViewEntity[] ents = repository.DetachedEntities.Where(x => x.BookGuid == guid).OrderBy(x => x.ChapterIndex).ThenBy(x => x.ParagraphIndex).ThenBy(x => x.DialogueIndc).ToArray();
+                AiSentenceEntity aiSentenceEntity;
+                string person = null;
+                int paragraphIndex = 0;
+
+                int totalLines = ents.Length;
+                int processedLines = 0; // 已处理行数计数器
+
+                foreach (var ent in ents)
+                {
+                    processedLines++;
+
+                    if (!string.IsNullOrEmpty(person) && paragraphIndex == ent.ParagraphIndex)
+                    {
+                        paragraphIndex = 0;
+                        aiSentenceEntity = repositoryAiSentenceEntity.Entities.Where(x => x.Guid == ent.Guid).FirstOrDefault();
+                        aiSentenceEntity.PersonName = person;
+
+                        repositoryAiSentenceEntity.UpdateNow(aiSentenceEntity);
+                        continue;
+                    }
+
+                    depRoot = hanlp.GetDepRoot(ent.Content);
+                    ners = hanlp.GetHanlpNer(ent.Content);
+                    srls = hanlp.GetHanlpSrl(ent.Content);
+
+                    person = speakerFeatureLearner.GetPerson(depRoot, ners, srls, cons);
+                    paragraphIndex = ent.ParagraphIndex;
+
+                    UpdateProgress(processedLines, totalLines);
+                }
+
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                python.Dispose();
+            }
+        }
+
+        private static void UpdateProgress(int processed, int total)
+        {
+            double progressPercent = (double)processed / total * 100;
+            int progressBarLength = 50; // 进度条总长度
+            int filledLength = (int)(progressPercent / 100 * progressBarLength);
+
+            // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]）
+            string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]";
+
+            string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} ";
+            Console.Write($"\r{progressInfo}");
+            Console.Out.Flush();
+        }
+    }
+}
@@ -0,0 +1,182 @@
+using CloudBuilder.AI.Entity;
+using CloudBuilder.AI.Service;
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using CloudBuilder.Core.DependencyInjection.Task;
+using CloudBuilder.Topshelf.Python;
+using CloudBuilder.Topshelf.Utility;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using System.Transactions;
+
+namespace CloudBuilder.Topshelf.Task.AI
+{
+    //task:NovelCutWordTask encoding:gb2312 file_path:D:\NET8\AI\12Novel.txt
+    public class NovelCutWordTask : IScheduleTask
+    {
+        private static readonly Regex _noNumberLetterRegex = new Regex(@"^(?!.*[0-9a-zA-Z]).+$", RegexOptions.Compiled);
+        private readonly Regex _hasPunctuationRegex = new Regex(@"[，。！？；：""''()（）【】《》、·￥…—\+\-*/<>＝=～@#￥%&*()_+{}|:<>?`\-=[\];\\,./{}~！@#￥%……&*（）——+|{}】‘；：”“’。，、？·｡＂＇｀｜〃〔〕〈〉「」『』．〖〗［］｛｝]",
+    RegexOptions.Compiled);
+
+        private readonly IApplicationService service;
+
+        public NovelCutWordTask(IApplicationService service)
+        {
+            this.service = service;
+        }
+
+        public void Run(Dictionary<string, string> bodyDict)
+        {
+            PythonCommand python = new PythonCommand(service);
+
+            Hanlp2 hanlp = new Hanlp2(python);
+
+            IAiWordService aiService = service.ServiceProvider.GetService<IAiWordService>();
+            AiWordEntity[] aiWords = aiService.FindAll();
+
+            string encoding = "utf-8";
+
+            if (bodyDict.ContainsKey("encoding"))
+                encoding = bodyDict["encoding"];
+            else
+                Console.WriteLine("文件字符集" + encoding);
+
+            string fileName = null;
+            if (bodyDict.ContainsKey("encoding"))
+                fileName = bodyDict["file_path"];
+            else
+                Console.WriteLine("文件路径不存在");
+
+            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+
+            string[] lines = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding));
+
+            int totalLines = lines.Length;
+            int processedLines = 0; // 已处理行数计数器
+            int totalValidWords = 0; // 可选：统计有效词总数，用于进度详情
+
+            Console.WriteLine($"开始处理文件：{fileName}");
+            Console.WriteLine($"文件总行数：{totalLines}\n");
+
+            SegmenterHelper segmenterHelper = new SegmenterHelper();
+
+            IEnumerable<string> words;
+            List<string> contents = new List<string>();
+
+            List<string> existingContents = new List<string>();
+
+            if (aiWords != null && aiWords.Length > 0)
+                existingContents = aiWords.Select(x => x.Content).ToList();
+
+            foreach (string line in lines)
+            {
+                processedLines++; // 已处理行数+1
+   
+                if (string.IsNullOrEmpty(line.Trim()))
+                {
+                    // 空行也计入进度，更新进度显示
+                    UpdateProgress(processedLines, totalLines, totalValidWords);
+                    continue;
+                }
+
+                if (string.IsNullOrEmpty(line.Trim())) continue;
+
+                try
+                {
+                    words = segmenterHelper.Cut(line, hanlp);
+                }
+                catch (Exception ex)
+                {
+                    Console.Write($"\r{ex.Message}");
+                    continue;
+                }
+
+                if (words == null || words.Count() == 0) continue;
+
+                var validDatas = words
+                    .Where(str =>
+                        !string.IsNullOrWhiteSpace(str)          // 过滤空/空白字符串
+                        && str.Length > 1                        // 过滤长度≤1的字符串
+                        && _noNumberLetterRegex.IsMatch(str)     // 过滤含数字/字母的字符串
+                        && !str.StartsWith("第")                 // 过滤以“第”开头的字符串
+                        && !_hasPunctuationRegex.IsMatch(str)    // 核心新增：过滤含任意标点的字符串
+                    )
+                    .Distinct(StringComparer.OrdinalIgnoreCase)  // 数组内去重（不区分大小写）
+                    .ToList();
+
+
+                // 无有效数据则直接返回
+                if (!validDatas.Any()) continue;
+
+                var newDatas = validDatas
+                   .Where(str => !existingContents.Contains(str))
+                   .ToList();
+
+                // 6. 批量插入（仅当有新数据时执行）
+                if (!newDatas.Any())
+                {
+                    UpdateProgress(processedLines, totalLines, totalValidWords);
+                    continue;
+                }
+
+                contents.AddRange(newDatas);
+
+                contents = contents
+                   .Where(str => !string.IsNullOrWhiteSpace(str)) // 过滤无效字符串
+                   .Distinct(StringComparer.OrdinalIgnoreCase)    // 数组内去重（不区分大小写）
+                   .ToList();
+
+                totalValidWords = contents.Count; // 更新有效词总数
+
+                // 3. 实时更新进度（每行处理完后刷新）
+                UpdateProgress(processedLines, totalLines, totalValidWords);
+            }
+
+            var transactionOptions = new TransactionOptions
+            {
+                IsolationLevel = IsolationLevel.ReadCommitted,
+                Timeout = TimeSpan.FromSeconds(60 * 10)
+            };
+
+            try
+            {
+                python.Dispose();
+            }
+            catch (Exception)
+            {
+
+            }
+
+
+            if (contents.Count() == 0) return;
+
+            using (TransactionScope _transactionScope = new TransactionScope(
+                TransactionScopeOption.Required,
+                transactionOptions,
+                TransactionScopeAsyncFlowOption.Enabled))
+            {
+                aiService.Insert(contents.ToArray());
+
+                _transactionScope.Complete();
+            }
+        }
+
+        private static void UpdateProgress(int processed, int total, int validWords)
+        {
+            double progressPercent = (double)processed / total * 100;
+            int progressBarLength = 50; // 进度条总长度
+            int filledLength = (int)(progressPercent / 100 * progressBarLength);
+
+            // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]）
+            string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]";
+
+            string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} 行 | 有效词数：{validWords}";
+            Console.Write($"\r{progressInfo}");
+            Console.Out.Flush();
+        }
+    }
+}
@@ -0,0 +1,64 @@
+using CloudBuilder.AI.Service;
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using CloudBuilder.Core.DependencyInjection.Task;
+using Microsoft.Extensions.DependencyInjection;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using System.Transactions;
+
+namespace CloudBuilder.Topshelf.Task.AI
+{
+    //task:WordInsertTask encoding:utf-8 file_path:D:\NET8\AI\Novel.txt
+    public class WordInsertTask : IScheduleTask
+    {
+        private readonly IApplicationService service;
+
+        public WordInsertTask(IApplicationService service)
+        {
+            this.service = service;
+        }
+
+        public void Run(Dictionary<string, string> bodyDict)
+        {
+            IAiWordService aiService = service.ServiceProvider.GetService<IAiWordService>();
+
+            string encoding = "utf-8";
+
+            if (bodyDict.ContainsKey("encoding"))
+                encoding = bodyDict["encoding"];
+            else
+                Console.WriteLine("文件字符集" + encoding);
+
+            string fileName = null;
+            if (bodyDict.ContainsKey("encoding"))
+                fileName = bodyDict["file_path"];
+            else
+                Console.WriteLine("文件路径不存在");
+
+            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+
+            string[] contents = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding));
+
+            var transactionOptions = new TransactionOptions
+            {
+                IsolationLevel = IsolationLevel.ReadCommitted,
+                Timeout = TimeSpan.FromSeconds(60 * 10)
+            };
+
+            using (TransactionScope _transactionScope = new TransactionScope(
+                TransactionScopeOption.Required,
+                transactionOptions,
+                TransactionScopeAsyncFlowOption.Enabled))
+            {
+                aiService.Insert(contents);
+
+                _transactionScope.Complete();
+            }
+
+
+        }
+    }
+}