using CloudBuilder.AI.Entity; using CloudBuilder.AI.Service; using CloudBuilder.Core.DatabaseAccessor.Entity; using CloudBuilder.Core.DependencyInjection.Task; using CloudBuilder.Topshelf.Python; using CloudBuilder.Topshelf.Utility; using Microsoft.Extensions.DependencyInjection; using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.RegularExpressions; using System.Threading.Tasks; using System.Transactions; namespace CloudBuilder.Topshelf.Task.AI { //task:NovelCutWordTask encoding:gb2312 file_path:D:\NET8\AI\12Novel.txt public class NovelCutWordTask : IScheduleTask { private static readonly Regex _noNumberLetterRegex = new Regex(@"^(?!.*[0-9a-zA-Z]).+$", RegexOptions.Compiled); private readonly Regex _hasPunctuationRegex = new Regex(@"[,。!?;:""''()()【】《》、·¥…—\+\-*/<>==~@#¥%&*()_+{}|:<>?`\-=[\];\\,./{}~!@#¥%……&*()——+|{}】‘;:”“’。,、?·。"'`|〃〔〕〈〉「」『』.〖〗[]{}]", RegexOptions.Compiled); private readonly IApplicationService service; public NovelCutWordTask(IApplicationService service) { this.service = service; } public void Run(Dictionary bodyDict) { PythonCommand python = new PythonCommand(service); Hanlp2 hanlp = new Hanlp2(python); IAiWordService aiService = service.ServiceProvider.GetService(); AiWordEntity[] aiWords = aiService.FindAll(); string encoding = "utf-8"; if (bodyDict.ContainsKey("encoding")) encoding = bodyDict["encoding"]; else Console.WriteLine("文件字符集" + encoding); string fileName = null; if (bodyDict.ContainsKey("encoding")) fileName = bodyDict["file_path"]; else Console.WriteLine("文件路径不存在"); System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance); string[] lines = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding)); int totalLines = lines.Length; int processedLines = 0; // 已处理行数计数器 int totalValidWords = 0; // 可选:统计有效词总数,用于进度详情 Console.WriteLine($"开始处理文件:{fileName}"); Console.WriteLine($"文件总行数:{totalLines}\n"); SegmenterHelper segmenterHelper = new SegmenterHelper(); IEnumerable words; List contents = new List(); List existingContents = new List(); if (aiWords != null && aiWords.Length > 0) existingContents = aiWords.Select(x => x.Content).ToList(); foreach (string line in lines) { processedLines++; // 已处理行数+1 if (string.IsNullOrEmpty(line.Trim())) { // 空行也计入进度,更新进度显示 UpdateProgress(processedLines, totalLines, totalValidWords); continue; } if (string.IsNullOrEmpty(line.Trim())) continue; try { words = segmenterHelper.Cut(line, hanlp); } catch (Exception ex) { Console.Write($"\r{ex.Message}"); continue; } if (words == null || words.Count() == 0) continue; var validDatas = words .Where(str => !string.IsNullOrWhiteSpace(str) // 过滤空/空白字符串 && str.Length > 1 // 过滤长度≤1的字符串 && _noNumberLetterRegex.IsMatch(str) // 过滤含数字/字母的字符串 && !str.StartsWith("第") // 过滤以“第”开头的字符串 && !_hasPunctuationRegex.IsMatch(str) // 核心新增:过滤含任意标点的字符串 ) .Distinct(StringComparer.OrdinalIgnoreCase) // 数组内去重(不区分大小写) .ToList(); // 无有效数据则直接返回 if (!validDatas.Any()) continue; var newDatas = validDatas .Where(str => !existingContents.Contains(str)) .ToList(); // 6. 批量插入(仅当有新数据时执行) if (!newDatas.Any()) { UpdateProgress(processedLines, totalLines, totalValidWords); continue; } contents.AddRange(newDatas); contents = contents .Where(str => !string.IsNullOrWhiteSpace(str)) // 过滤无效字符串 .Distinct(StringComparer.OrdinalIgnoreCase) // 数组内去重(不区分大小写) .ToList(); totalValidWords = contents.Count; // 更新有效词总数 // 3. 实时更新进度(每行处理完后刷新) UpdateProgress(processedLines, totalLines, totalValidWords); } var transactionOptions = new TransactionOptions { IsolationLevel = IsolationLevel.ReadCommitted, Timeout = TimeSpan.FromSeconds(60 * 10) }; try { python.Dispose(); } catch (Exception) { } if (contents.Count() == 0) return; using (TransactionScope _transactionScope = new TransactionScope( TransactionScopeOption.Required, transactionOptions, TransactionScopeAsyncFlowOption.Enabled)) { aiService.Insert(contents.ToArray()); _transactionScope.Complete(); } } private static void UpdateProgress(int processed, int total, int validWords) { double progressPercent = (double)processed / total * 100; int progressBarLength = 50; // 进度条总长度 int filledLength = (int)(progressPercent / 100 * progressBarLength); // 构建进度条(如:[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]) string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]"; string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理:{processed}/{total} 行 | 有效词数:{validWords}"; Console.Write($"\r{progressInfo}"); Console.Out.Flush(); } } }