CloudBuilder.Topshelf/Task/AI/NovelCutWordTask.cs

using CloudBuilder.AI.Entity;
using CloudBuilder.AI.Service;
using CloudBuilder.Core.DatabaseAccessor.Entity;
using CloudBuilder.Core.DependencyInjection.Task;
using CloudBuilder.Topshelf.Python;
using CloudBuilder.Topshelf.Utility;
using Microsoft.Extensions.DependencyInjection;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using System.Transactions;

namespace CloudBuilder.Topshelf.Task.AI
{
    //task:NovelCutWordTask encoding:gb2312 file_path:D:\NET8\AI\12Novel.txt
    public class NovelCutWordTask : IScheduleTask
    {
        private static readonly Regex _noNumberLetterRegex = new Regex(@"^(?!.*[0-9a-zA-Z]).+$", RegexOptions.Compiled);
        private readonly Regex _hasPunctuationRegex = new Regex(@"[，。！？；：""''()（）【】《》、·￥…—\+\-*/<>＝=～@#￥%&*()_+{}|:<>?`\-=[\];\\,./{}~！@#￥%……&*（）——+|{}】‘；：”“’。，、？·｡＂＇｀｜〃〔〕〈〉「」『』．〖〗［］｛｝]",
    RegexOptions.Compiled);

        private readonly IApplicationService service;

        public NovelCutWordTask(IApplicationService service)
        {
            this.service = service;
        }

        public void Run(Dictionary<string, string> bodyDict)
        {
            PythonCommand python = new PythonCommand(service);

            Hanlp2 hanlp = new Hanlp2(python);

            IAiWordService aiService = service.ServiceProvider.GetService<IAiWordService>();
            AiWordEntity[] aiWords = aiService.FindAll();

            string encoding = "utf-8";

            if (bodyDict.ContainsKey("encoding"))
                encoding = bodyDict["encoding"];
            else
                Console.WriteLine("文件字符集" + encoding);

            string fileName = null;
            if (bodyDict.ContainsKey("encoding"))
                fileName = bodyDict["file_path"];
            else
                Console.WriteLine("文件路径不存在");

            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);

            string[] lines = File.ReadAllLines(fileName, Encoding.GetEncoding(encoding));

            int totalLines = lines.Length;
            int processedLines = 0; // 已处理行数计数器
            int totalValidWords = 0; // 可选：统计有效词总数，用于进度详情

            Console.WriteLine($"开始处理文件：{fileName}");
            Console.WriteLine($"文件总行数：{totalLines}\n");

            SegmenterHelper segmenterHelper = new SegmenterHelper();

            IEnumerable<string> words;
            List<string> contents = new List<string>();

            List<string> existingContents = new List<string>();

            if (aiWords != null && aiWords.Length > 0)
                existingContents = aiWords.Select(x => x.Content).ToList();

            foreach (string line in lines)
            {
                processedLines++; // 已处理行数+1
   
                if (string.IsNullOrEmpty(line.Trim()))
                {
                    // 空行也计入进度，更新进度显示
                    UpdateProgress(processedLines, totalLines, totalValidWords);
                    continue;
                }

                if (string.IsNullOrEmpty(line.Trim())) continue;

                try
                {
                    words = segmenterHelper.Cut(line, hanlp);
                }
                catch (Exception ex)
                {
                    Console.Write($"\r{ex.Message}");
                    continue;
                }

                if (words == null || words.Count() == 0) continue;

                var validDatas = words
                    .Where(str =>
                        !string.IsNullOrWhiteSpace(str)          // 过滤空/空白字符串
                        && str.Length > 1                        // 过滤长度≤1的字符串
                        && _noNumberLetterRegex.IsMatch(str)     // 过滤含数字/字母的字符串
                        && !str.StartsWith("第")                 // 过滤以“第”开头的字符串
                        && !_hasPunctuationRegex.IsMatch(str)    // 核心新增：过滤含任意标点的字符串
                    )
                    .Distinct(StringComparer.OrdinalIgnoreCase)  // 数组内去重（不区分大小写）
                    .ToList();


                // 无有效数据则直接返回
                if (!validDatas.Any()) continue;

                var newDatas = validDatas
                   .Where(str => !existingContents.Contains(str))
                   .ToList();

                // 6. 批量插入（仅当有新数据时执行）
                if (!newDatas.Any())
                {
                    UpdateProgress(processedLines, totalLines, totalValidWords);
                    continue;
                }

                contents.AddRange(newDatas);

                contents = contents
                   .Where(str => !string.IsNullOrWhiteSpace(str)) // 过滤无效字符串
                   .Distinct(StringComparer.OrdinalIgnoreCase)    // 数组内去重（不区分大小写）
                   .ToList();

                totalValidWords = contents.Count; // 更新有效词总数

                // 3. 实时更新进度（每行处理完后刷新）
                UpdateProgress(processedLines, totalLines, totalValidWords);
            }

            var transactionOptions = new TransactionOptions
            {
                IsolationLevel = IsolationLevel.ReadCommitted,
                Timeout = TimeSpan.FromSeconds(60 * 10)
            };

            try
            {
                python.Dispose();
            }
            catch (Exception)
            {

            }


            if (contents.Count() == 0) return;

            using (TransactionScope _transactionScope = new TransactionScope(
                TransactionScopeOption.Required,
                transactionOptions,
                TransactionScopeAsyncFlowOption.Enabled))
            {
                aiService.Insert(contents.ToArray());

                _transactionScope.Complete();
            }
        }

        private static void UpdateProgress(int processed, int total, int validWords)
        {
            double progressPercent = (double)processed / total * 100;
            int progressBarLength = 50; // 进度条总长度
            int filledLength = (int)(progressPercent / 100 * progressBarLength);

            // 构建进度条（如：[██████████░░░░░░░░░░░░░░░░░░░░░░░░░░░░░░]）
            string progressBar = "[" + new string('█', filledLength) + new string('░', progressBarLength - filledLength) + "]";

            string progressInfo = $"{progressBar} {progressPercent:F2}% | 已处理：{processed}/{total} 行 | 有效词数：{validWords}";
            Console.Write($"\r{progressInfo}");
            Console.Out.Flush();
        }
    }
}