ow

CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs

@@ -0,0 +1,316 @@
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using CloudBuilder.Core.Service;
+using DocumentFormat.OpenXml.Wordprocessing;
+using EnvDTE80;
+using Irony.Parsing;
+using Microsoft.ML;
+using Microsoft.ML.Data;
+using Microsoft.ML.Transforms.Onnx;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CloudBuilder.AI.Service.Utility
+{
+    public class ChineseNameExtractor
+    {
+        private BertChineseTokenizer tokenizer;
+        private OnnxScoringEstimator pipeline;
+        private PredictionEngine<NerInput, NerOutput> engine;
+        public ChineseNameExtractor(IApplicationService service)
+        {
+            InitMLContext(service);
+        }
+
+        private void InitMLContext(IApplicationService service)
+        {
+            // 1. 初始化 MLContext
+            var mlContext = new MLContext();
+
+            string modelDir = service.Configuration["FileServiceSettings:OnnxDirectory"];
+            // 2. 检查必要文件
+            string modelPath = Path.Combine(modelDir, "model.onnx");
+            string vocabPath = Path.Combine(modelDir, "vocab.txt");
+
+            if (!File.Exists(modelPath) || !File.Exists(vocabPath))
+            {
+                throw new ValidatedException("请确保 model.onnx 和 vocab.txt 在程序目录下。");
+            }
+
+            // 3. 加载词汇表与标签映射
+            tokenizer = new BertChineseTokenizer(vocabPath);
+
+            // 4. 定义模型输入输出的精确形状
+            //    通过 Netron (https://netron.app/) 查看模型结构后，你可以确认这些形状。
+            var shapeDictionary = new Dictionary<string, int[]>
+                {
+                    { "input_ids", new[] { 1, 128 } },
+                    { "attention_mask", new[] { 1, 128 } },
+                    { "token_type_ids", new[] { 1, 128 } },
+                    { "logits", new[] { 1, 128, 29 } }   // 29 个标签
+            };
+
+            // 5. 构建管道（使用空数据视图和 shapeDictionary）
+            var emptyData = mlContext.Data.LoadFromEnumerable(new List<NerInput>());
+            pipeline = mlContext.Transforms.ApplyOnnxModel(
+               modelFile: modelPath,
+               inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" },
+               outputColumnNames: new[] { "logits" },
+               shapeDictionary: shapeDictionary,
+               gpuDeviceId: null,        // 若要用 GPU，可改为 0 等
+               fallbackToCpu: true
+           );
+
+            // 6. 拟合并创建预测引擎
+            //Console.WriteLine("正在加载 ONNX 模型...");
+            var transformer = pipeline.Fit(emptyData);
+            engine = mlContext.Model.CreatePredictionEngine<NerInput, NerOutput>(transformer);
+            //Console.WriteLine("模型加载完成。");
+        }
+
+        public List<string> ExtractNames(string text)
+        {
+            var tokenized = tokenizer.Tokenize(text);
+
+            var input = new NerInput
+            {
+                InputIds = tokenized.InputIds,
+                AttentionMask = tokenized.AttentionMask,
+                TokenTypeIds = tokenized.TokenTypeIds
+            };
+
+            var prediction = engine.Predict(input);
+
+            return ExtractNames(prediction, tokenized, tokenizer.IdToLabel);
+        }
+        /// <summary>
+        /// 根据模型输出的 logits 和分词信息，提取出人名（BIO 格式）。
+        /// </summary>
+        /// <param name="output">模型预测输出。</param>
+        /// <param name="tokenized">分词后的结果。</param>
+        /// <param name="idToLabel">标签ID到名称的映射数组。</param>
+        /// <returns>识别出的人名列表。</returns>
+        private List<string> ExtractNames(NerOutput output, TokenizedResult tokenized, string[] idToLabel)
+        {
+            var names = new List<string>();
+            string currentName = "";
+            int seqLen = 128;
+            int numLabels = idToLabel.Length;
+
+            for (int i = 0; i < seqLen; i++)
+            {
+                // 跳过填充 token（attention_mask 为 0）
+                if (tokenized.AttentionMask[i] == 0)
+                    continue;
+
+                int startIdx = i * numLabels;
+                float maxVal = float.MinValue;
+                int maxIdx = 0;
+                for (int j = 0; j < numLabels; j++)
+                {
+                    float val = output.Logits[startIdx + j];
+                    if (val > maxVal)
+                    {
+                        maxVal = val;
+                        maxIdx = j;
+                    }
+                }
+
+                string label = idToLabel[maxIdx];
+                string token = tokenized.Tokens[i];
+
+                if (label == "B-PER")
+                {
+                    if (currentName != "")
+                        names.Add(currentName);
+                    currentName = token;
+                }
+                else if (label == "I-PER")
+                {
+                    currentName += token;
+                }
+                else
+                {
+                    if (currentName != "")
+                    {
+                        names.Add(currentName);
+                        currentName = "";
+                    }
+                }
+            }
+
+            if (currentName != "")
+                names.Add(currentName);
+            return names;
+        }
+
+    }
+
+
+    // ========== 数据模型定义 ==========
+    // 以下类的 VectorType 和 ColumnName 必须与 shapeDictionary 及 ONNX 模型定义完全一致。
+
+    /// <summary>
+    /// ONNX 模型的输入。假设 batch=1，序列长度=128，三个 int64 张量。
+    /// </summary>
+    public class NerInput
+    {
+        [VectorType(1, 128)]
+        [ColumnName("input_ids")]
+        public long[] InputIds { get; set; }
+
+        [VectorType(1, 128)]
+        [ColumnName("attention_mask")]
+        public long[] AttentionMask { get; set; }
+
+        [VectorType(1, 128)]
+        [ColumnName("token_type_ids")]
+        public long[] TokenTypeIds { get; set; }
+    }
+
+    /// <summary>
+    /// ONNX 模型的输出。假设 logits 形状为 [1, 128, num_labels]。
+    /// </summary>
+    public class NerOutput
+    {
+        [VectorType(1, 128, 29)]   // 与 shapeDictionary 一致
+        [ColumnName("logits")]
+        public float[] Logits { get; set; }
+    }
+
+    // ========== 分词器与词汇表加载 ==========
+
+    /// <summary>
+    /// 简易的 BERT 中文分词器，按字进行分割，适用于中文 NER 任务。
+    /// 特殊 token：[CLS] = 101, [SEP] = 102, [PAD] = 0。
+    /// </summary>
+    public class BertChineseTokenizer
+    {
+        private readonly Dictionary<string, int> _tokenToId;
+        /// <summary>
+        /// 标签ID到名称的映射，其顺序和内容必须与模型输出完全一致。
+        /// 此处的映射基于 bert-base-chinese-finetuned-ner 模型常见的标签体系。
+        /// </summary>
+        public readonly string[] IdToLabel;
+
+        public BertChineseTokenizer(string vocabPath, int maxSeqLength = 128)
+        {
+            // 加载词汇表
+            _tokenToId = new Dictionary<string, int>();
+            var lines = File.ReadAllLines(vocabPath);
+            for (int i = 0; i < lines.Length; i++)
+                _tokenToId[lines[i]] = i;
+
+            // 默认标签映射，顺序务必与模型输出一致
+            IdToLabel = new string[] {
+    "O",           // 0
+    "B-ORG",       // 1
+    "I-ORG",       // 2
+    "B-PER",       // 3
+    "I-PER",       // 4
+    "B-TIME",      // 5
+    "I-TIME",      // 6
+    "B-LOC",       // 7
+    "I-LOC",       // 8
+    "B-POSITION",  // 9
+    "I-POSITION",  // 10
+    "B-COMPANY",   // 11
+    "I-COMPANY",   // 12
+    "B-GAME",      // 13
+    "I-GAME",      // 14
+    "B-GOVERNMENT",// 15
+    "I-GOVERNMENT",// 16
+    "B-SCENE",     // 17
+    "I-SCENE",     // 18
+    "B-SUBJECT",   // 19
+    "I-SUBJECT",   // 20
+    "B-CREATION",  // 21
+    "I-CREATION",  // 22
+    "B-FOOD",      // 23
+    "I-FOOD",      // 24
+    "B-MOVIE",     // 25
+    "I-MOVIE",     // 26
+    "[CLS]",       // 27
+    "[SEP]"        // 28
+};
+        }
+
+        /// <summary>
+        /// 对中文句子进行分词，并生成模型所需的 input_ids 等。
+        /// </summary>
+        /// <param name="text">输入的中文句子。</param>
+        /// <returns>分词后的结果，包括 token IDs 和 tokens 本身。</returns>
+        public TokenizedResult Tokenize(string text)
+        {
+            const int maxLen = 128;
+            var inputIds = new long[maxLen];
+            var attentionMask = new long[maxLen];
+            var tokenTypeIds = new long[maxLen];
+            var tokens = new string[maxLen];
+
+            // 将所有字符当作 token（BERT 中文模型常用）
+            var chars = text.ToCharArray();
+            int charIndex = 0;
+
+            for (int i = 0; i < maxLen; i++)
+            {
+                if (i == 0)  // [CLS]
+                {
+                    inputIds[i] = 101;
+                    tokens[i] = "[CLS]";
+                }
+                else if (i == chars.Length + 1)  // [SEP]
+                {
+                    inputIds[i] = 102;
+                    tokens[i] = "[SEP]";
+                }
+                else if (i > chars.Length + 1)  // padding
+                {
+                    inputIds[i] = 0;
+                    tokens[i] = "[PAD]";
+                }
+                else  // 实际字符
+                {
+                    string c = chars[charIndex].ToString();
+                    charIndex++;
+                    if (_tokenToId.TryGetValue(c, out int id))
+                    {
+                        inputIds[i] = id;
+                        tokens[i] = c;
+                    }
+                    else
+                    {
+                        // 如果词汇表没有这个字，用 [UNK] token
+                        inputIds[i] = _tokenToId["[UNK]"];
+                        tokens[i] = "[UNK]";
+                    }
+                }
+
+                // attention_mask：有效 token 为 1，padding 为 0
+                attentionMask[i] = (i <= chars.Length + 1) ? 1 : 0;
+                tokenTypeIds[i] = 0; // 单句都为 0
+            }
+
+            return new TokenizedResult
+            {
+                InputIds = inputIds,
+                AttentionMask = attentionMask,
+                TokenTypeIds = tokenTypeIds,
+                Tokens = tokens
+            };
+        }
+    }
+
+    /// <summary>
+    /// 存储分词后的结果。
+    /// </summary>
+    public class TokenizedResult
+    {
+        public long[] InputIds { get; set; }
+        public long[] AttentionMask { get; set; }
+        public long[] TokenTypeIds { get; set; }
+        public string[] Tokens { get; set; }
+    }
+}

CloudBuilder.AI/Entity/AiParagraphViewEntity.auto.cs

@@ -0,0 +1,127 @@
+using CloudBuilder.Core.Authorization;
+using CloudBuilder.Core.DatabaseAccessor.Entity;
+using System.ComponentModel.DataAnnotations;
+using System.ComponentModel.DataAnnotations.Schema;
+namespace CloudBuilder.AI.Entity
+ {
+    [TypeScript]
+    [Table("ai_paragraph_view")]
+    public class AiParagraphViewEntity : ViewEntityBase
+    {
+
+    public AiParagraphViewEntity() 
+    {
+    }
+
+    [Column("created_datetime")]
+    public DateTime CreatedDatetime  { get; set; }
+    [Column("created_by")]
+    public string CreatedBy  { get; set; }
+    [Column("updated_datetime")]
+    public DateTime UpdatedDatetime  { get; set; }
+    [Column("updated_by")]
+    public string UpdatedBy  { get; set; }
+    [Column("guid")]
+    public string Guid  { get; set; }
+    [Column("chapter_guid")]
+    public string ChapterGuid  { get; set; }
+    [Column("paragraph_index")]
+    public int ParagraphIndex  { get; set; }
+    [Column("content")]
+    public string Content  { get; set; }
+    [Column("dialogue_indc")]
+    public string? DialogueIndc  { get; set; }
+    [Column("book_guid")]
+    public string BookGuid  { get; set; }
+
+    public const string CREATED_DATETIME = "CreatedDatetime";
+    public const string CREATED_BY = "CreatedBy";
+    public const string UPDATED_DATETIME = "UpdatedDatetime";
+    public const string UPDATED_BY = "UpdatedBy";
+    public const string GUID = "Guid";
+    public const string CHAPTER_GUID = "ChapterGuid";
+    public const string PARAGRAPH_INDEX = "ParagraphIndex";
+    public const string CONTENT = "Content";
+    public const string DIALOGUE_INDC = "DialogueIndc";
+    public const string BOOK_GUID = "BookGuid";
+
+    public const string DB_NAME_AI_PARAGRAPH_VIEW = "ai_paragraph_view";
+    public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,chapter_guid,paragraph_index,content,dialogue_indc,book_guid";
+
+    public const string DB_CREATED_DATETIME = "created_datetime";
+    public const string DB_CREATED_BY = "created_by";
+    public const string DB_UPDATED_DATETIME = "updated_datetime";
+    public const string DB_UPDATED_BY = "updated_by";
+    public const string DB_GUID = "guid";
+    public const string DB_CHAPTER_GUID = "chapter_guid";
+    public const string DB_PARAGRAPH_INDEX = "paragraph_index";
+    public const string DB_CONTENT = "content";
+    public const string DB_DIALOGUE_INDC = "dialogue_indc";
+    public const string DB_BOOK_GUID = "book_guid";
+
+
+    public static AiParagraphViewEntity GetInstance()
+    {
+        return new AiParagraphViewEntity();
+    }
+
+    public virtual void CopyFrom(AiParagraphViewEntity source)
+    {
+        CopyFrom(source, true);
+    }
+
+    public virtual void CopyFrom(AiParagraphViewEntity source, bool includeSystemFields)
+    {
+        if (source == null)
+        {
+            throw new NullReferenceException("Source entity is null.");
+        }
+
+        this.CreatedDatetime =  source.CreatedDatetime;
+        this.CreatedBy =  source.CreatedBy;
+        this.UpdatedDatetime =  source.UpdatedDatetime;
+        this.UpdatedBy =  source.UpdatedBy;
+        this.Guid =  source.Guid;
+        this.ChapterGuid =  source.ChapterGuid;
+        this.ParagraphIndex =  source.ParagraphIndex;
+        this.Content =  source.Content;
+        this.DialogueIndc =  source.DialogueIndc;
+        this.BookGuid =  source.BookGuid;
+
+        if (includeSystemFields)
+        {
+        this.CreatedDatetime =  source.CreatedDatetime;
+        this.CreatedBy =  source.CreatedBy;
+        this.UpdatedDatetime =  source.UpdatedDatetime;
+        this.UpdatedBy =  source.UpdatedBy;
+        }
+    }
+
+    public override string GetSelectFrom()
+    {
+        return $" select {DB_NAME_FIELDS} from { DB_NAME_AI_PARAGRAPH_VIEW}  with(nolock) ";
+    }
+	
+    public override string GetSelectCountFrom(string sqlWhere)
+    {
+        return $" select count(*) from { DB_NAME_AI_PARAGRAPH_VIEW}  with(nolock) where 1=1 {sqlWhere}";
+    }
+	
+    public override string GetSelectTopFrom(int limit, int start,string orderby,string orderbytag,  string sqlWhere)
+    {
+            if (string.IsNullOrEmpty(orderbytag)  || orderbytag.Trim() != "desc") orderbytag = "asc";
+		
+        if (string.IsNullOrEmpty(orderby) || !DB_NAME_FIELDS.Contains(orderby)) orderby =  DB_CREATED_DATETIME ;  		string top = limit > 0 ? $"top {limit}" : "";
+        return $" select {top} {DB_NAME_FIELDS} from (select {DB_NAME_FIELDS}  ,row_number() over(order by {orderby} {orderbytag}) as num from { DB_NAME_AI_PARAGRAPH_VIEW}  with(nolock) where 1=1 {sqlWhere} ) a where num> {limit * (start - 1)}";
+    }
+
+    public AiParagraphViewEntity Clone()
+    {
+        AiParagraphViewEntity result = new AiParagraphViewEntity();
+        result.CopyFrom(this);
+
+        return result;
+    }
+
+    }
+}