diff --git a/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs b/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs new file mode 100644 index 0000000..e0cbb47 --- /dev/null +++ b/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs @@ -0,0 +1,316 @@ +using CloudBuilder.Core.DatabaseAccessor.Entity; +using CloudBuilder.Core.Service; +using DocumentFormat.OpenXml.Wordprocessing; +using EnvDTE80; +using Irony.Parsing; +using Microsoft.ML; +using Microsoft.ML.Data; +using Microsoft.ML.Transforms.Onnx; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CloudBuilder.AI.Service.Utility +{ + public class ChineseNameExtractor + { + private BertChineseTokenizer tokenizer; + private OnnxScoringEstimator pipeline; + private PredictionEngine engine; + public ChineseNameExtractor(IApplicationService service) + { + InitMLContext(service); + } + + private void InitMLContext(IApplicationService service) + { + // 1. 初始化 MLContext + var mlContext = new MLContext(); + + string modelDir = service.Configuration["FileServiceSettings:OnnxDirectory"]; + // 2. 检查必要文件 + string modelPath = Path.Combine(modelDir, "model.onnx"); + string vocabPath = Path.Combine(modelDir, "vocab.txt"); + + if (!File.Exists(modelPath) || !File.Exists(vocabPath)) + { + throw new ValidatedException("请确保 model.onnx 和 vocab.txt 在程序目录下。"); + } + + // 3. 加载词汇表与标签映射 + tokenizer = new BertChineseTokenizer(vocabPath); + + // 4. 定义模型输入输出的精确形状 + // 通过 Netron (https://netron.app/) 查看模型结构后，你可以确认这些形状。 + var shapeDictionary = new Dictionary + { + { "input_ids", new[] { 1, 128 } }, + { "attention_mask", new[] { 1, 128 } }, + { "token_type_ids", new[] { 1, 128 } }, + { "logits", new[] { 1, 128, 29 } } // 29 个标签 + }; + + // 5. 构建管道（使用空数据视图和 shapeDictionary） + var emptyData = mlContext.Data.LoadFromEnumerable(new List()); + pipeline = mlContext.Transforms.ApplyOnnxModel( + modelFile: modelPath, + inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" }, + outputColumnNames: new[] { "logits" }, + shapeDictionary: shapeDictionary, + gpuDeviceId: null, // 若要用 GPU，可改为 0 等 + fallbackToCpu: true + ); + + // 6. 拟合并创建预测引擎 + //Console.WriteLine("正在加载 ONNX 模型..."); + var transformer = pipeline.Fit(emptyData); + engine = mlContext.Model.CreatePredictionEngine(transformer); + //Console.WriteLine("模型加载完成。"); + } + + public List ExtractNames(string text) + { + var tokenized = tokenizer.Tokenize(text); + + var input = new NerInput + { + InputIds = tokenized.InputIds, + AttentionMask = tokenized.AttentionMask, + TokenTypeIds = tokenized.TokenTypeIds + }; + + var prediction = engine.Predict(input); + + return ExtractNames(prediction, tokenized, tokenizer.IdToLabel); + } + /// + /// 根据模型输出的 logits 和分词信息，提取出人名（BIO 格式）。 + /// + /// 模型预测输出。 + /// 分词后的结果。 + /// 标签ID到名称的映射数组。 + /// 识别出的人名列表。 + private List ExtractNames(NerOutput output, TokenizedResult tokenized, string[] idToLabel) + { + var names = new List(); + string currentName = ""; + int seqLen = 128; + int numLabels = idToLabel.Length; + + for (int i = 0; i < seqLen; i++) + { + // 跳过填充 token（attention_mask 为 0） + if (tokenized.AttentionMask[i] == 0) + continue; + + int startIdx = i * numLabels; + float maxVal = float.MinValue; + int maxIdx = 0; + for (int j = 0; j < numLabels; j++) + { + float val = output.Logits[startIdx + j]; + if (val > maxVal) + { + maxVal = val; + maxIdx = j; + } + } + + string label = idToLabel[maxIdx]; + string token = tokenized.Tokens[i]; + + if (label == "B-PER") + { + if (currentName != "") + names.Add(currentName); + currentName = token; + } + else if (label == "I-PER") + { + currentName += token; + } + else + { + if (currentName != "") + { + names.Add(currentName); + currentName = ""; + } + } + } + + if (currentName != "") + names.Add(currentName); + return names; + } + + } + + + // ========== 数据模型定义 ========== + // 以下类的 VectorType 和 ColumnName 必须与 shapeDictionary 及 ONNX 模型定义完全一致。 + + /// + /// ONNX 模型的输入。假设 batch=1，序列长度=128，三个 int64 张量。 + /// + public class NerInput + { + [VectorType(1, 128)] + [ColumnName("input_ids")] + public long[] InputIds { get; set; } + + [VectorType(1, 128)] + [ColumnName("attention_mask")] + public long[] AttentionMask { get; set; } + + [VectorType(1, 128)] + [ColumnName("token_type_ids")] + public long[] TokenTypeIds { get; set; } + } + + /// + /// ONNX 模型的输出。假设 logits 形状为 [1, 128, num_labels]。 + /// + public class NerOutput + { + [VectorType(1, 128, 29)] // 与 shapeDictionary 一致 + [ColumnName("logits")] + public float[] Logits { get; set; } + } + + // ========== 分词器与词汇表加载 ========== + + /// + /// 简易的 BERT 中文分词器，按字进行分割，适用于中文 NER 任务。 + /// 特殊 token：[CLS] = 101, [SEP] = 102, [PAD] = 0。 + /// + public class BertChineseTokenizer + { + private readonly Dictionary _tokenToId; + /// + /// 标签ID到名称的映射，其顺序和内容必须与模型输出完全一致。 + /// 此处的映射基于 bert-base-chinese-finetuned-ner 模型常见的标签体系。 + /// + public readonly string[] IdToLabel; + + public BertChineseTokenizer(string vocabPath, int maxSeqLength = 128) + { + // 加载词汇表 + _tokenToId = new Dictionary(); + var lines = File.ReadAllLines(vocabPath); + for (int i = 0; i < lines.Length; i++) + _tokenToId[lines[i]] = i; + + // 默认标签映射，顺序务必与模型输出一致 + IdToLabel = new string[] { + "O", // 0 + "B-ORG", // 1 + "I-ORG", // 2 + "B-PER", // 3 + "I-PER", // 4 + "B-TIME", // 5 + "I-TIME", // 6 + "B-LOC", // 7 + "I-LOC", // 8 + "B-POSITION", // 9 + "I-POSITION", // 10 + "B-COMPANY", // 11 + "I-COMPANY", // 12 + "B-GAME", // 13 + "I-GAME", // 14 + "B-GOVERNMENT",// 15 + "I-GOVERNMENT",// 16 + "B-SCENE", // 17 + "I-SCENE", // 18 + "B-SUBJECT", // 19 + "I-SUBJECT", // 20 + "B-CREATION", // 21 + "I-CREATION", // 22 + "B-FOOD", // 23 + "I-FOOD", // 24 + "B-MOVIE", // 25 + "I-MOVIE", // 26 + "[CLS]", // 27 + "[SEP]" // 28 +}; + } + + /// + /// 对中文句子进行分词，并生成模型所需的 input_ids 等。 + /// + /// 输入的中文句子。 + /// 分词后的结果，包括 token IDs 和 tokens 本身。 + public TokenizedResult Tokenize(string text) + { + const int maxLen = 128; + var inputIds = new long[maxLen]; + var attentionMask = new long[maxLen]; + var tokenTypeIds = new long[maxLen]; + var tokens = new string[maxLen]; + + // 将所有字符当作 token（BERT 中文模型常用） + var chars = text.ToCharArray(); + int charIndex = 0; + + for (int i = 0; i < maxLen; i++) + { + if (i == 0) // [CLS] + { + inputIds[i] = 101; + tokens[i] = "[CLS]"; + } + else if (i == chars.Length + 1) // [SEP] + { + inputIds[i] = 102; + tokens[i] = "[SEP]"; + } + else if (i > chars.Length + 1) // padding + { + inputIds[i] = 0; + tokens[i] = "[PAD]"; + } + else // 实际字符 + { + string c = chars[charIndex].ToString(); + charIndex++; + if (_tokenToId.TryGetValue(c, out int id)) + { + inputIds[i] = id; + tokens[i] = c; + } + else + { + // 如果词汇表没有这个字，用 [UNK] token + inputIds[i] = _tokenToId["[UNK]"]; + tokens[i] = "[UNK]"; + } + } + + // attention_mask：有效 token 为 1，padding 为 0 + attentionMask[i] = (i <= chars.Length + 1) ? 1 : 0; + tokenTypeIds[i] = 0; // 单句都为 0 + } + + return new TokenizedResult + { + InputIds = inputIds, + AttentionMask = attentionMask, + TokenTypeIds = tokenTypeIds, + Tokens = tokens + }; + } + } + + /// + /// 存储分词后的结果。 + /// + public class TokenizedResult + { + public long[] InputIds { get; set; } + public long[] AttentionMask { get; set; } + public long[] TokenTypeIds { get; set; } + public string[] Tokens { get; set; } + } +} diff --git a/CloudBuilder.AI/Entity/AiParagraphViewEntity.auto.cs b/CloudBuilder.AI/Entity/AiParagraphViewEntity.auto.cs new file mode 100644 index 0000000..ebbdd61 --- /dev/null +++ b/CloudBuilder.AI/Entity/AiParagraphViewEntity.auto.cs @@ -0,0 +1,127 @@ +using CloudBuilder.Core.Authorization; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +namespace CloudBuilder.AI.Entity + { + [TypeScript] + [Table("ai_paragraph_view")] + public class AiParagraphViewEntity : ViewEntityBase + { + + public AiParagraphViewEntity() + { + } + + [Column("created_datetime")] + public DateTime CreatedDatetime { get; set; } + [Column("created_by")] + public string CreatedBy { get; set; } + [Column("updated_datetime")] + public DateTime UpdatedDatetime { get; set; } + [Column("updated_by")] + public string UpdatedBy { get; set; } + [Column("guid")] + public string Guid { get; set; } + [Column("chapter_guid")] + public string ChapterGuid { get; set; } + [Column("paragraph_index")] + public int ParagraphIndex { get; set; } + [Column("content")] + public string Content { get; set; } + [Column("dialogue_indc")] + public string? DialogueIndc { get; set; } + [Column("book_guid")] + public string BookGuid { get; set; } + + public const string CREATED_DATETIME = "CreatedDatetime"; + public const string CREATED_BY = "CreatedBy"; + public const string UPDATED_DATETIME = "UpdatedDatetime"; + public const string UPDATED_BY = "UpdatedBy"; + public const string GUID = "Guid"; + public const string CHAPTER_GUID = "ChapterGuid"; + public const string PARAGRAPH_INDEX = "ParagraphIndex"; + public const string CONTENT = "Content"; + public const string DIALOGUE_INDC = "DialogueIndc"; + public const string BOOK_GUID = "BookGuid"; + + public const string DB_NAME_AI_PARAGRAPH_VIEW = "ai_paragraph_view"; + public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,chapter_guid,paragraph_index,content,dialogue_indc,book_guid"; + + public const string DB_CREATED_DATETIME = "created_datetime"; + public const string DB_CREATED_BY = "created_by"; + public const string DB_UPDATED_DATETIME = "updated_datetime"; + public const string DB_UPDATED_BY = "updated_by"; + public const string DB_GUID = "guid"; + public const string DB_CHAPTER_GUID = "chapter_guid"; + public const string DB_PARAGRAPH_INDEX = "paragraph_index"; + public const string DB_CONTENT = "content"; + public const string DB_DIALOGUE_INDC = "dialogue_indc"; + public const string DB_BOOK_GUID = "book_guid"; + + + public static AiParagraphViewEntity GetInstance() + { + return new AiParagraphViewEntity(); + } + + public virtual void CopyFrom(AiParagraphViewEntity source) + { + CopyFrom(source, true); + } + + public virtual void CopyFrom(AiParagraphViewEntity source, bool includeSystemFields) + { + if (source == null) + { + throw new NullReferenceException("Source entity is null."); + } + + this.CreatedDatetime = source.CreatedDatetime; + this.CreatedBy = source.CreatedBy; + this.UpdatedDatetime = source.UpdatedDatetime; + this.UpdatedBy = source.UpdatedBy; + this.Guid = source.Guid; + this.ChapterGuid = source.ChapterGuid; + this.ParagraphIndex = source.ParagraphIndex; + this.Content = source.Content; + this.DialogueIndc = source.DialogueIndc; + this.BookGuid = source.BookGuid; + + if (includeSystemFields) + { + this.CreatedDatetime = source.CreatedDatetime; + this.CreatedBy = source.CreatedBy; + this.UpdatedDatetime = source.UpdatedDatetime; + this.UpdatedBy = source.UpdatedBy; + } + } + + public override string GetSelectFrom() + { + return $" select {DB_NAME_FIELDS} from { DB_NAME_AI_PARAGRAPH_VIEW} with(nolock) "; + } + + public override string GetSelectCountFrom(string sqlWhere) + { + return $" select count(*) from { DB_NAME_AI_PARAGRAPH_VIEW} with(nolock) where 1=1 {sqlWhere}"; + } + + public override string GetSelectTopFrom(int limit, int start,string orderby,string orderbytag, string sqlWhere) + { + if (string.IsNullOrEmpty(orderbytag) || orderbytag.Trim() != "desc") orderbytag = "asc"; + + if (string.IsNullOrEmpty(orderby) || !DB_NAME_FIELDS.Contains(orderby)) orderby = DB_CREATED_DATETIME ; string top = limit > 0 ? $"top {limit}" : ""; + return $" select {top} {DB_NAME_FIELDS} from (select {DB_NAME_FIELDS} ,row_number() over(order by {orderby} {orderbytag}) as num from { DB_NAME_AI_PARAGRAPH_VIEW} with(nolock) where 1=1 {sqlWhere} ) a where num> {limit * (start - 1)}"; + } + + public AiParagraphViewEntity Clone() + { + AiParagraphViewEntity result = new AiParagraphViewEntity(); + result.CopyFrom(this); + + return result; + } + + } +} diff --git a/Data/ai.bak b/Data/ai.bak new file mode 100644 index 0000000..5df0d5a Binary files /dev/null and b/Data/ai.bak differ diff --git a/Data/ai.zip b/Data/ai.zip deleted file mode 100644 index 25e5c87..0000000 Binary files a/Data/ai.zip and /dev/null differ