diff --git a/CloudBuilder.AI.Gui/AiBookOperationPage.Designer.cs b/CloudBuilder.AI.Gui/AiBookOperationPage.Designer.cs index 9b7c419..751c83e 100644 --- a/CloudBuilder.AI.Gui/AiBookOperationPage.Designer.cs +++ b/CloudBuilder.AI.Gui/AiBookOperationPage.Designer.cs @@ -61,6 +61,9 @@ namespace CloudBuilder.AI.Gui contentCol = new DataGridViewTextBoxColumn(); statusCol = new DataGridViewTextBoxColumn(); tabPage2 = new TabPage(); + tableLayoutPanel7 = new TableLayoutPanel(); + kryptonDataGridView1 = new Krypton.Toolkit.KryptonDataGridView(); + kryptonDataGridView2 = new Krypton.Toolkit.KryptonDataGridView(); tableLayoutPanel2 = new TableLayoutPanel(); statusCombo = new Krypton.Toolkit.KryptonComboBox(); guidLabel = new Krypton.Toolkit.KryptonLabel(); @@ -98,9 +101,6 @@ namespace CloudBuilder.AI.Gui operationPanel = new TableLayoutPanel(); deleteBut = new Krypton.Toolkit.KryptonButton(); amendBut = new Krypton.Toolkit.KryptonButton(); - tableLayoutPanel7 = new TableLayoutPanel(); - kryptonDataGridView1 = new Krypton.Toolkit.KryptonDataGridView(); - kryptonDataGridView2 = new Krypton.Toolkit.KryptonDataGridView(); contentPanel.SuspendLayout(); tableLayoutPanel3.SuspendLayout(); tableLayoutPanel4.SuspendLayout(); @@ -112,6 +112,9 @@ namespace CloudBuilder.AI.Gui ((System.ComponentModel.ISupportInitialize)sentenceGv).BeginInit(); ((System.ComponentModel.ISupportInitialize)dataGv).BeginInit(); tabPage2.SuspendLayout(); + tableLayoutPanel7.SuspendLayout(); + ((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).BeginInit(); + ((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).BeginInit(); tableLayoutPanel2.SuspendLayout(); ((System.ComponentModel.ISupportInitialize)statusCombo).BeginInit(); processPanel.SuspendLayout(); @@ -123,9 +126,6 @@ namespace CloudBuilder.AI.Gui bottomPanel.SuspendLayout(); savePanel.SuspendLayout(); operationPanel.SuspendLayout(); - tableLayoutPanel7.SuspendLayout(); - ((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).BeginInit(); - ((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).BeginInit(); SuspendLayout(); // // contentPanel @@ -266,8 +266,8 @@ namespace CloudBuilder.AI.Gui // tableLayoutPanel6 // tableLayoutPanel6.ColumnCount = 2; - tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F)); - tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F)); + tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 30F)); + tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 70F)); tableLayoutPanel6.Controls.Add(gridViewPagination, 0, 1); tableLayoutPanel6.Controls.Add(sentenceGv, 1, 0); tableLayoutPanel6.Controls.Add(dataGv, 0, 0); @@ -290,7 +290,7 @@ namespace CloudBuilder.AI.Gui gridViewPagination.Margin = new Padding(0); gridViewPagination.Name = "gridViewPagination"; gridViewPagination.Page = 0; - gridViewPagination.Size = new Size(496, 30); + gridViewPagination.Size = new Size(297, 30); gridViewPagination.TabIndex = 1000; gridViewPagination.Total = 0; // @@ -301,11 +301,11 @@ namespace CloudBuilder.AI.Gui sentenceGv.BorderStyle = BorderStyle.None; sentenceGv.Columns.AddRange(new DataGridViewColumn[] { sentenceIndexCol, dialogueIndcCol, personNameCol, sentenceContentCol, genderCol, sentimentCol, sentenceStatusCol }); sentenceGv.Dock = DockStyle.Fill; - sentenceGv.Location = new Point(504, 0); + sentenceGv.Location = new Point(305, 0); sentenceGv.Margin = new Padding(8, 0, 0, 0); sentenceGv.Name = "sentenceGv"; tableLayoutPanel6.SetRowSpan(sentenceGv, 2); - sentenceGv.Size = new Size(488, 349); + sentenceGv.Size = new Size(687, 349); sentenceGv.TabIndex = 4; // // sentenceIndexCol @@ -376,7 +376,7 @@ namespace CloudBuilder.AI.Gui dataGv.Location = new Point(0, 0); dataGv.Margin = new Padding(0); dataGv.Name = "dataGv"; - dataGv.Size = new Size(496, 319); + dataGv.Size = new Size(297, 319); dataGv.TabIndex = 3; dataGv.CellClick += dataGv_CellClick; // @@ -419,6 +419,37 @@ namespace CloudBuilder.AI.Gui tabPage2.Text = "人物角色表"; tabPage2.UseVisualStyleBackColor = true; // + // tableLayoutPanel7 + // + tableLayoutPanel7.ColumnCount = 2; + tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F)); + tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F)); + tableLayoutPanel7.Controls.Add(kryptonDataGridView1, 0, 0); + tableLayoutPanel7.Controls.Add(kryptonDataGridView2, 1, 0); + tableLayoutPanel7.Location = new Point(76, 31); + tableLayoutPanel7.Name = "tableLayoutPanel7"; + tableLayoutPanel7.RowCount = 2; + tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 69.93671F)); + tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 30.0632915F)); + tableLayoutPanel7.Size = new Size(622, 316); + tableLayoutPanel7.TabIndex = 0; + // + // kryptonDataGridView1 + // + kryptonDataGridView1.BorderStyle = BorderStyle.None; + kryptonDataGridView1.Location = new Point(3, 3); + kryptonDataGridView1.Name = "kryptonDataGridView1"; + kryptonDataGridView1.Size = new Size(209, 173); + kryptonDataGridView1.TabIndex = 0; + // + // kryptonDataGridView2 + // + kryptonDataGridView2.BorderStyle = BorderStyle.None; + kryptonDataGridView2.Location = new Point(314, 3); + kryptonDataGridView2.Name = "kryptonDataGridView2"; + kryptonDataGridView2.Size = new Size(209, 173); + kryptonDataGridView2.TabIndex = 0; + // // tableLayoutPanel2 // tableLayoutPanel2.AutoSize = true; @@ -847,37 +878,6 @@ namespace CloudBuilder.AI.Gui amendBut.Values.Text = "修改(&M)"; amendBut.Click += amendBut_Click; // - // tableLayoutPanel7 - // - tableLayoutPanel7.ColumnCount = 2; - tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F)); - tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F)); - tableLayoutPanel7.Controls.Add(kryptonDataGridView1, 0, 0); - tableLayoutPanel7.Controls.Add(kryptonDataGridView2, 1, 0); - tableLayoutPanel7.Location = new Point(76, 31); - tableLayoutPanel7.Name = "tableLayoutPanel7"; - tableLayoutPanel7.RowCount = 2; - tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 69.93671F)); - tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 30.0632915F)); - tableLayoutPanel7.Size = new Size(622, 316); - tableLayoutPanel7.TabIndex = 0; - // - // kryptonDataGridView1 - // - kryptonDataGridView1.BorderStyle = BorderStyle.None; - kryptonDataGridView1.Location = new Point(3, 3); - kryptonDataGridView1.Name = "kryptonDataGridView1"; - kryptonDataGridView1.Size = new Size(209, 173); - kryptonDataGridView1.TabIndex = 0; - // - // kryptonDataGridView2 - // - kryptonDataGridView2.BorderStyle = BorderStyle.None; - kryptonDataGridView2.Location = new Point(314, 3); - kryptonDataGridView2.Name = "kryptonDataGridView2"; - kryptonDataGridView2.Size = new Size(209, 173); - kryptonDataGridView2.TabIndex = 0; - // // AiBookOperationPage // AutoScaleDimensions = new SizeF(7F, 15F); @@ -898,6 +898,9 @@ namespace CloudBuilder.AI.Gui ((System.ComponentModel.ISupportInitialize)sentenceGv).EndInit(); ((System.ComponentModel.ISupportInitialize)dataGv).EndInit(); tabPage2.ResumeLayout(false); + tableLayoutPanel7.ResumeLayout(false); + ((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).EndInit(); + ((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).EndInit(); tableLayoutPanel2.ResumeLayout(false); tableLayoutPanel2.PerformLayout(); ((System.ComponentModel.ISupportInitialize)statusCombo).EndInit(); @@ -913,9 +916,6 @@ namespace CloudBuilder.AI.Gui bottomPanel.PerformLayout(); savePanel.ResumeLayout(false); operationPanel.ResumeLayout(false); - tableLayoutPanel7.ResumeLayout(false); - ((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).EndInit(); - ((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).EndInit(); ResumeLayout(false); } @@ -966,15 +966,15 @@ namespace CloudBuilder.AI.Gui private TableLayoutPanel tableLayoutPanel5; private TabPage tabPage2; private Krypton.Toolkit.KryptonButton processBut; - private Krypton.Toolkit.KryptonDataGridView dataGv; - private TableLayoutPanel tableLayoutPanel6; - private Krypton.Toolkit.KryptonDataGridView sentenceGv; private Krypton.Toolkit.KryptonTextBox bookTypeTxt; - private DataGridViewTextBoxColumn chapterIndexCol; - private DataGridViewTextBoxColumn titleCol; - private DataGridViewTextBoxColumn contentCol; - private DataGridViewTextBoxColumn statusCol; + private TableLayoutPanel processPanel; + private Krypton.Toolkit.KryptonButton extractNamesBut; + private TableLayoutPanel tableLayoutPanel7; + private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView1; + private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView2; + private TableLayoutPanel tableLayoutPanel6; private CloudBuilder.Gui.GridViewPagination gridViewPagination; + private Krypton.Toolkit.KryptonDataGridView sentenceGv; private DataGridViewTextBoxColumn sentenceIndexCol; private Krypton.Toolkit.KryptonDataGridViewCheckBoxColumn dialogueIndcCol; private Krypton.Toolkit.KryptonDataGridViewTextBoxColumn personNameCol; @@ -982,10 +982,10 @@ namespace CloudBuilder.AI.Gui private Krypton.Toolkit.KryptonDataGridViewTextBoxColumn genderCol; private Krypton.Toolkit.KryptonDataGridViewTextBoxColumn sentimentCol; private Krypton.Toolkit.KryptonDataGridViewComboBoxColumn sentenceStatusCol; - private TableLayoutPanel processPanel; - private Krypton.Toolkit.KryptonButton extractNamesBut; - private TableLayoutPanel tableLayoutPanel7; - private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView1; - private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView2; + private Krypton.Toolkit.KryptonDataGridView dataGv; + private DataGridViewTextBoxColumn chapterIndexCol; + private DataGridViewTextBoxColumn titleCol; + private DataGridViewTextBoxColumn contentCol; + private DataGridViewTextBoxColumn statusCol; } } diff --git a/CloudBuilder.AI.Gui/AiBookOperationPage.cs b/CloudBuilder.AI.Gui/AiBookOperationPage.cs index 1d58baf..ed5c4c6 100644 --- a/CloudBuilder.AI.Gui/AiBookOperationPage.cs +++ b/CloudBuilder.AI.Gui/AiBookOperationPage.cs @@ -106,7 +106,7 @@ namespace CloudBuilder.AI.Gui private void InitGridViewPagination() { searchCriteria = new AiChapterSearchCriteria(); - gridViewPagination.SetLimit(searchCriteria.limit); + gridViewPagination.SetLimit(searchCriteria.limit); gridViewPagination.OnPaginationChanged += SearchChapter; } @@ -269,8 +269,9 @@ namespace CloudBuilder.AI.Gui savePanel.Visible = true; operationPanel.Visible = false; - guiOperateHelper.EnableControls(false, topPanel, processPanel, gridViewPagination); + guiOperateHelper.EnableControls(true, bottomPanel, contentPanel); + guiOperateHelper.EnableControls(false, topPanel, processPanel, gridViewPagination); if (this.DisplayMode == GuiDisplayMode.Amend) guiOperateHelper.EnableControls(false, guidTxt); } @@ -684,6 +685,9 @@ namespace CloudBuilder.AI.Gui aiBookRequest.Update(data); ShowMessageBox.ShowInfo(securityMessagePolicy.ComposeMessage(SecurityMessagePolicy.GUI_SUCCEED_UPDATED_NULL)); + + guidSearchTxt.Text = aiBookEntity.Guid; + Search(); } catch (ValidatedException ex) { @@ -779,6 +783,11 @@ namespace CloudBuilder.AI.Gui AiSentenceViewEntity[] aiSentences = aiSentenceRequest.FindByChapterGuid(chapter.Guid); sentenceGv.SetDataSource(aiSentences); + + ShowMessageBox.ShowInfo(securityMessagePolicy.ComposeMessage(SecurityMessagePolicy.GUI_SUCCEED_UPDATED_NULL)); + + guidSearchTxt.Text = data.AiBook.Guid; + Search(); } catch (ValidatedException ex) { diff --git a/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs b/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs index e0cbb47..3594e12 100644 --- a/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs +++ b/CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs @@ -1,24 +1,29 @@ using CloudBuilder.Core.DatabaseAccessor.Entity; using CloudBuilder.Core.Service; -using DocumentFormat.OpenXml.Wordprocessing; -using EnvDTE80; -using Irony.Parsing; using Microsoft.ML; using Microsoft.ML.Data; using Microsoft.ML.Transforms.Onnx; using System; using System.Collections.Generic; +using System.IO; using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace CloudBuilder.AI.Service.Utility { + /// + /// 基于 ONNX 模型的命名实体识别器，支持提取人名及人物角色（人名+职位）。 + /// public class ChineseNameExtractor { private BertChineseTokenizer tokenizer; private OnnxScoringEstimator pipeline; private PredictionEngine engine; + + /// + /// 开启后会将每个 token 的预测标签及置信度输出到控制台，方便调试。 + /// + public bool DebugMode { get; set; } = false; + public ChineseNameExtractor(IApplicationService service) { InitMLContext(service); @@ -26,54 +31,62 @@ namespace CloudBuilder.AI.Service.Utility private void InitMLContext(IApplicationService service) { - // 1. 初始化 MLContext var mlContext = new MLContext(); - string modelDir = service.Configuration["FileServiceSettings:OnnxDirectory"]; - // 2. 检查必要文件 string modelPath = Path.Combine(modelDir, "model.onnx"); string vocabPath = Path.Combine(modelDir, "vocab.txt"); if (!File.Exists(modelPath) || !File.Exists(vocabPath)) - { - throw new ValidatedException("请确保 model.onnx 和 vocab.txt 在程序目录下。"); - } + throw new Exception("请确保 model.onnx 和 vocab.txt 在指定目录下。"); - // 3. 加载词汇表与标签映射 tokenizer = new BertChineseTokenizer(vocabPath); - // 4. 定义模型输入输出的精确形状 - // 通过 Netron (https://netron.app/) 查看模型结构后，你可以确认这些形状。 var shapeDictionary = new Dictionary - { - { "input_ids", new[] { 1, 128 } }, - { "attention_mask", new[] { 1, 128 } }, - { "token_type_ids", new[] { 1, 128 } }, - { "logits", new[] { 1, 128, 29 } } // 29 个标签 + { + { "input_ids", new[] { 1, 128 } }, + { "attention_mask", new[] { 1, 128 } }, + { "token_type_ids", new[] { 1, 128 } }, + { "logits", new[] { 1, 128, tokenizer.LabelCount } } }; - // 5. 构建管道（使用空数据视图和 shapeDictionary） var emptyData = mlContext.Data.LoadFromEnumerable(new List()); pipeline = mlContext.Transforms.ApplyOnnxModel( - modelFile: modelPath, - inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" }, - outputColumnNames: new[] { "logits" }, - shapeDictionary: shapeDictionary, - gpuDeviceId: null, // 若要用 GPU，可改为 0 等 - fallbackToCpu: true - ); + modelFile: modelPath, + inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" }, + outputColumnNames: new[] { "logits" }, + shapeDictionary: shapeDictionary, + gpuDeviceId: null, + fallbackToCpu: true + ); - // 6. 拟合并创建预测引擎 - //Console.WriteLine("正在加载 ONNX 模型..."); var transformer = pipeline.Fit(emptyData); engine = mlContext.Model.CreatePredictionEngine(transformer); - //Console.WriteLine("模型加载完成。"); } + /// + /// 提取纯人名（标签为 B-PER / I-PER 的连续实体）。 + /// public List ExtractNames(string text) { - var tokenized = tokenizer.Tokenize(text); + return ExtractPersonEntities(text, onlyPureName: true); + } + /// + /// 提取人物角色（人名 + 紧接其后的职位，例如“岳堂主”、“张经理”等）。 + /// + public List ExtractPersonRoles(string text) + { + return ExtractPersonEntities(text, onlyPureName: false); + } + + /// + /// 核心实体提取方法，支持合并人名与其后的职位标签。 + /// + /// 输入文本 + /// true 只提取纯人名；false 提取人名+职位复合实体 + private List ExtractPersonEntities(string text, bool onlyPureName) + { + var tokenized = tokenizer.Tokenize(text); var input = new NerInput { InputIds = tokenized.InputIds, @@ -82,35 +95,26 @@ namespace CloudBuilder.AI.Service.Utility }; var prediction = engine.Predict(input); - - return ExtractNames(prediction, tokenized, tokenizer.IdToLabel); - } - /// - /// 根据模型输出的 logits 和分词信息，提取出人名（BIO 格式）。 - /// - /// 模型预测输出。 - /// 分词后的结果。 - /// 标签ID到名称的映射数组。 - /// 识别出的人名列表。 - private List ExtractNames(NerOutput output, TokenizedResult tokenized, string[] idToLabel) - { - var names = new List(); - string currentName = ""; + var idToLabel = tokenizer.IdToLabel; int seqLen = 128; int numLabels = idToLabel.Length; + var entities = new List(); + string currentEntity = ""; + bool insideEntity = false; + bool isPersonEntity = false; // 当前实体是否以 B-PER 开始 + for (int i = 0; i < seqLen; i++) { - // 跳过填充 token（attention_mask 为 0） - if (tokenized.AttentionMask[i] == 0) - continue; + if (tokenized.AttentionMask[i] == 0) continue; // 跳过 padding + // 找到当前 token 的预测标签 int startIdx = i * numLabels; float maxVal = float.MinValue; int maxIdx = 0; for (int j = 0; j < numLabels; j++) { - float val = output.Logits[startIdx + j]; + float val = prediction.Logits[startIdx + j]; if (val > maxVal) { maxVal = val; @@ -121,40 +125,99 @@ namespace CloudBuilder.AI.Service.Utility string label = idToLabel[maxIdx]; string token = tokenized.Tokens[i]; - if (label == "B-PER") + if (DebugMode) + Console.WriteLine($"{token} -> {label} (置信度: {maxVal:F3})"); + + // 判断当前标签类型 + bool isPerStart = label == "B-PER"; + bool isPerInside = label == "I-PER"; + bool isPositionStart = label == "B-POSITION"; + bool isPositionInside = label == "I-POSITION"; + + // ---- 仅提取纯人名的逻辑 ---- + if (onlyPureName) { - if (currentName != "") - names.Add(currentName); - currentName = token; + if (isPerStart) + { + if (insideEntity && currentEntity.Length > 0) entities.Add(currentEntity); + currentEntity = token; + insideEntity = true; + isPersonEntity = true; + } + else if (isPerInside && insideEntity && isPersonEntity) + { + currentEntity += token; + } + else + { + if (insideEntity && currentEntity.Length > 0) + { + entities.Add(currentEntity); + currentEntity = ""; + insideEntity = false; + isPersonEntity = false; + } + } + continue; } - else if (label == "I-PER") + + // ---- 提取人物角色（人名+职位）的逻辑 ---- + if (isPerStart) { - currentName += token; + // 无论之前是否有未结束的实体，都结束之前的，并开始一个新的人名 + if (insideEntity && currentEntity.Length > 0) + entities.Add(currentEntity); + + currentEntity = token; + insideEntity = true; + isPersonEntity = true; + } + else if (isPositionStart) + { + if (insideEntity && isPersonEntity) + { + // 当前正在收集人名，且遇到 B-POSITION，则合并职位 + currentEntity += token; + // 保持 isPersonEntity = true 不变，这样后续的 I-POSITION 也能继续合并 + } + else + { + // 不在人名内部，则视作新实体（提取单独的职位，如“堂主”） + if (insideEntity && currentEntity.Length > 0) + entities.Add(currentEntity); + currentEntity = token; + insideEntity = true; + isPersonEntity = false; // 标记为纯职位实体 + } + } + else if ((isPerInside || isPositionInside) && insideEntity) + { + // 无论是 I-PER 还是 I-POSITION，只要在实体内部都追加 + currentEntity += token; } else { - if (currentName != "") + // 遇到其他标签（O 或其他 B- 开头），结束当前实体 + if (insideEntity && currentEntity.Length > 0) { - names.Add(currentName); - currentName = ""; + entities.Add(currentEntity); + currentEntity = ""; + insideEntity = false; + isPersonEntity = false; } + // 注意：不处理非目标标签本身，因为它们不属于角色 } } - if (currentName != "") - names.Add(currentName); - return names; - } + // 收尾最后一个实体 + if (insideEntity && currentEntity.Length > 0) + entities.Add(currentEntity); + return entities; + } } - - // ========== 数据模型定义 ========== - // 以下类的 VectorType 和 ColumnName 必须与 shapeDictionary 及 ONNX 模型定义完全一致。 - - /// - /// ONNX 模型的输入。假设 batch=1，序列长度=128，三个 int64 张量。 - /// + // ==================== 数据模型 ==================== public class NerInput { [VectorType(1, 128)] @@ -170,78 +233,66 @@ namespace CloudBuilder.AI.Service.Utility public long[] TokenTypeIds { get; set; } } - /// - /// ONNX 模型的输出。假设 logits 形状为 [1, 128, num_labels]。 - /// public class NerOutput { - [VectorType(1, 128, 29)] // 与 shapeDictionary 一致 + [VectorType(1, 128, 29)] // 维度需与模型输出一致 [ColumnName("logits")] public float[] Logits { get; set; } } - // ========== 分词器与词汇表加载 ========== - - /// - /// 简易的 BERT 中文分词器，按字进行分割，适用于中文 NER 任务。 - /// 特殊 token：[CLS] = 101, [SEP] = 102, [PAD] = 0。 - /// + // ==================== 分词器 ==================== public class BertChineseTokenizer { private readonly Dictionary _tokenToId; + /// - /// 标签ID到名称的映射，其顺序和内容必须与模型输出完全一致。 - /// 此处的映射基于 bert-base-chinese-finetuned-ner 模型常见的标签体系。 + /// 标签映射，务必与模型训练时的标签顺序一致。 /// public readonly string[] IdToLabel; + public int LabelCount => IdToLabel.Length; public BertChineseTokenizer(string vocabPath, int maxSeqLength = 128) { - // 加载词汇表 _tokenToId = new Dictionary(); var lines = File.ReadAllLines(vocabPath); for (int i = 0; i < lines.Length; i++) _tokenToId[lines[i]] = i; - // 默认标签映射，顺序务必与模型输出一致 + // 请根据实际模型输出维度调整此数组！ + // 下面的顺序仅为示例，必须与模型 logits 第2维一一对应。 IdToLabel = new string[] { - "O", // 0 - "B-ORG", // 1 - "I-ORG", // 2 - "B-PER", // 3 - "I-PER", // 4 - "B-TIME", // 5 - "I-TIME", // 6 - "B-LOC", // 7 - "I-LOC", // 8 - "B-POSITION", // 9 - "I-POSITION", // 10 - "B-COMPANY", // 11 - "I-COMPANY", // 12 - "B-GAME", // 13 - "I-GAME", // 14 - "B-GOVERNMENT",// 15 - "I-GOVERNMENT",// 16 - "B-SCENE", // 17 - "I-SCENE", // 18 - "B-SUBJECT", // 19 - "I-SUBJECT", // 20 - "B-CREATION", // 21 - "I-CREATION", // 22 - "B-FOOD", // 23 - "I-FOOD", // 24 - "B-MOVIE", // 25 - "I-MOVIE", // 26 - "[CLS]", // 27 - "[SEP]" // 28 -}; + "O", // 0 + "B-ORG", // 1 + "I-ORG", // 2 + "B-PER", // 3 + "I-PER", // 4 + "B-TIME", // 5 + "I-TIME", // 6 + "B-LOC", // 7 + "I-LOC", // 8 + "B-POSITION", // 9 + "I-POSITION", // 10 + "B-COMPANY", // 11 + "I-COMPANY", // 12 + "B-GAME", // 13 + "I-GAME", // 14 + "B-GOVERNMENT", // 15 + "I-GOVERNMENT", // 16 + "B-SCENE", // 17 + "I-SCENE", // 18 + "B-SUBJECT", // 19 + "I-SUBJECT", // 20 + "B-CREATION", // 21 + "I-CREATION", // 22 + "B-FOOD", // 23 + "I-FOOD", // 24 + "B-MOVIE", // 25 + "I-MOVIE", // 26 + "[CLS]", // 27 （如果模型没有这些特殊标签，请删除） + "[SEP]" // 28 + }; } - /// - /// 对中文句子进行分词，并生成模型所需的 input_ids 等。 - /// - /// 输入的中文句子。 - /// 分词后的结果，包括 token IDs 和 tokens 本身。 public TokenizedResult Tokenize(string text) { const int maxLen = 128; @@ -250,28 +301,27 @@ namespace CloudBuilder.AI.Service.Utility var tokenTypeIds = new long[maxLen]; var tokens = new string[maxLen]; - // 将所有字符当作 token（BERT 中文模型常用） var chars = text.ToCharArray(); int charIndex = 0; for (int i = 0; i < maxLen; i++) { - if (i == 0) // [CLS] + if (i == 0) { inputIds[i] = 101; tokens[i] = "[CLS]"; } - else if (i == chars.Length + 1) // [SEP] + else if (i == chars.Length + 1) { inputIds[i] = 102; tokens[i] = "[SEP]"; } - else if (i > chars.Length + 1) // padding + else if (i > chars.Length + 1) { inputIds[i] = 0; tokens[i] = "[PAD]"; } - else // 实际字符 + else { string c = chars[charIndex].ToString(); charIndex++; @@ -282,15 +332,13 @@ namespace CloudBuilder.AI.Service.Utility } else { - // 如果词汇表没有这个字，用 [UNK] token inputIds[i] = _tokenToId["[UNK]"]; tokens[i] = "[UNK]"; } } - // attention_mask：有效 token 为 1，padding 为 0 attentionMask[i] = (i <= chars.Length + 1) ? 1 : 0; - tokenTypeIds[i] = 0; // 单句都为 0 + tokenTypeIds[i] = 0; } return new TokenizedResult @@ -303,9 +351,6 @@ namespace CloudBuilder.AI.Service.Utility } } - /// - /// 存储分词后的结果。 - /// public class TokenizedResult { public long[] InputIds { get; set; } @@ -313,4 +358,4 @@ namespace CloudBuilder.AI.Service.Utility public long[] TokenTypeIds { get; set; } public string[] Tokens { get; set; } } -} +} \ No newline at end of file diff --git a/CloudBuilder.AI/Entity/AiSentenceDialogueViewEntity.auto.cs b/CloudBuilder.AI/Entity/AiSentenceDialogueViewEntity.auto.cs new file mode 100644 index 0000000..60d7ba0 --- /dev/null +++ b/CloudBuilder.AI/Entity/AiSentenceDialogueViewEntity.auto.cs @@ -0,0 +1,162 @@ +using CloudBuilder.Core.Authorization; +using CloudBuilder.Core.DatabaseAccessor.Entity; +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; +namespace CloudBuilder.AI.Entity + { + [TypeScript] + [Table("ai_sentence_dialogue_view")] + public class AiSentenceDialogueViewEntity : ViewEntityBase + { + + public AiSentenceDialogueViewEntity() + { + } + + [Column("created_datetime")] + public DateTime CreatedDatetime { get; set; } + [Column("created_by")] + public string CreatedBy { get; set; } + [Column("updated_datetime")] + public DateTime UpdatedDatetime { get; set; } + [Column("updated_by")] + public string UpdatedBy { get; set; } + [Column("guid")] + public string Guid { get; set; } + [Column("paragraph_guid")] + public string ParagraphGuid { get; set; } + [Column("status")] + public string? Status { get; set; } + [Column("book_guid")] + public string BookGuid { get; set; } + [Column("chapter_guid")] + public string ChapterGuid { get; set; } + [Column("chapter_index")] + public int ChapterIndex { get; set; } + [Column("paragraph_index")] + public int ParagraphIndex { get; set; } + [Column("sentence_index")] + public int SentenceIndex { get; set; } + [Column("dialogue_indc")] + public string DialogueIndc { get; set; } + [Column("person_name")] + public string? PersonName { get; set; } + [Column("gender")] + public string? Gender { get; set; } + [Column("sentiment")] + public string? Sentiment { get; set; } + [Column("content")] + public string Content { get; set; } + + public const string CREATED_DATETIME = "CreatedDatetime"; + public const string CREATED_BY = "CreatedBy"; + public const string UPDATED_DATETIME = "UpdatedDatetime"; + public const string UPDATED_BY = "UpdatedBy"; + public const string GUID = "Guid"; + public const string PARAGRAPH_GUID = "ParagraphGuid"; + public const string STATUS = "Status"; + public const string BOOK_GUID = "BookGuid"; + public const string CHAPTER_GUID = "ChapterGuid"; + public const string CHAPTER_INDEX = "ChapterIndex"; + public const string PARAGRAPH_INDEX = "ParagraphIndex"; + public const string SENTENCE_INDEX = "SentenceIndex"; + public const string DIALOGUE_INDC = "DialogueIndc"; + public const string PERSON_NAME = "PersonName"; + public const string GENDER = "Gender"; + public const string SENTIMENT = "Sentiment"; + public const string CONTENT = "Content"; + + public const string DB_NAME_AI_SENTENCE_DIALOGUE_VIEW = "ai_sentence_dialogue_view"; + public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,paragraph_guid,status,book_guid,chapter_guid,chapter_index,paragraph_index,sentence_index,dialogue_indc,person_name,gender,sentiment,content"; + + public const string DB_CREATED_DATETIME = "created_datetime"; + public const string DB_CREATED_BY = "created_by"; + public const string DB_UPDATED_DATETIME = "updated_datetime"; + public const string DB_UPDATED_BY = "updated_by"; + public const string DB_GUID = "guid"; + public const string DB_PARAGRAPH_GUID = "paragraph_guid"; + public const string DB_STATUS = "status"; + public const string DB_BOOK_GUID = "book_guid"; + public const string DB_CHAPTER_GUID = "chapter_guid"; + public const string DB_CHAPTER_INDEX = "chapter_index"; + public const string DB_PARAGRAPH_INDEX = "paragraph_index"; + public const string DB_SENTENCE_INDEX = "sentence_index"; + public const string DB_DIALOGUE_INDC = "dialogue_indc"; + public const string DB_PERSON_NAME = "person_name"; + public const string DB_GENDER = "gender"; + public const string DB_SENTIMENT = "sentiment"; + public const string DB_CONTENT = "content"; + + + public static AiSentenceDialogueViewEntity GetInstance() + { + return new AiSentenceDialogueViewEntity(); + } + + public virtual void CopyFrom(AiSentenceDialogueViewEntity source) + { + CopyFrom(source, true); + } + + public virtual void CopyFrom(AiSentenceDialogueViewEntity source, bool includeSystemFields) + { + if (source == null) + { + throw new NullReferenceException("Source entity is null."); + } + + this.CreatedDatetime = source.CreatedDatetime; + this.CreatedBy = source.CreatedBy; + this.UpdatedDatetime = source.UpdatedDatetime; + this.UpdatedBy = source.UpdatedBy; + this.Guid = source.Guid; + this.ParagraphGuid = source.ParagraphGuid; + this.Status = source.Status; + this.BookGuid = source.BookGuid; + this.ChapterGuid = source.ChapterGuid; + this.ChapterIndex = source.ChapterIndex; + this.ParagraphIndex = source.ParagraphIndex; + this.SentenceIndex = source.SentenceIndex; + this.DialogueIndc = source.DialogueIndc; + this.PersonName = source.PersonName; + this.Gender = source.Gender; + this.Sentiment = source.Sentiment; + this.Content = source.Content; + + if (includeSystemFields) + { + this.CreatedDatetime = source.CreatedDatetime; + this.CreatedBy = source.CreatedBy; + this.UpdatedDatetime = source.UpdatedDatetime; + this.UpdatedBy = source.UpdatedBy; + } + } + + public override string GetSelectFrom() + { + return $" select {DB_NAME_FIELDS} from { DB_NAME_AI_SENTENCE_DIALOGUE_VIEW} with(nolock) "; + } + + public override string GetSelectCountFrom(string sqlWhere) + { + return $" select count(*) from { DB_NAME_AI_SENTENCE_DIALOGUE_VIEW} with(nolock) where 1=1 {sqlWhere}"; + } + + public override string GetSelectTopFrom(int limit, int start,string orderby,string orderbytag, string sqlWhere) + { + if (string.IsNullOrEmpty(orderbytag) || orderbytag.Trim() != "desc") orderbytag = "asc"; + + if (string.IsNullOrEmpty(orderby) || !DB_NAME_FIELDS.Contains(orderby)) orderby = DB_CREATED_DATETIME ; string top = limit > 0 ? $"top {limit}" : ""; + return $" select {top} {DB_NAME_FIELDS} from (select {DB_NAME_FIELDS} ,row_number() over(order by {orderby} {orderbytag}) as num from { DB_NAME_AI_SENTENCE_DIALOGUE_VIEW} with(nolock) where 1=1 {sqlWhere} ) a where num> {limit * (start - 1)}"; + } + + public AiSentenceDialogueViewEntity Clone() + { + AiSentenceDialogueViewEntity result = new AiSentenceDialogueViewEntity(); + result.CopyFrom(this); + + return result; + } + + } +} diff --git a/CloudBuilder.AI/Entity/AiSentenceViewEntity.auto.cs b/CloudBuilder.AI/Entity/AiSentenceViewEntity.auto.cs index 5429c19..5607434 100644 --- a/CloudBuilder.AI/Entity/AiSentenceViewEntity.auto.cs +++ b/CloudBuilder.AI/Entity/AiSentenceViewEntity.auto.cs @@ -27,8 +27,12 @@ namespace CloudBuilder.AI.Entity public string ParagraphGuid { get; set; } [Column("status")] public string? Status { get; set; } + [Column("book_guid")] + public string BookGuid { get; set; } [Column("chapter_guid")] public string ChapterGuid { get; set; } + [Column("chapter_index")] + public int ChapterIndex { get; set; } [Column("paragraph_index")] public int ParagraphIndex { get; set; } [Column("sentence_index")] @@ -51,7 +55,9 @@ namespace CloudBuilder.AI.Entity public const string GUID = "Guid"; public const string PARAGRAPH_GUID = "ParagraphGuid"; public const string STATUS = "Status"; + public const string BOOK_GUID = "BookGuid"; public const string CHAPTER_GUID = "ChapterGuid"; + public const string CHAPTER_INDEX = "ChapterIndex"; public const string PARAGRAPH_INDEX = "ParagraphIndex"; public const string SENTENCE_INDEX = "SentenceIndex"; public const string DIALOGUE_INDC = "DialogueIndc"; @@ -61,7 +67,7 @@ namespace CloudBuilder.AI.Entity public const string CONTENT = "Content"; public const string DB_NAME_AI_SENTENCE_VIEW = "ai_sentence_view"; - public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,paragraph_guid,status,chapter_guid,paragraph_index,sentence_index,dialogue_indc,person_name,gender,sentiment,content"; + public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,paragraph_guid,status,book_guid,chapter_guid,chapter_index,paragraph_index,sentence_index,dialogue_indc,person_name,gender,sentiment,content"; public const string DB_CREATED_DATETIME = "created_datetime"; public const string DB_CREATED_BY = "created_by"; @@ -70,7 +76,9 @@ namespace CloudBuilder.AI.Entity public const string DB_GUID = "guid"; public const string DB_PARAGRAPH_GUID = "paragraph_guid"; public const string DB_STATUS = "status"; + public const string DB_BOOK_GUID = "book_guid"; public const string DB_CHAPTER_GUID = "chapter_guid"; + public const string DB_CHAPTER_INDEX = "chapter_index"; public const string DB_PARAGRAPH_INDEX = "paragraph_index"; public const string DB_SENTENCE_INDEX = "sentence_index"; public const string DB_DIALOGUE_INDC = "dialogue_indc"; @@ -104,7 +112,9 @@ namespace CloudBuilder.AI.Entity this.Guid = source.Guid; this.ParagraphGuid = source.ParagraphGuid; this.Status = source.Status; + this.BookGuid = source.BookGuid; this.ChapterGuid = source.ChapterGuid; + this.ChapterIndex = source.ChapterIndex; this.ParagraphIndex = source.ParagraphIndex; this.SentenceIndex = source.SentenceIndex; this.DialogueIndc = source.DialogueIndc; diff --git a/CloudBuilder.AI/Entity/CloudBuilderAIRegistEntity.auto.cs b/CloudBuilder.AI/Entity/CloudBuilderAIRegistEntity.auto.cs index a7c8068..606ed85 100644 --- a/CloudBuilder.AI/Entity/CloudBuilderAIRegistEntity.auto.cs +++ b/CloudBuilder.AI/Entity/CloudBuilderAIRegistEntity.auto.cs @@ -41,6 +41,8 @@ namespace CloudBuilder.AI.Entity builder.Entity(eb => { eb.HasNoKey(); }); builder.Entity(eb => { eb.HasNoKey(); }); builder.Entity(eb => { eb.HasNoKey(); }); + builder.Entity(eb => { eb.HasNoKey(); }); + builder.Entity(eb => { eb.HasNoKey(); }); //***END*** } } diff --git a/CloudBuilder.AI/Policy/AIPrefixPolicy.cs b/CloudBuilder.AI/Policy/AIPrefixPolicy.cs index ab60c4e..0b91545 100644 --- a/CloudBuilder.AI/Policy/AIPrefixPolicy.cs +++ b/CloudBuilder.AI/Policy/AIPrefixPolicy.cs @@ -13,6 +13,8 @@ namespace CloudBuilder.AI.Policy public const string AI_BOOK_PREFIX = "B"; public const string AI_BOOK_GUID_FORMAT = "000000"; + public const string AI_CHAPTER = "AI_CHAPTER"; + public const string AI_CHAPTER_GUID_FORMAT = "00000000"; public const string BOOK_DMS_FILE = "BOOK_DMS_FILE"; public const string BOOK_DMS_FILE_BOOK_NAME = "BOOK_NAME";