itcsx/CloudBuilder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

ow

Browse Source
This commit is contained in:
owenchen 2026-05-26 14:03:34 +08:00
parent 293a11d863
commit e60118eb53
7 changed files with 421 additions and 191 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
CloudBuilder.AI.Gui/AiBookOperationPage.Designer.cs generated
View File

@ -61,6 +61,9 @@ namespace CloudBuilder.AI.Gui
contentCol = new DataGridViewTextBoxColumn();
statusCol = new DataGridViewTextBoxColumn();
tabPage2 = new TabPage();
tableLayoutPanel7 = new TableLayoutPanel();
kryptonDataGridView1 = new Krypton.Toolkit.KryptonDataGridView();
kryptonDataGridView2 = new Krypton.Toolkit.KryptonDataGridView();
tableLayoutPanel2 = new TableLayoutPanel();
statusCombo = new Krypton.Toolkit.KryptonComboBox();
guidLabel = new Krypton.Toolkit.KryptonLabel();
@ -98,9 +101,6 @@ namespace CloudBuilder.AI.Gui
operationPanel = new TableLayoutPanel();
deleteBut = new Krypton.Toolkit.KryptonButton();
amendBut = new Krypton.Toolkit.KryptonButton();
tableLayoutPanel7 = new TableLayoutPanel();
kryptonDataGridView1 = new Krypton.Toolkit.KryptonDataGridView();
kryptonDataGridView2 = new Krypton.Toolkit.KryptonDataGridView();
contentPanel.SuspendLayout();
tableLayoutPanel3.SuspendLayout();
tableLayoutPanel4.SuspendLayout();
@ -112,6 +112,9 @@ namespace CloudBuilder.AI.Gui
((System.ComponentModel.ISupportInitialize)sentenceGv).BeginInit();
((System.ComponentModel.ISupportInitialize)dataGv).BeginInit();
tabPage2.SuspendLayout();
tableLayoutPanel7.SuspendLayout();
((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).BeginInit();
((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).BeginInit();
tableLayoutPanel2.SuspendLayout();
((System.ComponentModel.ISupportInitialize)statusCombo).BeginInit();
processPanel.SuspendLayout();
@ -123,9 +126,6 @@ namespace CloudBuilder.AI.Gui
bottomPanel.SuspendLayout();
savePanel.SuspendLayout();
operationPanel.SuspendLayout();
tableLayoutPanel7.SuspendLayout();
((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).BeginInit();
((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).BeginInit();
SuspendLayout();
//
// contentPanel
@ -266,8 +266,8 @@ namespace CloudBuilder.AI.Gui
// tableLayoutPanel6
//
tableLayoutPanel6.ColumnCount = 2;
tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F));
tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F));
tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 30F));
tableLayoutPanel6.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 70F));
tableLayoutPanel6.Controls.Add(gridViewPagination, 0, 1);
tableLayoutPanel6.Controls.Add(sentenceGv, 1, 0);
tableLayoutPanel6.Controls.Add(dataGv, 0, 0);
@ -290,7 +290,7 @@ namespace CloudBuilder.AI.Gui
gridViewPagination.Margin = new Padding(0);
gridViewPagination.Name = "gridViewPagination";
gridViewPagination.Page = 0;
gridViewPagination.Size = new Size(496, 30);
gridViewPagination.Size = new Size(297, 30);
gridViewPagination.TabIndex = 1000;
gridViewPagination.Total = 0;
//
@ -301,11 +301,11 @@ namespace CloudBuilder.AI.Gui
sentenceGv.BorderStyle = BorderStyle.None;
sentenceGv.Columns.AddRange(new DataGridViewColumn[] { sentenceIndexCol, dialogueIndcCol, personNameCol, sentenceContentCol, genderCol, sentimentCol, sentenceStatusCol });
sentenceGv.Dock = DockStyle.Fill;
sentenceGv.Location = new Point(504, 0);
sentenceGv.Location = new Point(305, 0);
sentenceGv.Margin = new Padding(8, 0, 0, 0);
sentenceGv.Name = "sentenceGv";
tableLayoutPanel6.SetRowSpan(sentenceGv, 2);
sentenceGv.Size = new Size(488, 349);
sentenceGv.Size = new Size(687, 349);
sentenceGv.TabIndex = 4;
//
// sentenceIndexCol
@ -376,7 +376,7 @@ namespace CloudBuilder.AI.Gui
dataGv.Location = new Point(0, 0);
dataGv.Margin = new Padding(0);
dataGv.Name = "dataGv";
dataGv.Size = new Size(496, 319);
dataGv.Size = new Size(297, 319);
dataGv.TabIndex = 3;
dataGv.CellClick += dataGv_CellClick;
//
@ -419,6 +419,37 @@ namespace CloudBuilder.AI.Gui
tabPage2.Text = "人物角色表";
tabPage2.UseVisualStyleBackColor = true;
//
// tableLayoutPanel7
//
tableLayoutPanel7.ColumnCount = 2;
tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F));
tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F));
tableLayoutPanel7.Controls.Add(kryptonDataGridView1, 0, 0);
tableLayoutPanel7.Controls.Add(kryptonDataGridView2, 1, 0);
tableLayoutPanel7.Location = new Point(76, 31);
tableLayoutPanel7.Name = "tableLayoutPanel7";
tableLayoutPanel7.RowCount = 2;
tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 69.93671F));
tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 30.0632915F));
tableLayoutPanel7.Size = new Size(622, 316);
tableLayoutPanel7.TabIndex = 0;
//
// kryptonDataGridView1
//
kryptonDataGridView1.BorderStyle = BorderStyle.None;
kryptonDataGridView1.Location = new Point(3, 3);
kryptonDataGridView1.Name = "kryptonDataGridView1";
kryptonDataGridView1.Size = new Size(209, 173);
kryptonDataGridView1.TabIndex = 0;
//
// kryptonDataGridView2
//
kryptonDataGridView2.BorderStyle = BorderStyle.None;
kryptonDataGridView2.Location = new Point(314, 3);
kryptonDataGridView2.Name = "kryptonDataGridView2";
kryptonDataGridView2.Size = new Size(209, 173);
kryptonDataGridView2.TabIndex = 0;
//
// tableLayoutPanel2
//
tableLayoutPanel2.AutoSize = true;
@ -847,37 +878,6 @@ namespace CloudBuilder.AI.Gui
amendBut.Values.Text = "修改(&M)";
amendBut.Click += amendBut_Click;
//
// tableLayoutPanel7
//
tableLayoutPanel7.ColumnCount = 2;
tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F));
tableLayoutPanel7.ColumnStyles.Add(new ColumnStyle(SizeType.Percent, 50F));
tableLayoutPanel7.Controls.Add(kryptonDataGridView1, 0, 0);
tableLayoutPanel7.Controls.Add(kryptonDataGridView2, 1, 0);
tableLayoutPanel7.Location = new Point(76, 31);
tableLayoutPanel7.Name = "tableLayoutPanel7";
tableLayoutPanel7.RowCount = 2;
tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 69.93671F));
tableLayoutPanel7.RowStyles.Add(new RowStyle(SizeType.Percent, 30.0632915F));
tableLayoutPanel7.Size = new Size(622, 316);
tableLayoutPanel7.TabIndex = 0;
//
// kryptonDataGridView1
//
kryptonDataGridView1.BorderStyle = BorderStyle.None;
kryptonDataGridView1.Location = new Point(3, 3);
kryptonDataGridView1.Name = "kryptonDataGridView1";
kryptonDataGridView1.Size = new Size(209, 173);
kryptonDataGridView1.TabIndex = 0;
//
// kryptonDataGridView2
//
kryptonDataGridView2.BorderStyle = BorderStyle.None;
kryptonDataGridView2.Location = new Point(314, 3);
kryptonDataGridView2.Name = "kryptonDataGridView2";
kryptonDataGridView2.Size = new Size(209, 173);
kryptonDataGridView2.TabIndex = 0;
//
// AiBookOperationPage
//
AutoScaleDimensions = new SizeF(7F, 15F);
@ -898,6 +898,9 @@ namespace CloudBuilder.AI.Gui
((System.ComponentModel.ISupportInitialize)sentenceGv).EndInit();
((System.ComponentModel.ISupportInitialize)dataGv).EndInit();
tabPage2.ResumeLayout(false);
tableLayoutPanel7.ResumeLayout(false);
((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).EndInit();
((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).EndInit();
tableLayoutPanel2.ResumeLayout(false);
tableLayoutPanel2.PerformLayout();
((System.ComponentModel.ISupportInitialize)statusCombo).EndInit();
@ -913,9 +916,6 @@ namespace CloudBuilder.AI.Gui
bottomPanel.PerformLayout();
savePanel.ResumeLayout(false);
operationPanel.ResumeLayout(false);
tableLayoutPanel7.ResumeLayout(false);
((System.ComponentModel.ISupportInitialize)kryptonDataGridView1).EndInit();
((System.ComponentModel.ISupportInitialize)kryptonDataGridView2).EndInit();
ResumeLayout(false);
}
@ -966,15 +966,15 @@ namespace CloudBuilder.AI.Gui
private TableLayoutPanel tableLayoutPanel5;
private TabPage tabPage2;
private Krypton.Toolkit.KryptonButton processBut;
private Krypton.Toolkit.KryptonDataGridView dataGv;
private TableLayoutPanel tableLayoutPanel6;
private Krypton.Toolkit.KryptonDataGridView sentenceGv;
private Krypton.Toolkit.KryptonTextBox bookTypeTxt;
private DataGridViewTextBoxColumn chapterIndexCol;
private DataGridViewTextBoxColumn titleCol;
private DataGridViewTextBoxColumn contentCol;
private DataGridViewTextBoxColumn statusCol;
private TableLayoutPanel processPanel;
private Krypton.Toolkit.KryptonButton extractNamesBut;
private TableLayoutPanel tableLayoutPanel7;
private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView1;
private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView2;
private TableLayoutPanel tableLayoutPanel6;
private CloudBuilder.Gui.GridViewPagination gridViewPagination;
private Krypton.Toolkit.KryptonDataGridView sentenceGv;
private DataGridViewTextBoxColumn sentenceIndexCol;
private Krypton.Toolkit.KryptonDataGridViewCheckBoxColumn dialogueIndcCol;
private Krypton.Toolkit.KryptonDataGridViewTextBoxColumn personNameCol;
@ -982,10 +982,10 @@ namespace CloudBuilder.AI.Gui
private Krypton.Toolkit.KryptonDataGridViewTextBoxColumn genderCol;
private Krypton.Toolkit.KryptonDataGridViewTextBoxColumn sentimentCol;
private Krypton.Toolkit.KryptonDataGridViewComboBoxColumn sentenceStatusCol;
private TableLayoutPanel processPanel;
private Krypton.Toolkit.KryptonButton extractNamesBut;
private TableLayoutPanel tableLayoutPanel7;
private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView1;
private Krypton.Toolkit.KryptonDataGridView kryptonDataGridView2;
private Krypton.Toolkit.KryptonDataGridView dataGv;
private DataGridViewTextBoxColumn chapterIndexCol;
private DataGridViewTextBoxColumn titleCol;
private DataGridViewTextBoxColumn contentCol;
private DataGridViewTextBoxColumn statusCol;
}
}

11
CloudBuilder.AI.Gui/AiBookOperationPage.cs
View File

@ -269,8 +269,9 @@ namespace CloudBuilder.AI.Gui
savePanel.Visible = true;
operationPanel.Visible = false;
guiOperateHelper.EnableControls(false, topPanel, processPanel, gridViewPagination);
guiOperateHelper.EnableControls(true, bottomPanel, contentPanel);
guiOperateHelper.EnableControls(false, topPanel, processPanel, gridViewPagination);
if (this.DisplayMode == GuiDisplayMode.Amend) guiOperateHelper.EnableControls(false, guidTxt);
}
@ -684,6 +685,9 @@ namespace CloudBuilder.AI.Gui
aiBookRequest.Update(data);
ShowMessageBox.ShowInfo(securityMessagePolicy.ComposeMessage(SecurityMessagePolicy.GUI_SUCCEED_UPDATED_NULL));
guidSearchTxt.Text = aiBookEntity.Guid;
Search();
}
catch (ValidatedException ex)
{
@ -779,6 +783,11 @@ namespace CloudBuilder.AI.Gui
AiSentenceViewEntity[] aiSentences = aiSentenceRequest.FindByChapterGuid(chapter.Guid);
sentenceGv.SetDataSource(aiSentences);
ShowMessageBox.ShowInfo(securityMessagePolicy.ComposeMessage(SecurityMessagePolicy.GUI_SUCCEED_UPDATED_NULL));
guidSearchTxt.Text = data.AiBook.Guid;
Search();
}
catch (ValidatedException ex)
{

303
CloudBuilder.AI.Service/Utility/ChineseNameExtractor.cs
View File

@ -1,24 +1,29 @@
using CloudBuilder.Core.DatabaseAccessor.Entity;
using CloudBuilder.Core.Service;
using DocumentFormat.OpenXml.Wordprocessing;
using EnvDTE80;
using Irony.Parsing;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.Transforms.Onnx;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
namespace CloudBuilder.AI.Service.Utility
{
/// <summary>
/// 基于 ONNX 模型的命名实体识别器，支持提取人名及人物角色（人名+职位）。
/// </summary>
public class ChineseNameExtractor
{
private BertChineseTokenizer tokenizer;
private OnnxScoringEstimator pipeline;
private PredictionEngine<NerInput, NerOutput> engine;
/// <summary>
/// 开启后会将每个 token 的预测标签及置信度输出到控制台，方便调试。
/// </summary>
public bool DebugMode { get; set; } = false;
public ChineseNameExtractor(IApplicationService service)
{
InitMLContext(service);
@ -26,54 +31,62 @@ namespace CloudBuilder.AI.Service.Utility
private void InitMLContext(IApplicationService service)
{
// 1. 初始化 MLContext
var mlContext = new MLContext();
string modelDir = service.Configuration["FileServiceSettings:OnnxDirectory"];
// 2. 检查必要文件
string modelPath = Path.Combine(modelDir, "model.onnx");
string vocabPath = Path.Combine(modelDir, "vocab.txt");
if (!File.Exists(modelPath) || !File.Exists(vocabPath))
{
throw new ValidatedException("请确保 model.onnx 和 vocab.txt 在程序目录下。");
}
throw new Exception("请确保 model.onnx 和 vocab.txt 在指定目录下。");
// 3. 加载词汇表与标签映射
tokenizer = new BertChineseTokenizer(vocabPath);
// 4. 定义模型输入输出的精确形状
// 通过 Netron (https://netron.app/) 查看模型结构后，你可以确认这些形状。
var shapeDictionary = new Dictionary<string, int[]>
{
{ "input_ids", new[] { 1, 128 } },
{ "attention_mask", new[] { 1, 128 } },
{ "token_type_ids", new[] { 1, 128 } },
{ "logits", new[] { 1, 128, 29 } } // 29 个标签
{
{ "input_ids", new[] { 1, 128 } },
{ "attention_mask", new[] { 1, 128 } },
{ "token_type_ids", new[] { 1, 128 } },
{ "logits", new[] { 1, 128, tokenizer.LabelCount } }
};
// 5. 构建管道（使用空数据视图和 shapeDictionary）
var emptyData = mlContext.Data.LoadFromEnumerable(new List<NerInput>());
pipeline = mlContext.Transforms.ApplyOnnxModel(
modelFile: modelPath,
inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" },
outputColumnNames: new[] { "logits" },
shapeDictionary: shapeDictionary,
gpuDeviceId: null, // 若要用 GPU，可改为 0 等
fallbackToCpu: true
);
modelFile: modelPath,
inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" },
outputColumnNames: new[] { "logits" },
shapeDictionary: shapeDictionary,
gpuDeviceId: null,
fallbackToCpu: true
);
// 6. 拟合并创建预测引擎
//Console.WriteLine("正在加载 ONNX 模型...");
var transformer = pipeline.Fit(emptyData);
engine = mlContext.Model.CreatePredictionEngine<NerInput, NerOutput>(transformer);
//Console.WriteLine("模型加载完成。");
}
/// <summary>
/// 提取纯人名（标签为 B-PER / I-PER 的连续实体）。
/// </summary>
public List<string> ExtractNames(string text)
{
var tokenized = tokenizer.Tokenize(text);
return ExtractPersonEntities(text, onlyPureName: true);
}
/// <summary>
/// 提取人物角色（人名 + 紧接其后的职位，例如“岳堂主”、“张经理”等）。
/// </summary>
public List<string> ExtractPersonRoles(string text)
{
return ExtractPersonEntities(text, onlyPureName: false);
}
/// <summary>
/// 核心实体提取方法，支持合并人名与其后的职位标签。
/// </summary>
/// <param name="text">输入文本</param>
/// <param name="onlyPureName">true 只提取纯人名；false 提取人名+职位复合实体</param>
private List<string> ExtractPersonEntities(string text, bool onlyPureName)
{
var tokenized = tokenizer.Tokenize(text);
var input = new NerInput
{
InputIds = tokenized.InputIds,
@ -82,35 +95,26 @@ namespace CloudBuilder.AI.Service.Utility
};
var prediction = engine.Predict(input);
return ExtractNames(prediction, tokenized, tokenizer.IdToLabel);
}
/// <summary>
/// 根据模型输出的 logits 和分词信息，提取出人名（BIO 格式）。
/// </summary>
/// <param name="output">模型预测输出。</param>
/// <param name="tokenized">分词后的结果。</param>
/// <param name="idToLabel">标签ID到名称的映射数组。</param>
/// <returns>识别出的人名列表。</returns>
private List<string> ExtractNames(NerOutput output, TokenizedResult tokenized, string[] idToLabel)
{
var names = new List<string>();
string currentName = "";
var idToLabel = tokenizer.IdToLabel;
int seqLen = 128;
int numLabels = idToLabel.Length;
var entities = new List<string>();
string currentEntity = "";
bool insideEntity = false;
bool isPersonEntity = false; // 当前实体是否以 B-PER 开始
for (int i = 0; i < seqLen; i++)
{
// 跳过填充 token（attention_mask 为 0）
if (tokenized.AttentionMask[i] == 0)
continue;
if (tokenized.AttentionMask[i] == 0) continue; // 跳过 padding
// 找到当前 token 的预测标签
int startIdx = i * numLabels;
float maxVal = float.MinValue;
int maxIdx = 0;
for (int j = 0; j < numLabels; j++)
{
float val = output.Logits[startIdx + j];
float val = prediction.Logits[startIdx + j];
if (val > maxVal)
{
maxVal = val;
@ -121,40 +125,99 @@ namespace CloudBuilder.AI.Service.Utility
string label = idToLabel[maxIdx];
string token = tokenized.Tokens[i];
if (label == "B-PER")
if (DebugMode)
Console.WriteLine($"{token} -> {label} (置信度: {maxVal:F3})");
// 判断当前标签类型
bool isPerStart = label == "B-PER";
bool isPerInside = label == "I-PER";
bool isPositionStart = label == "B-POSITION";
bool isPositionInside = label == "I-POSITION";
// ---- 仅提取纯人名的逻辑 ----
if (onlyPureName)
{
if (currentName != "")
names.Add(currentName);
currentName = token;
if (isPerStart)
{
if (insideEntity && currentEntity.Length > 0) entities.Add(currentEntity);
currentEntity = token;
insideEntity = true;
isPersonEntity = true;
}
else if (isPerInside && insideEntity && isPersonEntity)
{
currentEntity += token;
}
else
{
if (insideEntity && currentEntity.Length > 0)
{
entities.Add(currentEntity);
currentEntity = "";
insideEntity = false;
isPersonEntity = false;
}
}
continue;
}
else if (label == "I-PER")
// ---- 提取人物角色（人名+职位）的逻辑 ----
if (isPerStart)
{
currentName += token;
// 无论之前是否有未结束的实体，都结束之前的，并开始一个新的人名
if (insideEntity && currentEntity.Length > 0)
entities.Add(currentEntity);
currentEntity = token;
insideEntity = true;
isPersonEntity = true;
}
else if (isPositionStart)
{
if (insideEntity && isPersonEntity)
{
// 当前正在收集人名，且遇到 B-POSITION，则合并职位
currentEntity += token;
// 保持 isPersonEntity = true 不变，这样后续的 I-POSITION 也能继续合并
}
else
{
// 不在人名内部，则视作新实体（提取单独的职位，如“堂主”）
if (insideEntity && currentEntity.Length > 0)
entities.Add(currentEntity);
currentEntity = token;
insideEntity = true;
isPersonEntity = false; // 标记为纯职位实体
}
}
else if ((isPerInside || isPositionInside) && insideEntity)
{
// 无论是 I-PER 还是 I-POSITION，只要在实体内部都追加
currentEntity += token;
}
else
{
if (currentName != "")
// 遇到其他标签（O 或其他 B- 开头），结束当前实体
if (insideEntity && currentEntity.Length > 0)
{
names.Add(currentName);
currentName = "";
entities.Add(currentEntity);
currentEntity = "";
insideEntity = false;
isPersonEntity = false;
}
// 注意：不处理非目标标签本身，因为它们不属于角色
}
}
if (currentName != "")
names.Add(currentName);
return names;
}
// 收尾最后一个实体
if (insideEntity && currentEntity.Length > 0)
entities.Add(currentEntity);
return entities;
}
}
// ========== 数据模型定义 ==========
// 以下类的 VectorType 和 ColumnName 必须与 shapeDictionary 及 ONNX 模型定义完全一致。
/// <summary>
/// ONNX 模型的输入。假设 batch=1，序列长度=128，三个 int64 张量。
/// </summary>
// ==================== 数据模型 ====================
public class NerInput
{
[VectorType(1, 128)]
@ -170,78 +233,66 @@ namespace CloudBuilder.AI.Service.Utility
public long[] TokenTypeIds { get; set; }
}
/// <summary>
/// ONNX 模型的输出。假设 logits 形状为 [1, 128, num_labels]。
/// </summary>
public class NerOutput
{
[VectorType(1, 128, 29)] // 与 shapeDictionary 一致
[VectorType(1, 128, 29)] // 维度需与模型输出一致
[ColumnName("logits")]
public float[] Logits { get; set; }
}
// ========== 分词器与词汇表加载 ==========
/// <summary>
/// 简易的 BERT 中文分词器，按字进行分割，适用于中文 NER 任务。
/// 特殊 token：[CLS] = 101, [SEP] = 102, [PAD] = 0。
/// </summary>
// ==================== 分词器 ====================
public class BertChineseTokenizer
{
private readonly Dictionary<string, int> _tokenToId;
/// <summary>
/// 标签ID到名称的映射，其顺序和内容必须与模型输出完全一致。
/// 此处的映射基于 bert-base-chinese-finetuned-ner 模型常见的标签体系。
/// 标签映射，务必与模型训练时的标签顺序一致。
/// </summary>
public readonly string[] IdToLabel;
public int LabelCount => IdToLabel.Length;
public BertChineseTokenizer(string vocabPath, int maxSeqLength = 128)
{
// 加载词汇表
_tokenToId = new Dictionary<string, int>();
var lines = File.ReadAllLines(vocabPath);
for (int i = 0; i < lines.Length; i++)
_tokenToId[lines[i]] = i;
// 默认标签映射，顺序务必与模型输出一致
// 请根据实际模型输出维度调整此数组！
// 下面的顺序仅为示例，必须与模型 logits 第2维一一对应。
IdToLabel = new string[] {
"O", // 0
"B-ORG", // 1
"I-ORG", // 2
"B-PER", // 3
"I-PER", // 4
"B-TIME", // 5
"I-TIME", // 6
"B-LOC", // 7
"I-LOC", // 8
"B-POSITION", // 9
"I-POSITION", // 10
"B-COMPANY", // 11
"I-COMPANY", // 12
"B-GAME", // 13
"I-GAME", // 14
"B-GOVERNMENT",// 15
"I-GOVERNMENT",// 16
"B-SCENE", // 17
"I-SCENE", // 18
"B-SUBJECT", // 19
"I-SUBJECT", // 20
"B-CREATION", // 21
"I-CREATION", // 22
"B-FOOD", // 23
"I-FOOD", // 24
"B-MOVIE", // 25
"I-MOVIE", // 26
"[CLS]", // 27
"[SEP]" // 28
};
"O", // 0
"B-ORG", // 1
"I-ORG", // 2
"B-PER", // 3
"I-PER", // 4
"B-TIME", // 5
"I-TIME", // 6
"B-LOC", // 7
"I-LOC", // 8
"B-POSITION", // 9
"I-POSITION", // 10
"B-COMPANY", // 11
"I-COMPANY", // 12
"B-GAME", // 13
"I-GAME", // 14
"B-GOVERNMENT", // 15
"I-GOVERNMENT", // 16
"B-SCENE", // 17
"I-SCENE", // 18
"B-SUBJECT", // 19
"I-SUBJECT", // 20
"B-CREATION", // 21
"I-CREATION", // 22
"B-FOOD", // 23
"I-FOOD", // 24
"B-MOVIE", // 25
"I-MOVIE", // 26
"[CLS]", // 27 （如果模型没有这些特殊标签，请删除）
"[SEP]" // 28
};
}
/// <summary>
/// 对中文句子进行分词，并生成模型所需的 input_ids 等。
/// </summary>
/// <param name="text">输入的中文句子。</param>
/// <returns>分词后的结果，包括 token IDs 和 tokens 本身。</returns>
public TokenizedResult Tokenize(string text)
{
const int maxLen = 128;
@ -250,28 +301,27 @@ namespace CloudBuilder.AI.Service.Utility
var tokenTypeIds = new long[maxLen];
var tokens = new string[maxLen];
// 将所有字符当作 token（BERT 中文模型常用）
var chars = text.ToCharArray();
int charIndex = 0;
for (int i = 0; i < maxLen; i++)
{
if (i == 0) // [CLS]
if (i == 0)
{
inputIds[i] = 101;
tokens[i] = "[CLS]";
}
else if (i == chars.Length + 1) // [SEP]
else if (i == chars.Length + 1)
{
inputIds[i] = 102;
tokens[i] = "[SEP]";
}
else if (i > chars.Length + 1) // padding
else if (i > chars.Length + 1)
{
inputIds[i] = 0;
tokens[i] = "[PAD]";
}
else // 实际字符
else
{
string c = chars[charIndex].ToString();
charIndex++;
@ -282,15 +332,13 @@ namespace CloudBuilder.AI.Service.Utility
}
else
{
// 如果词汇表没有这个字，用 [UNK] token
inputIds[i] = _tokenToId["[UNK]"];
tokens[i] = "[UNK]";
}
}
// attention_mask：有效 token 为 1，padding 为 0
attentionMask[i] = (i <= chars.Length + 1) ? 1 : 0;
tokenTypeIds[i] = 0; // 单句都为 0
tokenTypeIds[i] = 0;
}
return new TokenizedResult
@ -303,9 +351,6 @@ namespace CloudBuilder.AI.Service.Utility
}
}
/// <summary>
/// 存储分词后的结果。
/// </summary>
public class TokenizedResult
{
public long[] InputIds { get; set; }

162
CloudBuilder.AI/Entity/AiSentenceDialogueViewEntity.auto.cs Normal file
View File

@ -0,0 +1,162 @@
using CloudBuilder.Core.Authorization;
using CloudBuilder.Core.DatabaseAccessor.Entity;
using System.ComponentModel.DataAnnotations;
using System.ComponentModel.DataAnnotations.Schema;
namespace CloudBuilder.AI.Entity
{
[TypeScript]
[Table("ai_sentence_dialogue_view")]
public class AiSentenceDialogueViewEntity : ViewEntityBase
{
public AiSentenceDialogueViewEntity()
{
}
[Column("created_datetime")]
public DateTime CreatedDatetime { get; set; }
[Column("created_by")]
public string CreatedBy { get; set; }
[Column("updated_datetime")]
public DateTime UpdatedDatetime { get; set; }
[Column("updated_by")]
public string UpdatedBy { get; set; }
[Column("guid")]
public string Guid { get; set; }
[Column("paragraph_guid")]
public string ParagraphGuid { get; set; }
[Column("status")]
public string? Status { get; set; }
[Column("book_guid")]
public string BookGuid { get; set; }
[Column("chapter_guid")]
public string ChapterGuid { get; set; }
[Column("chapter_index")]
public int ChapterIndex { get; set; }
[Column("paragraph_index")]
public int ParagraphIndex { get; set; }
[Column("sentence_index")]
public int SentenceIndex { get; set; }
[Column("dialogue_indc")]
public string DialogueIndc { get; set; }
[Column("person_name")]
public string? PersonName { get; set; }
[Column("gender")]
public string? Gender { get; set; }
[Column("sentiment")]
public string? Sentiment { get; set; }
[Column("content")]
public string Content { get; set; }
public const string CREATED_DATETIME = "CreatedDatetime";
public const string CREATED_BY = "CreatedBy";
public const string UPDATED_DATETIME = "UpdatedDatetime";
public const string UPDATED_BY = "UpdatedBy";
public const string GUID = "Guid";
public const string PARAGRAPH_GUID = "ParagraphGuid";
public const string STATUS = "Status";
public const string BOOK_GUID = "BookGuid";
public const string CHAPTER_GUID = "ChapterGuid";
public const string CHAPTER_INDEX = "ChapterIndex";
public const string PARAGRAPH_INDEX = "ParagraphIndex";
public const string SENTENCE_INDEX = "SentenceIndex";
public const string DIALOGUE_INDC = "DialogueIndc";
public const string PERSON_NAME = "PersonName";
public const string GENDER = "Gender";
public const string SENTIMENT = "Sentiment";
public const string CONTENT = "Content";
public const string DB_NAME_AI_SENTENCE_DIALOGUE_VIEW = "ai_sentence_dialogue_view";
public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,paragraph_guid,status,book_guid,chapter_guid,chapter_index,paragraph_index,sentence_index,dialogue_indc,person_name,gender,sentiment,content";
public const string DB_CREATED_DATETIME = "created_datetime";
public const string DB_CREATED_BY = "created_by";
public const string DB_UPDATED_DATETIME = "updated_datetime";
public const string DB_UPDATED_BY = "updated_by";
public const string DB_GUID = "guid";
public const string DB_PARAGRAPH_GUID = "paragraph_guid";
public const string DB_STATUS = "status";
public const string DB_BOOK_GUID = "book_guid";
public const string DB_CHAPTER_GUID = "chapter_guid";
public const string DB_CHAPTER_INDEX = "chapter_index";
public const string DB_PARAGRAPH_INDEX = "paragraph_index";
public const string DB_SENTENCE_INDEX = "sentence_index";
public const string DB_DIALOGUE_INDC = "dialogue_indc";
public const string DB_PERSON_NAME = "person_name";
public const string DB_GENDER = "gender";
public const string DB_SENTIMENT = "sentiment";
public const string DB_CONTENT = "content";
public static AiSentenceDialogueViewEntity GetInstance()
{
return new AiSentenceDialogueViewEntity();
}
public virtual void CopyFrom(AiSentenceDialogueViewEntity source)
{
CopyFrom(source, true);
}
public virtual void CopyFrom(AiSentenceDialogueViewEntity source, bool includeSystemFields)
{
if (source == null)
{
throw new NullReferenceException("Source entity is null.");
}
this.CreatedDatetime = source.CreatedDatetime;
this.CreatedBy = source.CreatedBy;
this.UpdatedDatetime = source.UpdatedDatetime;
this.UpdatedBy = source.UpdatedBy;
this.Guid = source.Guid;
this.ParagraphGuid = source.ParagraphGuid;
this.Status = source.Status;
this.BookGuid = source.BookGuid;
this.ChapterGuid = source.ChapterGuid;
this.ChapterIndex = source.ChapterIndex;
this.ParagraphIndex = source.ParagraphIndex;
this.SentenceIndex = source.SentenceIndex;
this.DialogueIndc = source.DialogueIndc;
this.PersonName = source.PersonName;
this.Gender = source.Gender;
this.Sentiment = source.Sentiment;
this.Content = source.Content;
if (includeSystemFields)
{
this.CreatedDatetime = source.CreatedDatetime;
this.CreatedBy = source.CreatedBy;
this.UpdatedDatetime = source.UpdatedDatetime;
this.UpdatedBy = source.UpdatedBy;
}
}
public override string GetSelectFrom()
{
return $" select {DB_NAME_FIELDS} from { DB_NAME_AI_SENTENCE_DIALOGUE_VIEW} with(nolock) ";
}
public override string GetSelectCountFrom(string sqlWhere)
{
return $" select count(*) from { DB_NAME_AI_SENTENCE_DIALOGUE_VIEW} with(nolock) where 1=1 {sqlWhere}";
}
public override string GetSelectTopFrom(int limit, int start,string orderby,string orderbytag, string sqlWhere)
{
if (string.IsNullOrEmpty(orderbytag) || orderbytag.Trim() != "desc") orderbytag = "asc";
if (string.IsNullOrEmpty(orderby) || !DB_NAME_FIELDS.Contains(orderby)) orderby = DB_CREATED_DATETIME ; string top = limit > 0 ? $"top {limit}" : "";
return $" select {top} {DB_NAME_FIELDS} from (select {DB_NAME_FIELDS} ,row_number() over(order by {orderby} {orderbytag}) as num from { DB_NAME_AI_SENTENCE_DIALOGUE_VIEW} with(nolock) where 1=1 {sqlWhere} ) a where num> {limit * (start - 1)}";
}
public AiSentenceDialogueViewEntity Clone()
{
AiSentenceDialogueViewEntity result = new AiSentenceDialogueViewEntity();
result.CopyFrom(this);
return result;
}
}
}

12
CloudBuilder.AI/Entity/AiSentenceViewEntity.auto.cs
View File

@ -27,8 +27,12 @@ namespace CloudBuilder.AI.Entity
public string ParagraphGuid { get; set; }
[Column("status")]
public string? Status { get; set; }
[Column("book_guid")]
public string BookGuid { get; set; }
[Column("chapter_guid")]
public string ChapterGuid { get; set; }
[Column("chapter_index")]
public int ChapterIndex { get; set; }
[Column("paragraph_index")]
public int ParagraphIndex { get; set; }
[Column("sentence_index")]
@ -51,7 +55,9 @@ namespace CloudBuilder.AI.Entity
public const string GUID = "Guid";
public const string PARAGRAPH_GUID = "ParagraphGuid";
public const string STATUS = "Status";
public const string BOOK_GUID = "BookGuid";
public const string CHAPTER_GUID = "ChapterGuid";
public const string CHAPTER_INDEX = "ChapterIndex";
public const string PARAGRAPH_INDEX = "ParagraphIndex";
public const string SENTENCE_INDEX = "SentenceIndex";
public const string DIALOGUE_INDC = "DialogueIndc";
@ -61,7 +67,7 @@ namespace CloudBuilder.AI.Entity
public const string CONTENT = "Content";
public const string DB_NAME_AI_SENTENCE_VIEW = "ai_sentence_view";
public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,paragraph_guid,status,chapter_guid,paragraph_index,sentence_index,dialogue_indc,person_name,gender,sentiment,content";
public const string DB_NAME_FIELDS="created_datetime,created_by,updated_datetime,updated_by,guid,paragraph_guid,status,book_guid,chapter_guid,chapter_index,paragraph_index,sentence_index,dialogue_indc,person_name,gender,sentiment,content";
public const string DB_CREATED_DATETIME = "created_datetime";
public const string DB_CREATED_BY = "created_by";
@ -70,7 +76,9 @@ namespace CloudBuilder.AI.Entity
public const string DB_GUID = "guid";
public const string DB_PARAGRAPH_GUID = "paragraph_guid";
public const string DB_STATUS = "status";
public const string DB_BOOK_GUID = "book_guid";
public const string DB_CHAPTER_GUID = "chapter_guid";
public const string DB_CHAPTER_INDEX = "chapter_index";
public const string DB_PARAGRAPH_INDEX = "paragraph_index";
public const string DB_SENTENCE_INDEX = "sentence_index";
public const string DB_DIALOGUE_INDC = "dialogue_indc";
@ -104,7 +112,9 @@ namespace CloudBuilder.AI.Entity
this.Guid = source.Guid;
this.ParagraphGuid = source.ParagraphGuid;
this.Status = source.Status;
this.BookGuid = source.BookGuid;
this.ChapterGuid = source.ChapterGuid;
this.ChapterIndex = source.ChapterIndex;
this.ParagraphIndex = source.ParagraphIndex;
this.SentenceIndex = source.SentenceIndex;
this.DialogueIndc = source.DialogueIndc;

2
CloudBuilder.AI/Entity/CloudBuilderAIRegistEntity.auto.cs
View File

@ -41,6 +41,8 @@ namespace CloudBuilder.AI.Entity
builder.Entity<AiSentenceViewEntity>(eb => { eb.HasNoKey(); });
builder.Entity<AiSentenceViewEntity>(eb => { eb.HasNoKey(); });
builder.Entity<AiParagraphViewEntity>(eb => { eb.HasNoKey(); });
builder.Entity<AiSentenceDialogueViewEntity>(eb => { eb.HasNoKey(); });
builder.Entity<AiSentenceViewEntity>(eb => { eb.HasNoKey(); });
//***END***
}
}

2
CloudBuilder.AI/Policy/AIPrefixPolicy.cs
View File

@ -13,6 +13,8 @@ namespace CloudBuilder.AI.Policy
public const string AI_BOOK_PREFIX = "B";
public const string AI_BOOK_GUID_FORMAT = "000000";
public const string AI_CHAPTER = "AI_CHAPTER";
public const string AI_CHAPTER_GUID_FORMAT = "00000000";
public const string BOOK_DMS_FILE = "BOOK_DMS_FILE";
public const string BOOK_DMS_FILE_BOOK_NAME = "BOOK_NAME";