using CloudBuilder.Core.DatabaseAccessor.Entity;
using CloudBuilder.Core.Service;
using Microsoft.ML;
using Microsoft.ML.Data;
using Microsoft.ML.Transforms.Onnx;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
namespace CloudBuilder.AI.Service.Utility
{
///
/// 基于 ONNX 模型的命名实体识别器,支持提取人名及人物角色(人名+职位)。
///
public class ChineseNameExtractor
{
private BertChineseTokenizer tokenizer;
private OnnxScoringEstimator pipeline;
private PredictionEngine engine;
///
/// 开启后会将每个 token 的预测标签及置信度输出到控制台,方便调试。
///
public bool DebugMode { get; set; } = false;
public ChineseNameExtractor(IApplicationService service)
{
InitMLContext(service);
}
private void InitMLContext(IApplicationService service)
{
var mlContext = new MLContext();
string modelDir = service.Configuration["FileServiceSettings:OnnxDirectory"];
string modelPath = Path.Combine(modelDir, "model.onnx");
string vocabPath = Path.Combine(modelDir, "vocab.txt");
if (!File.Exists(modelPath) || !File.Exists(vocabPath))
throw new Exception("请确保 model.onnx 和 vocab.txt 在指定目录下。");
tokenizer = new BertChineseTokenizer(vocabPath);
var shapeDictionary = new Dictionary
{
{ "input_ids", new[] { 1, 128 } },
{ "attention_mask", new[] { 1, 128 } },
{ "token_type_ids", new[] { 1, 128 } },
{ "logits", new[] { 1, 128, tokenizer.LabelCount } }
};
var emptyData = mlContext.Data.LoadFromEnumerable(new List());
pipeline = mlContext.Transforms.ApplyOnnxModel(
modelFile: modelPath,
inputColumnNames: new[] { "input_ids", "attention_mask", "token_type_ids" },
outputColumnNames: new[] { "logits" },
shapeDictionary: shapeDictionary,
gpuDeviceId: null,
fallbackToCpu: true
);
var transformer = pipeline.Fit(emptyData);
engine = mlContext.Model.CreatePredictionEngine(transformer);
}
///
/// 提取纯人名(标签为 B-PER / I-PER 的连续实体)。
///
public List ExtractNames(string text)
{
return ExtractPersonEntities(text, onlyPureName: true);
}
///
/// 提取人物角色(人名 + 紧接其后的职位,例如“岳堂主”、“张经理”等)。
///
public List ExtractPersonRoles(string text)
{
return ExtractPersonEntities(text, onlyPureName: false);
}
///
/// 核心实体提取方法,支持合并人名与其后的职位标签。
///
/// 输入文本
/// true 只提取纯人名;false 提取人名+职位复合实体
private List ExtractPersonEntities(string text, bool onlyPureName)
{
var tokenized = tokenizer.Tokenize(text);
var input = new NerInput
{
InputIds = tokenized.InputIds,
AttentionMask = tokenized.AttentionMask,
TokenTypeIds = tokenized.TokenTypeIds
};
var prediction = engine.Predict(input);
var idToLabel = tokenizer.IdToLabel;
int seqLen = 128;
int numLabels = idToLabel.Length;
var entities = new List();
string currentEntity = "";
bool insideEntity = false;
bool isPersonEntity = false; // 当前实体是否以 B-PER 开始
for (int i = 0; i < seqLen; i++)
{
if (tokenized.AttentionMask[i] == 0) continue; // 跳过 padding
// 找到当前 token 的预测标签
int startIdx = i * numLabels;
float maxVal = float.MinValue;
int maxIdx = 0;
for (int j = 0; j < numLabels; j++)
{
float val = prediction.Logits[startIdx + j];
if (val > maxVal)
{
maxVal = val;
maxIdx = j;
}
}
string label = idToLabel[maxIdx];
string token = tokenized.Tokens[i];
if (DebugMode)
Console.WriteLine($"{token} -> {label} (置信度: {maxVal:F3})");
// 判断当前标签类型
bool isPerStart = label == "B-PER";
bool isPerInside = label == "I-PER";
bool isPositionStart = label == "B-POSITION";
bool isPositionInside = label == "I-POSITION";
// ---- 仅提取纯人名的逻辑 ----
if (onlyPureName)
{
if (isPerStart)
{
if (insideEntity && currentEntity.Length > 0) entities.Add(currentEntity);
currentEntity = token;
insideEntity = true;
isPersonEntity = true;
}
else if (isPerInside && insideEntity && isPersonEntity)
{
currentEntity += token;
}
else
{
if (insideEntity && currentEntity.Length > 0)
{
entities.Add(currentEntity);
currentEntity = "";
insideEntity = false;
isPersonEntity = false;
}
}
continue;
}
// ---- 提取人物角色(人名+职位)的逻辑 ----
if (isPerStart)
{
// 无论之前是否有未结束的实体,都结束之前的,并开始一个新的人名
if (insideEntity && currentEntity.Length > 0)
entities.Add(currentEntity);
currentEntity = token;
insideEntity = true;
isPersonEntity = true;
}
else if (isPositionStart)
{
if (insideEntity && isPersonEntity)
{
// 当前正在收集人名,且遇到 B-POSITION,则合并职位
currentEntity += token;
// 保持 isPersonEntity = true 不变,这样后续的 I-POSITION 也能继续合并
}
else
{
// 不在人名内部,则视作新实体(提取单独的职位,如“堂主”)
if (insideEntity && currentEntity.Length > 0)
entities.Add(currentEntity);
currentEntity = token;
insideEntity = true;
isPersonEntity = false; // 标记为纯职位实体
}
}
else if ((isPerInside || isPositionInside) && insideEntity)
{
// 无论是 I-PER 还是 I-POSITION,只要在实体内部都追加
currentEntity += token;
}
else
{
// 遇到其他标签(O 或其他 B- 开头),结束当前实体
if (insideEntity && currentEntity.Length > 0)
{
entities.Add(currentEntity);
currentEntity = "";
insideEntity = false;
isPersonEntity = false;
}
// 注意:不处理非目标标签本身,因为它们不属于角色
}
}
// 收尾最后一个实体
if (insideEntity && currentEntity.Length > 0)
entities.Add(currentEntity);
return entities;
}
}
// ==================== 数据模型 ====================
public class NerInput
{
[VectorType(1, 128)]
[ColumnName("input_ids")]
public long[] InputIds { get; set; }
[VectorType(1, 128)]
[ColumnName("attention_mask")]
public long[] AttentionMask { get; set; }
[VectorType(1, 128)]
[ColumnName("token_type_ids")]
public long[] TokenTypeIds { get; set; }
}
public class NerOutput
{
[VectorType(1, 128, 29)] // 维度需与模型输出一致
[ColumnName("logits")]
public float[] Logits { get; set; }
}
// ==================== 分词器 ====================
public class BertChineseTokenizer
{
private readonly Dictionary _tokenToId;
///
/// 标签映射,务必与模型训练时的标签顺序一致。
///
public readonly string[] IdToLabel;
public int LabelCount => IdToLabel.Length;
public BertChineseTokenizer(string vocabPath, int maxSeqLength = 128)
{
_tokenToId = new Dictionary();
var lines = File.ReadAllLines(vocabPath);
for (int i = 0; i < lines.Length; i++)
_tokenToId[lines[i]] = i;
// 请根据实际模型输出维度调整此数组!
// 下面的顺序仅为示例,必须与模型 logits 第2维一一对应。
IdToLabel = new string[] {
"O", // 0
"B-ORG", // 1
"I-ORG", // 2
"B-PER", // 3
"I-PER", // 4
"B-TIME", // 5
"I-TIME", // 6
"B-LOC", // 7
"I-LOC", // 8
"B-POSITION", // 9
"I-POSITION", // 10
"B-COMPANY", // 11
"I-COMPANY", // 12
"B-GAME", // 13
"I-GAME", // 14
"B-GOVERNMENT", // 15
"I-GOVERNMENT", // 16
"B-SCENE", // 17
"I-SCENE", // 18
"B-SUBJECT", // 19
"I-SUBJECT", // 20
"B-CREATION", // 21
"I-CREATION", // 22
"B-FOOD", // 23
"I-FOOD", // 24
"B-MOVIE", // 25
"I-MOVIE", // 26
"[CLS]", // 27 (如果模型没有这些特殊标签,请删除)
"[SEP]" // 28
};
}
public TokenizedResult Tokenize(string text)
{
const int maxLen = 128;
var inputIds = new long[maxLen];
var attentionMask = new long[maxLen];
var tokenTypeIds = new long[maxLen];
var tokens = new string[maxLen];
var chars = text.ToCharArray();
int charIndex = 0;
for (int i = 0; i < maxLen; i++)
{
if (i == 0)
{
inputIds[i] = 101;
tokens[i] = "[CLS]";
}
else if (i == chars.Length + 1)
{
inputIds[i] = 102;
tokens[i] = "[SEP]";
}
else if (i > chars.Length + 1)
{
inputIds[i] = 0;
tokens[i] = "[PAD]";
}
else
{
string c = chars[charIndex].ToString();
charIndex++;
if (_tokenToId.TryGetValue(c, out int id))
{
inputIds[i] = id;
tokens[i] = c;
}
else
{
inputIds[i] = _tokenToId["[UNK]"];
tokens[i] = "[UNK]";
}
}
attentionMask[i] = (i <= chars.Length + 1) ? 1 : 0;
tokenTypeIds[i] = 0;
}
return new TokenizedResult
{
InputIds = inputIds,
AttentionMask = attentionMask,
TokenTypeIds = tokenTypeIds,
Tokens = tokens
};
}
}
public class TokenizedResult
{
public long[] InputIds { get; set; }
public long[] AttentionMask { get; set; }
public long[] TokenTypeIds { get; set; }
public string[] Tokens { get; set; }
}
}