HuggingFace的AI混合工具如何使用？开发AI模型的实用操作教程

from transformers import pipeline

# 假设我们想先识别文本情绪，然后对积极情绪的文本进行摘要
# 这是一个概念上的“混合”，即一个模型的输出作为另一个模型的潜在输入或决策依据

# 情绪分析模型
sentiment_analyzer = pipeline("sentiment-analysis", model="distilbert-base-uncased-finetuned-sst-2-english")

# 文本摘要模型
summarizer = pipeline("summarization", model="facebook/bart-large-cnn")

text_input = "HuggingFace is truly amazing! I love how easy it makes working with large language models. This platform has revolutionized my workflow and made AI development accessible to so many."

# 步骤1: 情绪分析
sentiment_result = sentiment_analyzer(text_input)[0]
print(f"Sentiment: {sentiment_result['label']} (Score: {sentiment_result['score']:.2f})")

# 步骤2: 如果情绪是积极的，进行摘要
if sentiment_result['label'] == 'POSITIVE':
    summary_result = summarizer(text_input, max_length=50, min_length=10, do_sample=False)[0]
    print(f"Summary: {summary_result['summary_text']}")
else:
    print("Text sentiment is not positive, skipping summarization.")

# 这种通过Python逻辑将不同Pipeline连接起来，就是一种最直接的“混合”应用。

AutoModel

AutoTokenizer

AutoProcessor

from transformers import pipeline

# 步骤1: 命名实体识别 (NER)
ner_pipeline = pipeline("ner", model="dbmdz/bert-large-cased-finetuned-conll03-english")

# 步骤2: 问答生成 (Question Generation)
# 注意：HuggingFace官方没有直接的QG pipeline，这里我们用一个text2text-generation模型来模拟
# 实际应用中，你可能需要一个专门训练过的QG模型
qg_pipeline = pipeline("text2text-generation", model="mrm8488/t5-base-finetuned-question-generation-ap")

text = "HuggingFace is a company based in New York City that builds tools for machine learning. Its headquarters are located in Brooklyn."

# 执行NER
ner_results = ner_pipeline(text)
print("NER Results:")
for entity in ner_results:
    print(f"  Entity: {entity['word']}, Type: {entity['entity']}, Score: {entity['score']:.2f}")

# 提取PERSON和ORG类型的实体，作为潜在的问题生成焦点
relevant_entities = [entity['word'] for entity in ner_results if entity['entity'].startswith('B-ORG') or entity['entity'].startswith('B-LOC')]
relevant_entities = list(set(relevant_entities)) # 去重

print("\nGenerating questions for relevant entities:")
for entity_word in relevant_entities:
    # 构造QG模型的输入，通常是 'context: <text> question: <entity>'
    # 这里我们简化为 'generate question about <entity_word> from context: <text>'
    # 实际QG模型输入格式可能不同，需要查阅模型文档
    qg_input = f"generate question about {entity_word} from context: {text}"
    generated_question = qg_pipeline(qg_input, max_length=60, num_return_sequences=1)[0]['generated_text']
    print(f"  Entity '{entity_word}': {generated_question}")

# 这种手动的串联，虽然不如一个端到端Pipeline自动化，但提供了极大的灵活性。
# 这也是我们日常开发中最常见的“混合”方式之一。

Trainer

Trainer

from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
from datasets import load_dataset
import numpy as np
import evaluate

# 步骤1: 加载数据集（这里使用一个示例数据集，实际应用中会是你的自定义数据集）
# 假设我们有一个二分类任务的数据集
dataset = load_dataset("imdb")

# 步骤2: 加载预训练模型和分词器
model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=2)

# 步骤3: 数据预处理函数
def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)

tokenized_datasets = dataset.map(tokenize_function, batched=True)

# 训练集和验证集
train_dataset = tokenized_datasets["train"].shuffle(seed=42).select(range(2000)) # 示例，取2000条
eval_dataset = tokenized_datasets["test"].shuffle(seed=42).select(range(500))   # 示例，取500条

# 步骤4: 定义评估指标
metric = evaluate.load("accuracy")

def compute_metrics(eval_pred):
    logits, labels = eval_pred
    predictions = np.argmax(logits, axis=-1)
    return metric.compute(predictions=predictions, references=labels)

# 步骤5: 定义训练参数
training_args = TrainingArguments(
    output_dir="./results",
    learning_rate=2e-5,
    per_device_train_batch_size=8,
    per_device_eval_batch_size=8,
    num_train_epochs=3,
    weight_decay=0.01,
    evaluation_strategy="epoch",
    save_strategy="epoch",
    load_best_model_at_end=True,
    metric_for_best_model="accuracy",
)

# 步骤6: 初始化Trainer并开始训练
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset,
    eval_dataset=eval_dataset,
    tokenizer=tokenizer,
    compute_metrics=compute_metrics,
)

trainer.train()

# 微调后的模型就可以保存并部署，成为你“混合”系统中的一个定制化组件。
# trainer.save_model("./my_finetuned_sentiment_model")