甘肃省网站建设_网站建设公司_Ruby_seo优化

一、加载tokenizer

importtorchfromtransformersimportAutoTokenizer#加载tokenizertokenizer=AutoTokenizer.from_pretrained('google-bert/bert-base-chinese')tokenizer

二、加载数据集和编码

fromdatasetsimportload_dataset#加载数据集dataset=load_dataset(path='lansinuote/ChnSentiCorp')#编码f=lambdax:tokenizer(x['text'],truncation=True,max_length=500)dataset=dataset.map(f,remove_columns=['text'])#设置数据类型dataset.set_format('pt')dataset,dataset['train'][0]

三、加载模型

#定义模型fromtransformersimportBertConfig,BertForSequenceClassification#在线加载一个语句分类模型model=BertForSequenceClassification.from_pretrained('google-bert/bert-base-chinese',num_labels=2)model.config

四、执行训练

fromtransformersimportTrainingArguments,Trainer,DataCollatorWithPadding#配置训练参数args=TrainingArguments(output_dir='output_dir',use_cpu=True,num_train_epochs=1,max_steps=300,eval_strategy='no',per_device_train_batch_size=8)#创建trainertrainer=Trainer(model=model,args=args,train_dataset=dataset['train'],data_collator=DataCollatorWithPadding(tokenizer))#执行训练trainer.train()

五、执行测试

#执行测试deftest():loader_test=torch.utils.data.DataLoader(dataset['test'],batch_size=8,shuffle=True,drop_last=True,collate_fn=DataCollatorWithPadding(tokenizer))correct=0total=0fori,datainenumerate(loader_test):withtorch.no_grad():out=model(**data).logits out=out.argmax(dim=1)correct+=(out==data.labels).sum().item()total+=len(data.labels)print(i,len(loader_test),correct/total)ifi==5:breakreturncorrect/total test()