ceshi_org
/
statistical-learning-method-solutions-manual
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

✔ 完善习题27.1

This commit is contained in:
Relph1119 2023-04-19 14:23:53 +08:00
parent 10491ac3f7
commit ca5613f819
5 changed files with 763 additions and 400 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -35,7 +35,7 @@
4. 安装PyTorch
访问[PyTorch官网](https://pytorch.org/get-started/locally/)，选择合适的版本安装PyTorch，有条件的小伙伴可以下载GPU版本
```shell
pip3 install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cu117
pip3 install torch==1.12.1+cu116 torchvision==0.13.1+cu116 torchaudio==0.12.1+cu116 -f https://download.pytorch.org/whl/torch_stable.html
```
5. docsify框架运行
@ -162,9 +162,10 @@ requirements.txt-----------------------------------运行环境依赖包
- [王昊文](https://github.com/whw199833) （帝国理工学院-算法工程师）
**其他**
1. 特别感谢 [@Sm1les](https://github.com/Sm1les)、[@LSGOMYP](https://github.com/LSGOMYP) 对本项目的帮助与支持
2. 感谢[@GYHHAHA](https://github.com/GYHHAHA)，指出了第7章习题7.4的解答问题，并完善了该题的解答
3. 感谢范佳慧、汪健麟、张宇明、范致远、兰坤、李拙等同学对项目提供的完善建议
1. 特别感谢 [@Sm1les](https://github.com/Sm1les)、[@LSGOMYP](https://github.com/LSGOMYP) 对本项目的帮助与支持；
2. 感谢[@GYHHAHA](https://github.com/GYHHAHA)，指出了第7章习题7.4的解答问题，并完善了该题的解答；
3. 感谢范佳慧、汪健麟、张宇明、兰坤、李拙等同学对项目提供的完善建议；
4. 感觉张帆同学对习题27.1解答的帮助，解决了ELMo预训练模型的代码问题。
## 参考文献
1. [李航《统计学习方法笔记》中的代码、notebook、参考文献、Errata](https://github.com/SmirkCao/Lihang)

183
codes/ch27/bi-lstm-text-classification.py
View File

@ -5,158 +5,143 @@
@file: bi-lstm-text-classification.py
@time: 2023/3/15 14:30
@project: statistical-learning-method-solutions-manual
@desc: 习题27.1 基于双向LSTM的预训练语言模型，假设下游任务是文本分类
@desc: 习题27.1 基于双向LSTM的ELMo预训练语言模型，假设下游任务是文本分类
"""
import os
import time
import torch
import torch.nn as nn
import wget
from allennlp.modules.elmo import Elmo
from allennlp.modules.elmo import batch_to_ids
from torch.utils.data import DataLoader
from torch.utils.data.dataset import random_split
from torchtext.data.functional import to_map_style_dataset
from torchtext.data.utils import get_tokenizer
from torchtext.datasets import AG_NEWS
from torchtext.vocab import build_vocab_from_iterator
def get_elmo_model():
elmo_options_file = './data/elmo_2x1024_128_2048cnn_1xhighway_options.json'
elmo_weight_file = './data/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5'
url = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_options.json"
if (not os.path.exists(elmo_options_file)):
wget.download(url, elmo_options_file)
url = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x1024_128_2048cnn_1xhighway/elmo_2x1024_128_2048cnn_1xhighway_weights.hdf5"
if (not os.path.exists(elmo_weight_file)):
wget.download(url, elmo_weight_file)
elmo = Elmo(elmo_options_file, elmo_weight_file, 1)
return elmo
# 加载ELMo模型
elmo = get_elmo_model()
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# 加载AG_NEWS数据集
train_iter, test_iter = AG_NEWS(root='./data')
# 定义tokenizer
tokenizer = get_tokenizer('basic_english')
# 定义数据处理函数
def yield_tokens(data_iter):
for _, text in data_iter:
yield tokenizer(text)
# 构建词汇表
vocab = build_vocab_from_iterator(yield_tokens(train_iter), specials=["<unk>"])
vocab.set_default_index(vocab["<unk>"])
# 将数据集映射到MapStyleDataset格式
train_dataset = to_map_style_dataset(train_iter)
test_dataset = to_map_style_dataset(test_iter)
# 划分验证集
num_train = int(len(train_dataset) * 0.95)
split_train_, split_valid_ = random_split(train_dataset, [num_train, len(train_dataset) - num_train])
# 设置文本和标签的处理函数
text_pipeline = lambda x: vocab(tokenizer(x))
label_pipeline = lambda x: int(x) - 1
def collate_batch(batch):
"""
对数据集进行数据处理
"""
label_list, text_list, offsets = [], [], [0]
label_list, text_list = [], []
for (_label, _text) in batch:
label_list.append(label_pipeline(_label))
processed_text = torch.tensor(text_pipeline(_text), dtype=torch.int64)
text_list.append(processed_text)
offsets.append(processed_text.size(0))
text_list.append(_text.split())
label_list = torch.tensor(label_list, dtype=torch.int64)
offsets = torch.tensor(offsets[:-1]).cumsum(dim=0)
text_list = torch.cat(text_list)
return label_list.to(device), text_list.to(device), offsets.to(device)
return label_list.to(device), text_list
# 构建数据集的数据加载器
BATCH_SIZE = 256
# 加载AG_NEWS数据集
train_iter, test_iter = AG_NEWS(root='./data')
train_dataset = to_map_style_dataset(train_iter)
test_dataset = to_map_style_dataset(test_iter)
num_train = int(len(train_dataset) * 0.95)
split_train_, split_valid_ = \
random_split(train_dataset, [num_train, len(train_dataset) - num_train])
BATCH_SIZE = 128
train_dataloader = DataLoader(split_train_, batch_size=BATCH_SIZE,
shuffle=True, collate_fn=collate_batch)
valid_dataloader = DataLoader(split_valid_, batch_size=BATCH_SIZE,
shuffle=True, collate_fn=collate_batch)
shuffle=False, collate_fn=collate_batch)
test_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE,
shuffle=True, collate_fn=collate_batch)
shuffle=False, collate_fn=collate_batch)
class TextClassifier(nn.Module):
"""
基于双向LSTM的文本分类模型
"""
def __init__(self, vocab_size, embedding_dim, hidden_dim, num_classes):
def __init__(self, embedding_dim, hidden_dim, num_classes):
super().__init__()
self.embedding = nn.EmbeddingBag(vocab_size, embedding_dim, sparse=False)
# 使用预训练的ELMO
self.elmo = elmo
# 使用双向LSTM
self.lstm = nn.LSTM(embedding_dim, hidden_dim, bidirectional=True, batch_first=True)
# 使用线性函数进行文本分类任务
self.fc = nn.Linear(hidden_dim * 2, num_classes)
self.dropout = nn.Dropout(0.5)
self.init_weights()
def init_weights(self):
initrange = 0.5
self.embedding.weight.data.uniform_(-initrange, initrange)
initrange = 0.1
self.fc.weight.data.uniform_(-initrange, initrange)
self.fc.bias.data.zero_()
self.fc.bias.data.uniform_(-initrange, initrange)
def load_elmo_weights(self, elmo):
self.embedding.weight.data.copy_(elmo.embedding.weight.data)
self.embedding.weight.requires_grad = False
self.lstm.weight_ih_l0.data.copy_(elmo.lstm.weight_ih_l0.data)
self.lstm.weight_hh_l0.data.copy_(elmo.lstm.weight_hh_l0.data)
self.lstm.bias_ih_l0.data.copy_(elmo.lstm.bias_ih_l0.data)
self.lstm.bias_hh_l0.data.copy_(elmo.lstm.bias_hh_l0.data)
self.lstm.weight_ih_l0_reverse.data.copy_(elmo.lstm.weight_ih_l0_reverse.data)
self.lstm.weight_hh_l0_reverse.data.copy_(elmo.lstm.weight_hh_l0_reverse.data)
self.lstm.bias_ih_l0_reverse.data.copy_(elmo.lstm.bias_ih_l0_reverse.data)
self.lstm.bias_hh_l0_reverse.data.copy_(elmo.lstm.bias_hh_l0_reverse.data)
self.fc.weight.data.copy_(elmo.fc.weight.data)
self.fc.bias.data.copy_(elmo.fc.bias.data)
def forward(self, sentence_lists):
character_ids = batch_to_ids(sentence_lists)
character_ids = character_ids.to(device)
embeddings = self.elmo(character_ids)
embedded = embeddings['elmo_representations'][0]
def forward(self, text, offsets):
embedded = self.embedding(text, offsets)
x, _ = self.lstm(embedded)
x = x.mean(1)
x = self.dropout(x)
x = self.fc(x)
return x
# 设置超参数
EMBED_DIM = 64
EMBED_DIM = 256
HIDDEN_DIM = 64
NUM_CLASSES = 4
LEARNING_RATE = 1e-2
NUM_EPOCHS = 10
NUM_EPOCHS = 1
# 创建模型、优化器和损失函数
model = TextClassifier(len(vocab), EMBED_DIM, HIDDEN_DIM, NUM_CLASSES).to(device)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
model = TextClassifier(EMBED_DIM, HIDDEN_DIM, NUM_CLASSES).to(device)
def train(dataloader):
"""
模型训练
"""
model.train()
for idx, (label, text, offsets) in enumerate(dataloader):
for idx, (label, text) in enumerate(dataloader):
optimizer.zero_grad()
predicted_label = model(text, offsets)
predicted_label = model(text)
loss = criterion(predicted_label, label)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
optimizer.step()
def evaluate(dataloader):
"""
模型验证
"""
model.eval()
total_acc, total_count = 0, 0
with torch.no_grad():
for idx, (label, text, offsets) in enumerate(dataloader):
predicted_label = model(text, offsets)
criterion(predicted_label, label)
for idx, (label, text) in enumerate(dataloader):
predicted_label = model(text)
total_acc += (predicted_label.argmax(1) == label).sum().item()
total_count += label.size(0)
return total_acc / total_count
# 使用交叉熵损失函数
criterion = nn.CrossEntropyLoss().to(device)
# 设置优化器
optimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE)
for epoch in range(1, NUM_EPOCHS + 1):
epoch_start_time = time.time()
train(train_dataloader)
@ -168,26 +153,24 @@ for epoch in range(1, NUM_EPOCHS + 1):
accu_val * 100))
print('-' * 59)
# 新闻的分类标签
ag_news_label = {1: "World",
2: "Sports",
3: "Business",
4: "Sci/Tec"}
ag_news_label = {1: "World", 2: "Sports", 3: "Business", 4: "Sci/Tec"}
def predict(text, text_pipeline):
def predict(text):
with torch.no_grad():
text = torch.tensor(text_pipeline(text))
output = model(text, torch.tensor([0]))
output = model([text])
return output.argmax(1).item() + 1
# 预测一个文本的类别
ex_text_str = """
Our younger Fox Cubs (Y2-Y4) also had a great second experience of swimming competition in February when they travelled
over to NIS at the end of February to compete in the SSL Development Series R2 event. For students aged 9 and under
these SSL Development Series events are a great introduction to competitive swimming, focussed on fun and participation
whilst also building basic skills and confidence as students build up to joining the full SSL team in Year 5 and beyond.
Our younger Fox Cubs (Y2-Y4) also had a great second experience
of swimming competition in February when they travelled over to
NIS at the end of February to compete in the SSL Development
Series R2 event. For students aged 9 and under these SSL
Development Series events are a great introduction to
competitive swimming, focussed on fun and participation whilst
also building basic skills and confidence as students build up
to joining the full SSL team in Year 5 and beyond.
"""
model = model.to("cpu")
print("This is a %s news" % ag_news_label[predict(ex_text_str, text_pipeline)])
print("This is a %s news" % ag_news_label[predict(ex_text_str)])

7
docs/README.md
View File

@ -118,9 +118,10 @@ requirements.txt-----------------------------------运行环境依赖包
- [王昊文](https://github.com/whw199833) （帝国理工学院-算法工程师）
**其他**
1. 特别感谢 [@Sm1les](https://github.com/Sm1les)、[@LSGOMYP](https://github.com/LSGOMYP) 对本项目的帮助与支持
2. 感谢[@GYHHAHA](https://github.com/GYHHAHA)，指出了第7章习题7.4的解答问题，并完善了该题的解答
3. 感谢范佳慧、汪健麟、张宇明、范致远、兰坤、李拙等同学对项目提供的完善建议
1. 特别感谢 [@Sm1les](https://github.com/Sm1les)、[@LSGOMYP](https://github.com/LSGOMYP) 对本项目的帮助与支持；
2. 感谢[@GYHHAHA](https://github.com/GYHHAHA)，指出了第7章习题7.4的解答问题，并完善了该题的解答；
3. 感谢范佳慧、汪健麟、张宇明、兰坤、李拙等同学对项目提供的完善建议；
4. 感觉张帆同学对习题27.1解答的帮助，解决了ELMo预训练模型的代码问题。
## 参考文献
1. [李航《统计学习方法笔记》中的代码、notebook、参考文献、Errata](https://github.com/SmirkCao/Lihang)

886
notebook/part03/notes/ch27.ipynb
View File

File diff suppressed because one or more lines are too long

78
requirements.txt
View File

@ -1,3 +1,4 @@
allennlp==2.10.1
anyio==3.6.2
argon2-cffi==21.3.0
argon2-cffi-bindings==21.2.0
@ -6,24 +7,51 @@ asttokens==2.2.1
attrs==22.2.0
autopep8==2.0.1
backcall==0.2.0
base58==2.1.1
beautifulsoup4==4.11.2
bleach==6.0.0
blis==0.7.9
boto3==1.26.114
botocore==1.29.114
cached-path==1.1.6
cachetools==5.3.0
catalogue==2.0.8
certifi==2022.12.7
cffi==1.15.1
charset-normalizer==3.0.1
click==8.1.3
colorama==0.4.6
comm==0.1.2
commonmark==0.9.1
contourpy==1.0.7
cycler==0.11.0
cymem==2.0.7
debugpy==1.6.6
decorator==5.1.1
defusedxml==0.7.1
dill==0.3.6
docker-pycreds==0.4.0
exceptiongroup==1.1.1
executing==1.2.0
fairscale==0.4.6
fastjsonschema==2.16.3
filelock==3.7.1
fonttools==4.38.0
fqdn==1.5.1
gitdb==4.0.10
GitPython==3.1.31
google-api-core==2.11.0
google-auth==2.17.3
google-cloud-core==2.3.2
google-cloud-storage==2.8.0
google-crc32c==1.5.0
google-resumable-media==2.4.1
googleapis-common-protos==1.59.0
graphviz==0.20.1
h5py==3.8.0
huggingface-hub==0.10.1
idna==3.4
iniconfig==2.0.0
ipykernel==6.21.2
ipython==8.11.0
ipython-genutils==0.2.0
@ -31,6 +59,7 @@ ipywidgets==8.0.4
isoduration==20.11.0
jedi==0.18.2
Jinja2==3.1.2
jmespath==1.0.1
joblib==1.2.0
jsonpointer==2.3
jsonschema==4.17.3
@ -44,15 +73,20 @@ jupyter_server_terminals==0.4.4
jupyterlab-pygments==0.2.2
jupyterlab-widgets==3.0.5
kiwisolver==1.4.4
langcodes==3.3.0
lmdb==1.4.0
MarkupSafe==2.1.2
matplotlib==3.7.0
matplotlib-inline==0.1.6
mistune==2.0.5
more-itertools==9.1.0
murmurhash==1.0.9
nbclassic==0.5.2
nbclient==0.7.2
nbconvert==7.2.9
nbformat==5.7.3
nest-asyncio==1.5.6
nltk==3.8.1
notebook==6.5.3
notebook_shim==0.2.2
numpy==1.24.2
@ -60,19 +94,29 @@ packaging==23.0
pandas==1.5.3
pandocfilters==1.5.0
parso==0.8.3
pathtools==0.1.2
pathy==0.10.1
pickleshare==0.7.5
Pillow==9.4.0
platformdirs==3.0.0
pluggy==1.0.0
portalocker==2.7.0
preshed==3.0.8
prometheus-client==0.16.0
promise==2.3
prompt-toolkit==3.0.38
protobuf==3.20.3
psutil==5.9.4
pure-eval==0.2.2
pyasn1==0.4.8
pyasn1-modules==0.2.8
pycodestyle==2.10.0
pycparser==2.21
pydantic==1.8.2
Pygments==2.14.0
pyparsing==3.0.9
pyrsistent==0.19.3
pytest==7.2.2
python-dateutil==2.8.2
python-json-logger==2.0.7
pytz==2022.7.1
@ -80,34 +124,58 @@ pywin32==305
pywinpty==2.0.10
PyYAML==6.0
pyzmq==25.0.0
regex==2023.3.23
requests==2.28.2
rfc3339-validator==0.1.4
rfc3986-validator==0.1.1
rich==12.6.0
rsa==4.9
s3transfer==0.6.0
sacremoses==0.0.53
scikit-learn==1.2.1
scipy==1.10.1
Send2Trash==1.8.0
sentencepiece==0.1.97
sentry-sdk==1.19.1
setproctitle==1.3.2
shortuuid==1.0.11
six==1.16.0
smart-open==6.3.0
smmap==5.0.0
sniffio==1.3.0
soupsieve==2.4
spacy==3.3.2
spacy-legacy==3.0.12
spacy-loggers==1.0.4
srsly==2.4.6
stack-data==0.6.2
tensorboardX==2.6
termcolor==1.1.0
terminado==0.17.1
thinc==8.0.17
threadpoolctl==3.1.0
tinycss2==1.2.1
tokenizers==0.12.1
tomli==2.0.1
torch==1.13.1+cu117
torchaudio==0.13.1+cu117
torchdata==0.5.1
torchtext==0.14.1
torchvision==0.14.1+cu117
torch==1.12.1+cu116
torchaudio==0.12.1+cu116
torchdata==0.4.1
torchtext==0.13.1
torchvision==0.13.1+cu116
torchviz==0.0.2
tornado==6.2
tqdm==4.65.0
traitlets==5.9.0
transformers==4.20.1
typer==0.4.2
typing_extensions==4.5.0
uri-template==1.2.0
urllib3==1.26.14
wandb==0.12.21
wasabi==0.10.1
wcwidth==0.2.6
webcolors==1.12
webencodings==0.5.1
websocket-client==1.5.1
wget==3.2
widgetsnbextension==4.0.5