takaito

BERTを用いたbaseline．LB: 0.954

はじめに

.ipynbを共有するのは初めてなので，読みにくいのはご了承ください．

初日から，LBのF1スコア高くてびっくりしています．

スコアが高いbaselineモデルのトピックがすでに上がっていたので，投稿するか迷いましたがせっかくなので共有させてもらえたらと思います．

以前のコンペで，bertを使ったのでその時に使用したシンプルなモデルのコードをほとんどそのまま流用しています．

詳しい解説は【PyTorch】BERTのfine-tuningを試してみた(SIGNATE Student Cup 2020) に記載しているので，もしこのモデルを使ってみたい場合は合わせて読んでもらえたらと思います．

全然工夫をしていないので，まだまだスコアをあげることができると思うので，何かの参考になれば幸いです．

不均衡データ対策

ラベルの割合で重み付け．

学習に時間がかかる

学習データの通常メールとスパムメールの割合を10:1になるようにサンプリング → './data/bert_train_data.csv' （データ数が不均衡なので，とりあえずはデータ数を少なくして学習させてみた）

'test_data.csv'に'y'のコラム(0埋め)を追加したデータ → './data/bert_test_data.csv'

import sys
print(sys.version)
!pip freeze

3.7.7 (default, May  7 2020, 12:27:09) 
[Clang 11.0.3 (clang-1103.0.32.59)]
amply==0.1.2
appnope==0.1.0
attrdict==2.0.1
attrs==19.2.0
backcall==0.1.0
beautifulsoup4==4.8.2
bleach==3.1.0
boto==2.49.0
boto3==1.11.9
botocore==1.14.9
bs4==0.0.1
catboost==0.24.1
certifi==2019.9.11
chardet==3.0.4
click==7.1.2
cycler==0.10.0
Cython==0.29.15
decorator==4.4.0
defusedxml==0.6.0
docutils==0.15.2
dyNET==2.1
emoji==0.5.4
entrypoints==0.3
filelock==3.0.12
future==0.18.2
gensim==3.8.1
get==2019.4.13
graphviz==0.14.1
idna==2.8
ipykernel==5.1.2
ipython==7.8.0
ipython-genutils==0.2.0
ipywidgets==7.5.1
jedi==0.15.1
Jinja2==2.10.1
jmespath==0.9.4
joblib==0.14.0
jsonschema==3.0.2
jupyter==1.0.0
jupyter-client==5.3.3
jupyter-console==6.0.0
jupyter-core==4.5.0
kiwisolver==1.1.0
lightgbm==2.3.1
MarkupSafe==1.1.1
matplotlib==3.2.0
mecab==0.996
mistune==0.8.4
mojimoji==0.0.9
nagisa==0.2.5
nbconvert==5.6.0
nbformat==4.4.0
notebook==6.0.1
numpy==1.17.2
opencv-python==4.2.0.34
packaging==20.4
pandas==1.1.1
pandocfilters==1.4.2
parso==0.5.1
pexpect==4.7.0
pickleshare==0.7.5
Pillow==6.2.0
plotly==4.10.0
post==2019.4.13
prometheus-client==0.7.1
prompt-toolkit==2.0.9
ptyprocess==0.6.0
public==2019.4.13
PuLP==2.3
Pygments==2.4.2
pyparsing==2.4.2
pyrsistent==0.15.4
python-dateutil==2.8.0
pytz==2019.3
pyzmq==18.1.0
qtconsole==4.5.5
query-string==2019.4.13
regex==2020.10.15
request==2019.4.13
requests==2.22.0
retrying==1.3.3
s3transfer==0.3.2
sacremoses==0.0.43
scikit-learn==0.21.3
scipy==1.3.1
seaborn==0.10.0
Send2Trash==1.5.0
sentencepiece==0.1.91
six==1.12.0
smart-open==1.9.0
soupsieve==2.0
terminado==0.8.2
testpath==0.4.2
tokenizers==0.8.1rc2
torch==1.6.0
torchtext==0.7.0
torchvision==0.6.0
tornado==6.0.3
tqdm==4.48.2
traitlets==4.3.2
transformers==3.3.1
urllib3==1.25.6
wcwidth==0.1.7
webencodings==0.5.1
widgetsnbextension==3.5.1
wordcloud==1.7.0

import random
import time
import numpy as np
import math
from tqdm import tqdm
import torch 
from torch import nn
import torch.optim as optim
import torchtext

from sklearn.metrics import f1_score

import pandas as pd
# 前処理と単語分割をまとめた関数を作成
import re
import string
from utils.bert import BertTokenizer
# フォルダ「utils」のbert.pyより

# 乱数のシードを設定
seed = 42
torch.manual_seed(seed)
if torch.cuda.is_available():
    torch.cuda.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
np.random.seed(seed)
random.seed(seed)

def preprocessing_text(text):
    '''前処理'''
    # 改行コードを消去
    text = re.sub('<br />', '', text)

    # カンマ、ピリオド以外の記号をスペースに置換
    for p in string.punctuation:
        if (p == ".") or (p == ","):
            continue
        else:
            text = text.replace(p, " ")

    # ピリオドなどの前後にはスペースを入れておく
    text = text.replace(".", " . ")
    text = text.replace(",", " , ")
    return text


# 単語分割用のTokenizerを用意
tokenizer_bert = BertTokenizer(
    vocab_file="./vocab/bert-base-uncased-vocab.txt", do_lower_case=True)


# 前処理と単語分割をまとめた関数を定義
# 単語分割の関数を渡すので、tokenizer_bertではなく、tokenizer_bert.tokenizeを渡す点に注意
def tokenizer_with_preprocessing(text, tokenizer=tokenizer_bert.tokenize):
    text = preprocessing_text(text)
    ret = tokenizer(text)  # tokenizer_bert
    return ret

# データを読み込んだときに、読み込んだ内容に対して行う処理を定義します
max_length = 256

TEXT = torchtext.data.Field(sequential=True, tokenize=tokenizer_with_preprocessing, use_vocab=True,
                            lower=True, include_lengths=True, batch_first=True, fix_length=max_length, init_token="[CLS]", eos_token="[SEP]", pad_token='[PAD]', unk_token='[UNK]')
LABEL = torchtext.data.Field(sequential=False, use_vocab=False)
ID = torchtext.data.Field(sequential=False, use_vocab=False)
# (注釈)：各引数を再確認
# sequential: データの長さが可変か？文章は長さがいろいろなのでTrue.ラベルはFalse
# tokenize: 文章を読み込んだときに、前処理や単語分割をするための関数を定義
# use_vocab：単語をボキャブラリーに追加するかどうか
# lower：アルファベットがあったときに小文字に変換するかどうか
# include_length: 文章の単語数のデータを保持するか
# batch_first：ミニバッチの次元を先頭に用意するかどうか
# fix_length：全部の文章を指定した長さと同じになるように、paddingします
# init_token, eos_token, pad_token, unk_token：文頭、文末、padding、未知語に対して、どんな単語を与えるかを指定

/Users/takanokaito/env/lib/python3.7/site-packages/torchtext/data/field.py:150: UserWarning: Field class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)

# フォルダ「data」から各csvファイルを読み込みます
# BERT用で処理するので、時間がかかります
train_val_ds, test_ds = torchtext.data.TabularDataset.splits(
    path='./data/', train='bert_train_data.csv',
    test='bert_test_data.csv', format='csv',
    skip_header=True,
    fields=[('id', ID), ('contents', TEXT), ('y', LABEL)])

# torchtext.data.Datasetのsplit関数で訓練データとvalidationデータを分ける
train_ds, val_ds = train_val_ds.split(
    split_ratio=0.95, random_state=random.seed(seed))

/Users/takanokaito/env/lib/python3.7/site-packages/torchtext/data/example.py:68: UserWarning: Example class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('Example class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.', UserWarning)
/Users/takanokaito/env/lib/python3.7/site-packages/torchtext/data/example.py:78: UserWarning: Example class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('Example class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.', UserWarning)

# BERTはBERTが持つ全単語でBertEmbeddingモジュールを作成しているので、ボキャブラリーとしては全単語を使用します
# そのため訓練データからボキャブラリーは作成しません

# まずBERT用の単語辞書を辞書型変数に用意します
from utils.bert import BertTokenizer, load_vocab

vocab_bert, ids_to_tokens_bert = load_vocab(
    vocab_file="./vocab/bert-base-uncased-vocab.txt")


# このまま、TEXT.vocab.stoi= vocab_bert (stoiはstring_to_IDで、単語からIDへの辞書)としたいですが、
# 一度bulild_vocabを実行しないとTEXTオブジェクトがvocabのメンバ変数をもってくれないです。
# （'Field' object has no attribute 'vocab' というエラーをはきます）

# 1度適当にbuild_vocabでボキャブラリーを作成してから、BERTのボキャブラリーを上書きします
TEXT.build_vocab(train_ds, min_freq=1)
vocab_bert['[unk]'] = vocab_bert['[UNK]']
TEXT.vocab.stoi = vocab_bert

# DataLoaderを作成します（torchtextの文脈では単純にiteraterと呼ばれています）
batch_size = 32  # BERTでは16、32あたりを使用する

train_dl = torchtext.data.Iterator(
    train_ds, batch_size=batch_size, train=True)

val_dl = torchtext.data.Iterator(
    val_ds, batch_size=batch_size, train=False, sort=False)

test_dl = torchtext.data.Iterator(
    test_ds, batch_size=batch_size, train=False, sort=False)

# 辞書オブジェクトにまとめる
dataloaders_dict = {"train": train_dl, "val": val_dl}

/Users/takanokaito/env/lib/python3.7/site-packages/torchtext/data/iterator.py:48: UserWarning: Iterator class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)

from utils.bert import get_config, BertModel, set_learned_params

# モデル設定のJOSNファイルをオブジェクト変数として読み込みます
config = get_config(file_path="./weights/bert_config.json")

# BERTモデルを作成します
net_bert = BertModel(config)

# BERTモデルに学習済みパラメータセットします
net_bert = set_learned_params(
    net_bert, weights_path="./weights/pytorch_model.bin")

bert.embeddings.word_embeddings.weight→embeddings.word_embeddings.weight
bert.embeddings.position_embeddings.weight→embeddings.position_embeddings.weight
bert.embeddings.token_type_embeddings.weight→embeddings.token_type_embeddings.weight
bert.embeddings.LayerNorm.gamma→embeddings.LayerNorm.gamma
bert.embeddings.LayerNorm.beta→embeddings.LayerNorm.beta
bert.encoder.layer.0.attention.self.query.weight→encoder.layer.0.attention.selfattn.query.weight
bert.encoder.layer.0.attention.self.query.bias→encoder.layer.0.attention.selfattn.query.bias
bert.encoder.layer.0.attention.self.key.weight→encoder.layer.0.attention.selfattn.key.weight
bert.encoder.layer.0.attention.self.key.bias→encoder.layer.0.attention.selfattn.key.bias
bert.encoder.layer.0.attention.self.value.weight→encoder.layer.0.attention.selfattn.value.weight
bert.encoder.layer.0.attention.self.value.bias→encoder.layer.0.attention.selfattn.value.bias
bert.encoder.layer.0.attention.output.dense.weight→encoder.layer.0.attention.output.dense.weight
bert.encoder.layer.0.attention.output.dense.bias→encoder.layer.0.attention.output.dense.bias
bert.encoder.layer.0.attention.output.LayerNorm.gamma→encoder.layer.0.attention.output.LayerNorm.gamma
bert.encoder.layer.0.attention.output.LayerNorm.beta→encoder.layer.0.attention.output.LayerNorm.beta
bert.encoder.layer.0.intermediate.dense.weight→encoder.layer.0.intermediate.dense.weight
bert.encoder.layer.0.intermediate.dense.bias→encoder.layer.0.intermediate.dense.bias
bert.encoder.layer.0.output.dense.weight→encoder.layer.0.output.dense.weight
bert.encoder.layer.0.output.dense.bias→encoder.layer.0.output.dense.bias
bert.encoder.layer.0.output.LayerNorm.gamma→encoder.layer.0.output.LayerNorm.gamma
bert.encoder.layer.0.output.LayerNorm.beta→encoder.layer.0.output.LayerNorm.beta
bert.encoder.layer.1.attention.self.query.weight→encoder.layer.1.attention.selfattn.query.weight
bert.encoder.layer.1.attention.self.query.bias→encoder.layer.1.attention.selfattn.query.bias
bert.encoder.layer.1.attention.self.key.weight→encoder.layer.1.attention.selfattn.key.weight
bert.encoder.layer.1.attention.self.key.bias→encoder.layer.1.attention.selfattn.key.bias
bert.encoder.layer.1.attention.self.value.weight→encoder.layer.1.attention.selfattn.value.weight
bert.encoder.layer.1.attention.self.value.bias→encoder.layer.1.attention.selfattn.value.bias
bert.encoder.layer.1.attention.output.dense.weight→encoder.layer.1.attention.output.dense.weight
bert.encoder.layer.1.attention.output.dense.bias→encoder.layer.1.attention.output.dense.bias
bert.encoder.layer.1.attention.output.LayerNorm.gamma→encoder.layer.1.attention.output.LayerNorm.gamma
bert.encoder.layer.1.attention.output.LayerNorm.beta→encoder.layer.1.attention.output.LayerNorm.beta
bert.encoder.layer.1.intermediate.dense.weight→encoder.layer.1.intermediate.dense.weight
bert.encoder.layer.1.intermediate.dense.bias→encoder.layer.1.intermediate.dense.bias
bert.encoder.layer.1.output.dense.weight→encoder.layer.1.output.dense.weight
bert.encoder.layer.1.output.dense.bias→encoder.layer.1.output.dense.bias
bert.encoder.layer.1.output.LayerNorm.gamma→encoder.layer.1.output.LayerNorm.gamma
bert.encoder.layer.1.output.LayerNorm.beta→encoder.layer.1.output.LayerNorm.beta
bert.encoder.layer.2.attention.self.query.weight→encoder.layer.2.attention.selfattn.query.weight
bert.encoder.layer.2.attention.self.query.bias→encoder.layer.2.attention.selfattn.query.bias
bert.encoder.layer.2.attention.self.key.weight→encoder.layer.2.attention.selfattn.key.weight
bert.encoder.layer.2.attention.self.key.bias→encoder.layer.2.attention.selfattn.key.bias
bert.encoder.layer.2.attention.self.value.weight→encoder.layer.2.attention.selfattn.value.weight
bert.encoder.layer.2.attention.self.value.bias→encoder.layer.2.attention.selfattn.value.bias
bert.encoder.layer.2.attention.output.dense.weight→encoder.layer.2.attention.output.dense.weight
bert.encoder.layer.2.attention.output.dense.bias→encoder.layer.2.attention.output.dense.bias
bert.encoder.layer.2.attention.output.LayerNorm.gamma→encoder.layer.2.attention.output.LayerNorm.gamma
bert.encoder.layer.2.attention.output.LayerNorm.beta→encoder.layer.2.attention.output.LayerNorm.beta
bert.encoder.layer.2.intermediate.dense.weight→encoder.layer.2.intermediate.dense.weight
bert.encoder.layer.2.intermediate.dense.bias→encoder.layer.2.intermediate.dense.bias
bert.encoder.layer.2.output.dense.weight→encoder.layer.2.output.dense.weight
bert.encoder.layer.2.output.dense.bias→encoder.layer.2.output.dense.bias
bert.encoder.layer.2.output.LayerNorm.gamma→encoder.layer.2.output.LayerNorm.gamma
bert.encoder.layer.2.output.LayerNorm.beta→encoder.layer.2.output.LayerNorm.beta
bert.encoder.layer.3.attention.self.query.weight→encoder.layer.3.attention.selfattn.query.weight
bert.encoder.layer.3.attention.self.query.bias→encoder.layer.3.attention.selfattn.query.bias
bert.encoder.layer.3.attention.self.key.weight→encoder.layer.3.attention.selfattn.key.weight
bert.encoder.layer.3.attention.self.key.bias→encoder.layer.3.attention.selfattn.key.bias
bert.encoder.layer.3.attention.self.value.weight→encoder.layer.3.attention.selfattn.value.weight
bert.encoder.layer.3.attention.self.value.bias→encoder.layer.3.attention.selfattn.value.bias
bert.encoder.layer.3.attention.output.dense.weight→encoder.layer.3.attention.output.dense.weight
bert.encoder.layer.3.attention.output.dense.bias→encoder.layer.3.attention.output.dense.bias
bert.encoder.layer.3.attention.output.LayerNorm.gamma→encoder.layer.3.attention.output.LayerNorm.gamma
bert.encoder.layer.3.attention.output.LayerNorm.beta→encoder.layer.3.attention.output.LayerNorm.beta
bert.encoder.layer.3.intermediate.dense.weight→encoder.layer.3.intermediate.dense.weight
bert.encoder.layer.3.intermediate.dense.bias→encoder.layer.3.intermediate.dense.bias
bert.encoder.layer.3.output.dense.weight→encoder.layer.3.output.dense.weight
bert.encoder.layer.3.output.dense.bias→encoder.layer.3.output.dense.bias
bert.encoder.layer.3.output.LayerNorm.gamma→encoder.layer.3.output.LayerNorm.gamma
bert.encoder.layer.3.output.LayerNorm.beta→encoder.layer.3.output.LayerNorm.beta
bert.encoder.layer.4.attention.self.query.weight→encoder.layer.4.attention.selfattn.query.weight
bert.encoder.layer.4.attention.self.query.bias→encoder.layer.4.attention.selfattn.query.bias
bert.encoder.layer.4.attention.self.key.weight→encoder.layer.4.attention.selfattn.key.weight
bert.encoder.layer.4.attention.self.key.bias→encoder.layer.4.attention.selfattn.key.bias
bert.encoder.layer.4.attention.self.value.weight→encoder.layer.4.attention.selfattn.value.weight
bert.encoder.layer.4.attention.self.value.bias→encoder.layer.4.attention.selfattn.value.bias
bert.encoder.layer.4.attention.output.dense.weight→encoder.layer.4.attention.output.dense.weight
bert.encoder.layer.4.attention.output.dense.bias→encoder.layer.4.attention.output.dense.bias
bert.encoder.layer.4.attention.output.LayerNorm.gamma→encoder.layer.4.attention.output.LayerNorm.gamma
bert.encoder.layer.4.attention.output.LayerNorm.beta→encoder.layer.4.attention.output.LayerNorm.beta
bert.encoder.layer.4.intermediate.dense.weight→encoder.layer.4.intermediate.dense.weight
bert.encoder.layer.4.intermediate.dense.bias→encoder.layer.4.intermediate.dense.bias
bert.encoder.layer.4.output.dense.weight→encoder.layer.4.output.dense.weight
bert.encoder.layer.4.output.dense.bias→encoder.layer.4.output.dense.bias
bert.encoder.layer.4.output.LayerNorm.gamma→encoder.layer.4.output.LayerNorm.gamma
bert.encoder.layer.4.output.LayerNorm.beta→encoder.layer.4.output.LayerNorm.beta
bert.encoder.layer.5.attention.self.query.weight→encoder.layer.5.attention.selfattn.query.weight
bert.encoder.layer.5.attention.self.query.bias→encoder.layer.5.attention.selfattn.query.bias
bert.encoder.layer.5.attention.self.key.weight→encoder.layer.5.attention.selfattn.key.weight
bert.encoder.layer.5.attention.self.key.bias→encoder.layer.5.attention.selfattn.key.bias
bert.encoder.layer.5.attention.self.value.weight→encoder.layer.5.attention.selfattn.value.weight
bert.encoder.layer.5.attention.self.value.bias→encoder.layer.5.attention.selfattn.value.bias
bert.encoder.layer.5.attention.output.dense.weight→encoder.layer.5.attention.output.dense.weight
bert.encoder.layer.5.attention.output.dense.bias→encoder.layer.5.attention.output.dense.bias
bert.encoder.layer.5.attention.output.LayerNorm.gamma→encoder.layer.5.attention.output.LayerNorm.gamma
bert.encoder.layer.5.attention.output.LayerNorm.beta→encoder.layer.5.attention.output.LayerNorm.beta
bert.encoder.layer.5.intermediate.dense.weight→encoder.layer.5.intermediate.dense.weight
bert.encoder.layer.5.intermediate.dense.bias→encoder.layer.5.intermediate.dense.bias
bert.encoder.layer.5.output.dense.weight→encoder.layer.5.output.dense.weight
bert.encoder.layer.5.output.dense.bias→encoder.layer.5.output.dense.bias
bert.encoder.layer.5.output.LayerNorm.gamma→encoder.layer.5.output.LayerNorm.gamma
bert.encoder.layer.5.output.LayerNorm.beta→encoder.layer.5.output.LayerNorm.beta
bert.encoder.layer.6.attention.self.query.weight→encoder.layer.6.attention.selfattn.query.weight
bert.encoder.layer.6.attention.self.query.bias→encoder.layer.6.attention.selfattn.query.bias
bert.encoder.layer.6.attention.self.key.weight→encoder.layer.6.attention.selfattn.key.weight
bert.encoder.layer.6.attention.self.key.bias→encoder.layer.6.attention.selfattn.key.bias
bert.encoder.layer.6.attention.self.value.weight→encoder.layer.6.attention.selfattn.value.weight
bert.encoder.layer.6.attention.self.value.bias→encoder.layer.6.attention.selfattn.value.bias
bert.encoder.layer.6.attention.output.dense.weight→encoder.layer.6.attention.output.dense.weight
bert.encoder.layer.6.attention.output.dense.bias→encoder.layer.6.attention.output.dense.bias
bert.encoder.layer.6.attention.output.LayerNorm.gamma→encoder.layer.6.attention.output.LayerNorm.gamma
bert.encoder.layer.6.attention.output.LayerNorm.beta→encoder.layer.6.attention.output.LayerNorm.beta
bert.encoder.layer.6.intermediate.dense.weight→encoder.layer.6.intermediate.dense.weight
bert.encoder.layer.6.intermediate.dense.bias→encoder.layer.6.intermediate.dense.bias
bert.encoder.layer.6.output.dense.weight→encoder.layer.6.output.dense.weight
bert.encoder.layer.6.output.dense.bias→encoder.layer.6.output.dense.bias
bert.encoder.layer.6.output.LayerNorm.gamma→encoder.layer.6.output.LayerNorm.gamma
bert.encoder.layer.6.output.LayerNorm.beta→encoder.layer.6.output.LayerNorm.beta
bert.encoder.layer.7.attention.self.query.weight→encoder.layer.7.attention.selfattn.query.weight
bert.encoder.layer.7.attention.self.query.bias→encoder.layer.7.attention.selfattn.query.bias
bert.encoder.layer.7.attention.self.key.weight→encoder.layer.7.attention.selfattn.key.weight
bert.encoder.layer.7.attention.self.key.bias→encoder.layer.7.attention.selfattn.key.bias
bert.encoder.layer.7.attention.self.value.weight→encoder.layer.7.attention.selfattn.value.weight
bert.encoder.layer.7.attention.self.value.bias→encoder.layer.7.attention.selfattn.value.bias
bert.encoder.layer.7.attention.output.dense.weight→encoder.layer.7.attention.output.dense.weight
bert.encoder.layer.7.attention.output.dense.bias→encoder.layer.7.attention.output.dense.bias
bert.encoder.layer.7.attention.output.LayerNorm.gamma→encoder.layer.7.attention.output.LayerNorm.gamma
bert.encoder.layer.7.attention.output.LayerNorm.beta→encoder.layer.7.attention.output.LayerNorm.beta
bert.encoder.layer.7.intermediate.dense.weight→encoder.layer.7.intermediate.dense.weight
bert.encoder.layer.7.intermediate.dense.bias→encoder.layer.7.intermediate.dense.bias
bert.encoder.layer.7.output.dense.weight→encoder.layer.7.output.dense.weight
bert.encoder.layer.7.output.dense.bias→encoder.layer.7.output.dense.bias
bert.encoder.layer.7.output.LayerNorm.gamma→encoder.layer.7.output.LayerNorm.gamma
bert.encoder.layer.7.output.LayerNorm.beta→encoder.layer.7.output.LayerNorm.beta
bert.encoder.layer.8.attention.self.query.weight→encoder.layer.8.attention.selfattn.query.weight
bert.encoder.layer.8.attention.self.query.bias→encoder.layer.8.attention.selfattn.query.bias
bert.encoder.layer.8.attention.self.key.weight→encoder.layer.8.attention.selfattn.key.weight
bert.encoder.layer.8.attention.self.key.bias→encoder.layer.8.attention.selfattn.key.bias
bert.encoder.layer.8.attention.self.value.weight→encoder.layer.8.attention.selfattn.value.weight
bert.encoder.layer.8.attention.self.value.bias→encoder.layer.8.attention.selfattn.value.bias
bert.encoder.layer.8.attention.output.dense.weight→encoder.layer.8.attention.output.dense.weight
bert.encoder.layer.8.attention.output.dense.bias→encoder.layer.8.attention.output.dense.bias
bert.encoder.layer.8.attention.output.LayerNorm.gamma→encoder.layer.8.attention.output.LayerNorm.gamma
bert.encoder.layer.8.attention.output.LayerNorm.beta→encoder.layer.8.attention.output.LayerNorm.beta
bert.encoder.layer.8.intermediate.dense.weight→encoder.layer.8.intermediate.dense.weight
bert.encoder.layer.8.intermediate.dense.bias→encoder.layer.8.intermediate.dense.bias
bert.encoder.layer.8.output.dense.weight→encoder.layer.8.output.dense.weight
bert.encoder.layer.8.output.dense.bias→encoder.layer.8.output.dense.bias
bert.encoder.layer.8.output.LayerNorm.gamma→encoder.layer.8.output.LayerNorm.gamma
bert.encoder.layer.8.output.LayerNorm.beta→encoder.layer.8.output.LayerNorm.beta
bert.encoder.layer.9.attention.self.query.weight→encoder.layer.9.attention.selfattn.query.weight
bert.encoder.layer.9.attention.self.query.bias→encoder.layer.9.attention.selfattn.query.bias
bert.encoder.layer.9.attention.self.key.weight→encoder.layer.9.attention.selfattn.key.weight
bert.encoder.layer.9.attention.self.key.bias→encoder.layer.9.attention.selfattn.key.bias
bert.encoder.layer.9.attention.self.value.weight→encoder.layer.9.attention.selfattn.value.weight
bert.encoder.layer.9.attention.self.value.bias→encoder.layer.9.attention.selfattn.value.bias
bert.encoder.layer.9.attention.output.dense.weight→encoder.layer.9.attention.output.dense.weight
bert.encoder.layer.9.attention.output.dense.bias→encoder.layer.9.attention.output.dense.bias
bert.encoder.layer.9.attention.output.LayerNorm.gamma→encoder.layer.9.attention.output.LayerNorm.gamma
bert.encoder.layer.9.attention.output.LayerNorm.beta→encoder.layer.9.attention.output.LayerNorm.beta
bert.encoder.layer.9.intermediate.dense.weight→encoder.layer.9.intermediate.dense.weight
bert.encoder.layer.9.intermediate.dense.bias→encoder.layer.9.intermediate.dense.bias
bert.encoder.layer.9.output.dense.weight→encoder.layer.9.output.dense.weight
bert.encoder.layer.9.output.dense.bias→encoder.layer.9.output.dense.bias
bert.encoder.layer.9.output.LayerNorm.gamma→encoder.layer.9.output.LayerNorm.gamma
bert.encoder.layer.9.output.LayerNorm.beta→encoder.layer.9.output.LayerNorm.beta
bert.encoder.layer.10.attention.self.query.weight→encoder.layer.10.attention.selfattn.query.weight
bert.encoder.layer.10.attention.self.query.bias→encoder.layer.10.attention.selfattn.query.bias
bert.encoder.layer.10.attention.self.key.weight→encoder.layer.10.attention.selfattn.key.weight
bert.encoder.layer.10.attention.self.key.bias→encoder.layer.10.attention.selfattn.key.bias
bert.encoder.layer.10.attention.self.value.weight→encoder.layer.10.attention.selfattn.value.weight
bert.encoder.layer.10.attention.self.value.bias→encoder.layer.10.attention.selfattn.value.bias
bert.encoder.layer.10.attention.output.dense.weight→encoder.layer.10.attention.output.dense.weight
bert.encoder.layer.10.attention.output.dense.bias→encoder.layer.10.attention.output.dense.bias
bert.encoder.layer.10.attention.output.LayerNorm.gamma→encoder.layer.10.attention.output.LayerNorm.gamma
bert.encoder.layer.10.attention.output.LayerNorm.beta→encoder.layer.10.attention.output.LayerNorm.beta
bert.encoder.layer.10.intermediate.dense.weight→encoder.layer.10.intermediate.dense.weight
bert.encoder.layer.10.intermediate.dense.bias→encoder.layer.10.intermediate.dense.bias
bert.encoder.layer.10.output.dense.weight→encoder.layer.10.output.dense.weight
bert.encoder.layer.10.output.dense.bias→encoder.layer.10.output.dense.bias
bert.encoder.layer.10.output.LayerNorm.gamma→encoder.layer.10.output.LayerNorm.gamma
bert.encoder.layer.10.output.LayerNorm.beta→encoder.layer.10.output.LayerNorm.beta
bert.encoder.layer.11.attention.self.query.weight→encoder.layer.11.attention.selfattn.query.weight
bert.encoder.layer.11.attention.self.query.bias→encoder.layer.11.attention.selfattn.query.bias
bert.encoder.layer.11.attention.self.key.weight→encoder.layer.11.attention.selfattn.key.weight
bert.encoder.layer.11.attention.self.key.bias→encoder.layer.11.attention.selfattn.key.bias
bert.encoder.layer.11.attention.self.value.weight→encoder.layer.11.attention.selfattn.value.weight
bert.encoder.layer.11.attention.self.value.bias→encoder.layer.11.attention.selfattn.value.bias
bert.encoder.layer.11.attention.output.dense.weight→encoder.layer.11.attention.output.dense.weight
bert.encoder.layer.11.attention.output.dense.bias→encoder.layer.11.attention.output.dense.bias
bert.encoder.layer.11.attention.output.LayerNorm.gamma→encoder.layer.11.attention.output.LayerNorm.gamma
bert.encoder.layer.11.attention.output.LayerNorm.beta→encoder.layer.11.attention.output.LayerNorm.beta
bert.encoder.layer.11.intermediate.dense.weight→encoder.layer.11.intermediate.dense.weight
bert.encoder.layer.11.intermediate.dense.bias→encoder.layer.11.intermediate.dense.bias
bert.encoder.layer.11.output.dense.weight→encoder.layer.11.output.dense.weight
bert.encoder.layer.11.output.dense.bias→encoder.layer.11.output.dense.bias
bert.encoder.layer.11.output.LayerNorm.gamma→encoder.layer.11.output.LayerNorm.gamma
bert.encoder.layer.11.output.LayerNorm.beta→encoder.layer.11.output.LayerNorm.beta
bert.pooler.dense.weight→pooler.dense.weight
bert.pooler.dense.bias→pooler.dense.bias

def gelu(x):
    return x * 0.5 * (1.0 + torch.erf(x / math.sqrt(2.0)))

class BertForClassify(nn.Module):

    def __init__(self, net_bert):
        super(BertForClassify, self).__init__()

        # BERTモジュール
        self.bert = net_bert  # BERTモデル

        # 入力はBERTの出力特徴量の次元、出力は1:Data scientist, 2:Machine learning engineer, 3:Software engineer, 4:Consultantの4つ
        self.cls1 = nn.Linear(in_features=768, out_features=64)
        self.cls2 = nn.Linear(in_features=64, out_features=2)

        # 重み初期化処理
        nn.init.normal_(self.cls1.weight, std=0.02)
        nn.init.normal_(self.cls1.bias, 0)
        nn.init.normal_(self.cls2.weight, std=0.02)
        nn.init.normal_(self.cls2.bias, 0)
        
        self.dropout = nn.Dropout(0.1)
        
        # 活性化関数gelu
        self.intermediate_act_fn = gelu

    def forward(self, input_ids, token_type_ids=None, attention_mask=None, output_all_encoded_layers=False, attention_show_flg=False):
        '''
        input_ids： [batch_size, sequence_length]の文章の単語IDの羅列
        token_type_ids： [batch_size, sequence_length]の、各単語が1文目なのか、2文目なのかを示すid
        attention_mask：Transformerのマスクと同じ働きのマスキングです
        output_all_encoded_layers：最終出力に12段のTransformerの全部をリストで返すか、最後だけかを指定
        attention_show_flg：Self-Attentionの重みを返すかのフラグ
        '''

        # BERTの基本モデル部分の順伝搬
        # 順伝搬させる
        if attention_show_flg == True:
            '''attention_showのときは、attention_probsもリターンする'''
            encoded_layers, pooled_output, attention_probs = self.bert(
                input_ids, token_type_ids, attention_mask, output_all_encoded_layers, attention_show_flg)
        elif attention_show_flg == False:
            encoded_layers, pooled_output = self.bert(
                input_ids, token_type_ids, attention_mask, output_all_encoded_layers, attention_show_flg)

        # 入力文章の1単語目[CLS]の特徴量を使用して、多値分類
        vec_0 = encoded_layers[:, 0, :]
        vec_0 = self.dropout(vec_0.view(-1, 768))  # sizeを[batch_size, hidden_sizeに変換
        hidden = self.cls1(vec_0)
        out = self.cls2(self.intermediate_act_fn(self.dropout(hidden)))

        # attention_showのときは、attention_probs（1番最後の）もリターンする
        if attention_show_flg == True:
            return out, attention_probs, hidden
        elif attention_show_flg == False:
            return out, hidden

# モデル構築
net = BertForClassify(net_bert)

# 訓練モードに設定
net.train()

print('ネットワーク設定完了')

ネットワーク設定完了

# 勾配計算を最後のBertLayerモジュールと追加した分類アダプターのみ実行

# 1. まず全部を、勾配計算Falseにしてしまう
for name, param in net.named_parameters():
    param.requires_grad = False

# 2. 最後のBertLayerモジュールを勾配計算ありに変更
for name, param in net.bert.encoder.layer[-1].named_parameters():
    param.requires_grad = True

# 3. 識別器を勾配計算ありに変更
for name, param in net.cls1.named_parameters():
    param.requires_grad = True
for name, param in net.cls2.named_parameters():
    param.requires_grad = True

# GPUが使えるかを確認
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# 最適化手法の設定

# BERTの元の部分はファインチューニング
optimizer = optim.Adam([
    {'params': net.bert.encoder.layer[-1].parameters(), 'lr': 5e-5},
    {'params': net.cls1.parameters(), 'lr': 5e-5},
    {'params': net.cls2.parameters(), 'lr': 5e-5}
], betas=(0.9, 0.999))

# 損失関数の設定
weights = np.array([0, 0])
for example in train_ds:
        weights[int(example.y)] += 1
weights = 1 / weights
weights /= np.sum(weights)
print(weights)
weights = torch.tensor(weights, dtype=torch.float32, device=device)
criterion = nn.CrossEntropyLoss(weight=weights)
# nn.LogSoftmax()を計算してからnn.NLLLoss(negative log likelihood loss)を計算

[0.09009513 0.90990487]

# モデルを学習させる関数を作成


def train_model(net, dataloaders_dict, criterion, optimizer, num_epochs):

    # GPUが使えるかを確認
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print("使用デバイス：", device)
    print('-----start-------')

    # ネットワークをGPUへ
    net.to(device)

    # ネットワークがある程度固定であれば、高速化させる
    torch.backends.cudnn.benchmark = True

    # ミニバッチのサイズ
    batch_size = dataloaders_dict["train"].batch_size
    
    # epochのループ
    for epoch in range(num_epochs):
        # epochごとの訓練と検証のループ
        for phase in ['train', 'val']:
            if phase == 'train':
                net.train()  # モデルを訓練モードに
            else:
                net.eval()   # モデルを検証モードに

            epoch_loss = 0.0  # epochの損失和
            epoch_corrects = 0  # epochの正解数
            iteration = 1
            
            # f1
            predlist=torch.zeros(0, dtype=torch.long, device='cpu')
            lbllist=torch.zeros(0, dtype=torch.long, device='cpu')

            # 開始時刻を保存
            t_epoch_start = time.time()
            t_iter_start = time.time()

            # データローダーからミニバッチを取り出すループ
            for batch in (dataloaders_dict[phase]):
                # batchはTextとLableの辞書型変数

                # GPUが使えるならGPUにデータを送る
                inputs = batch.contents[0].to(device)  # 文章
                labels = batch.y.to(device)  # ラベル

                # optimizerを初期化
                optimizer.zero_grad()

                # 順伝搬（forward）計算
                with torch.set_grad_enabled(phase == 'train'):

                    # BertForClassifyに入力
                    outputs, hidden = net(inputs, token_type_ids=None, attention_mask=None,
                                  output_all_encoded_layers=False, attention_show_flg=False)

                    loss = criterion(outputs, labels)  # 損失を計算

                    _, preds = torch.max(outputs, 1)  # ラベルを予測

                    # 訓練時はバックプロパゲーション
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                        if (iteration % 10 == 0):  # 10iterに1度、lossを表示
                            t_iter_finish = time.time()
                            duration = t_iter_finish - t_iter_start
                            acc = (torch.sum(preds == labels.data)
                                   ).double()/batch_size
                            print('イテレーション {} || Loss: {:.4f} || 10iter: {:.4f} sec. || 本イテレーションの正解率：{}'.format(
                                iteration, loss.item(), duration, acc))
                            print(preds[0:10])
                            t_iter_start = time.time()

                    iteration += 1

                    # 損失と正解数の合計を更新
                    epoch_loss += loss.item() * batch_size
                    epoch_corrects += torch.sum(preds == labels.data)
                    
                    # f1
                    predlist=torch.cat([predlist,preds.detach().view(-1).cpu()])
                    lbllist=torch.cat([lbllist,labels.view(-1).cpu()])

            # epochごとのlossと正解率
            t_epoch_finish = time.time()
            epoch_loss = epoch_loss / len(dataloaders_dict[phase].dataset)
            epoch_acc = epoch_corrects.double() / len(dataloaders_dict[phase].dataset)

            print('Epoch {}/{} | {:^5} |  Loss: {:.4f} Acc: {:.4f}'.format(epoch+1, num_epochs,
                                                                           phase, epoch_loss, epoch_acc))
            # f1
            f1 = f1_score(lbllist, predlist)
            print('f1_score {:^5}'.format(f1))
            t_epoch_start = time.time()
    return net

# 学習・検証を実行する。
num_epochs = 5
net_trained = train_model(net, dataloaders_dict,
                          criterion, optimizer, num_epochs=num_epochs)

使用デバイス： cpu
-----start-------
/Users/takanokaito/env/lib/python3.7/site-packages/torchtext/data/batch.py:23: UserWarning: Batch class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.
  warnings.warn('{} class will be retired in the 0.8.0 release and moved to torchtext.legacy. Please see 0.7.0 release notes for further information.'.format(self.__class__.__name__), UserWarning)
イテレーション 10 || Loss: 0.6992 || 10iter: 314.6658 sec. || 本イテレーションの正解率：0.21875
tensor([1, 0, 1, 1, 1, 1, 1, 1, 1, 1])
イテレーション 20 || Loss: 0.6841 || 10iter: 316.3488 sec. || 本イテレーションの正解率：0.53125
tensor([0, 0, 0, 1, 0, 0, 1, 1, 0, 1])
イテレーション 30 || Loss: 0.6945 || 10iter: 309.8897 sec. || 本イテレーションの正解率：0.5625
tensor([0, 1, 1, 0, 1, 1, 1, 0, 0, 1])
イテレーション 40 || Loss: 0.6443 || 10iter: 308.1357 sec. || 本イテレーションの正解率：0.5625
tensor([1, 0, 0, 1, 1, 0, 1, 0, 0, 0])
イテレーション 50 || Loss: 0.6574 || 10iter: 311.2739 sec. || 本イテレーションの正解率：0.5
tensor([1, 1, 1, 0, 0, 1, 1, 0, 0, 1])
Epoch 1/5 | train |  Loss: 0.6825 Acc: 0.4314
f1_score 0.1858974358974359
Epoch 1/5 |  val  |  Loss: 0.6598 Acc: 0.6596
f1_score 0.30434782608695654
イテレーション 10 || Loss: 0.6127 || 10iter: 309.3094 sec. || 本イテレーションの正解率：0.59375
tensor([0, 0, 0, 0, 0, 0, 1, 1, 0, 1])
イテレーション 20 || Loss: 0.6042 || 10iter: 307.4083 sec. || 本イテレーションの正解率：0.5
tensor([1, 1, 1, 1, 1, 1, 1, 1, 0, 1])
イテレーション 30 || Loss: 0.5008 || 10iter: 309.7979 sec. || 本イテレーションの正解率：0.96875
tensor([0, 1, 0, 0, 0, 0, 0, 0, 0, 0])
イテレーション 40 || Loss: 0.4990 || 10iter: 311.8554 sec. || 本イテレーションの正解率：0.8125
tensor([0, 1, 1, 0, 1, 0, 0, 0, 0, 1])
イテレーション 50 || Loss: 0.5484 || 10iter: 312.6560 sec. || 本イテレーションの正解率：0.84375
tensor([0, 0, 1, 0, 0, 0, 0, 0, 0, 0])
Epoch 2/5 | train |  Loss: 0.5198 Acc: 0.7879
f1_score 0.3955342902711324
Epoch 2/5 |  val  |  Loss: 0.3021 Acc: 0.9681
f1_score 0.8421052631578948
イテレーション 10 || Loss: 0.2054 || 10iter: 314.0214 sec. || 本イテレーションの正解率：0.96875
tensor([0, 0, 0, 0, 0, 0, 0, 0, 1, 0])
イテレーション 20 || Loss: 0.2310 || 10iter: 301.7984 sec. || 本イテレーションの正解率：0.96875
tensor([0, 0, 0, 0, 1, 1, 0, 0, 0, 0])
イテレーション 30 || Loss: 0.1743 || 10iter: 305.6015 sec. || 本イテレーションの正解率：0.9375
tensor([0, 0, 1, 0, 0, 0, 0, 0, 0, 0])
イテレーション 40 || Loss: 0.1855 || 10iter: 300.2320 sec. || 本イテレーションの正解率：0.96875
tensor([1, 0, 1, 0, 1, 0, 0, 0, 0, 0])
イテレーション 50 || Loss: 0.1402 || 10iter: 300.8098 sec. || 本イテレーションの正解率：0.9375
tensor([1, 1, 0, 0, 0, 0, 0, 0, 0, 0])
Epoch 3/5 | train |  Loss: 0.2479 Acc: 0.9407
f1_score 0.7389162561576356
Epoch 3/5 |  val  |  Loss: 0.1817 Acc: 0.9787
f1_score  0.9 
イテレーション 10 || Loss: 0.1093 || 10iter: 308.6435 sec. || 本イテレーションの正解率：0.9375
tensor([1, 1, 0, 1, 1, 0, 0, 0, 0, 0])
イテレーション 20 || Loss: 0.3410 || 10iter: 299.6432 sec. || 本イテレーションの正解率：0.9375
tensor([0, 0, 0, 1, 0, 0, 0, 0, 0, 0])
イテレーション 30 || Loss: 0.1796 || 10iter: 299.2126 sec. || 本イテレーションの正解率：0.90625
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
イテレーション 40 || Loss: 0.0584 || 10iter: 300.2765 sec. || 本イテレーションの正解率：1.0
tensor([0, 0, 0, 0, 0, 0, 0, 0, 1, 0])
イテレーション 50 || Loss: 0.0743 || 10iter: 309.4848 sec. || 本イテレーションの正解率：0.96875
tensor([1, 0, 0, 0, 0, 0, 0, 0, 0, 1])
Epoch 4/5 | train |  Loss: 0.1324 Acc: 0.9659
f1_score 0.8355795148247979
Epoch 4/5 |  val  |  Loss: 0.1379 Acc: 0.9255
f1_score 0.7407407407407407
イテレーション 10 || Loss: 0.0915 || 10iter: 302.7578 sec. || 本イテレーションの正解率：1.0
tensor([0, 0, 0, 0, 0, 0, 1, 0, 0, 0])
イテレーション 20 || Loss: 0.0432 || 10iter: 303.4872 sec. || 本イテレーションの正解率：1.0
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
イテレーション 30 || Loss: 0.0363 || 10iter: 305.0603 sec. || 本イテレーションの正解率：1.0
tensor([0, 0, 0, 0, 0, 0, 0, 1, 0, 0])
イテレーション 40 || Loss: 0.0462 || 10iter: 303.0074 sec. || 本イテレーションの正解率：1.0
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
イテレーション 50 || Loss: 0.1114 || 10iter: 301.9711 sec. || 本イテレーションの正解率：0.96875
tensor([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
Epoch 5/5 | train |  Loss: 0.0856 Acc: 0.9743
f1_score 0.8736263736263736
Epoch 5/5 |  val  |  Loss: 0.1774 Acc: 0.9681
f1_score 0.8571428571428572

# 学習したネットワークパラメータを保存します
save_path = './weights/bert_fine_tuning_' + str(num_epochs) + '.pth'
torch.save(net_trained.state_dict(), save_path)

# テストデータ
pred_label_list = torch.zeros(0, dtype=torch.long, device='cpu')

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

net_trained.eval()   # モデルを検証モードに
net_trained.to(device)  # GPUが使えるならGPUへ送る

for batch in tqdm(test_dl):  # testデータのDataLoader
    # batchはTextとLableの辞書オブジェクト
    # GPUが使えるならGPUにデータを送る
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    inputs = batch.contents[0].to(device)  # 文章
    # labels = batch.Label.to(device)  # ラベル
    
    # 順伝搬（forward）計算
    with torch.set_grad_enabled(False):
        # BertForClassifyに入力
        outputs, hidden = net_trained(inputs, token_type_ids=None, attention_mask=None,
                              output_all_encoded_layers=False, attention_show_flg=False)

        _, preds = torch.max(outputs, 1)  # ラベルを予測
        pred_label_list = torch.cat([pred_label_list,preds.detach().view(-1).cpu()])

100%|██████████| 777/777 [5:08:09<00:00, 23.80s/it]

sub_df  = pd.read_csv('./data/bert_test_data.csv')
sub_df['y'] = np.array(pred_label_list)
sub_df[['id', 'y']].to_csv('./submit/bert_submit.csv', header=True, index=False)

sub_df.head()

	id	contents	y
0	1	Subject: re : weather and energy price data\r\...	0
1	2	Subject: organizational study\r\ngpg and eott ...	0
2	3	Subject: re [ 7 ] : talk about our meds\r\nske...	1
3	4	Subject: report about your cable service\r\nhi...	1
4	5	Subject: start date : 1 / 26 / 02 ; hourahead ...	0

np.sum(sub_df['y'])

添付データ

baseline_bert.ipynb?X-Amz-Expires=10800&X-Amz-Date=20241230T154001Z&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAIP7GCBGMWPMZ42PQ

BERTを用いたbaseline．LB: 0.954

はじめに

不均衡データ対策

学習に時間がかかる

添付データ

takaito

Apolo_py

takaito

takaito

new user