import re
import string

import matplotlib.pyplot as plt
import nltk
import numpy as np
import pandas as pd
import scipy.stats as stats
import seaborn as sns
from bs4 import BeautifulSoup
from imblearn.under_sampling import RandomUnderSampler
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.metrics import f1_score
from sklearn.naive_bayes import MultinomialNB
from wordcloud import STOPWORDS, WordCloud

df = pd.read_csv("train_data.csv")
test = pd.read_csv("test_data.csv")

df.head()

stop = set(stopwords.words("english"))

# string.punctuation = ['!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~']
punctuation = list(string.punctuation)

# 手動で追加
org_stop = ["Subject"]

# stopwordsの定義更新
add_stop = punctuation + org_stop
stop.update(add_stop)

# htmlの分割
def strip_html(text):
    soup = BeautifulSoup(text, "html.parser")
    return soup.get_text()

# []で囲まれた文章の削除（脚注、linkなど）
def remove_between_square_brackets(text):
    return re.sub('\[[^]]*\]', '', text)
# URLの削除
def remove_between_square_brackets(text):
    return re.sub(r'http\S+', '', text)

# stopwordsの削除
def remove_stopwords(text):
    final_text = []
    for i in text.split():
        if i.strip().lower() not in stop:
            if i.strip().isalpha():
                final_text.append(i.strip())
    return " ".join(final_text)

# ↑の関数をまとめて適用する関数
def denoise_text(text):
    text = strip_html(text)
    text = remove_between_square_brackets(text)
    text = remove_stopwords(text)
    return text

# ↑の関数の適用
df['contents']=df['contents'].apply(denoise_text)

sns.countplot(df["y"])

# 非spam
plt.figure(figsize=(20,20))
wc = WordCloud(max_words = 2000, width = 1600, height = 800, stopwords = STOPWORDS).generate(" ".join(df[df.y == 0].contents))
plt.imshow(wc, interpolation="bilinear")

# spam
plt.figure(figsize=(20,20))
wc = WordCloud(max_words = 2000, width = 1600, height = 800, stopwords = STOPWORDS).generate(" ".join(df[df.y == 1].contents))
plt.imshow(wc, interpolation="bilinear")

df["length"] = df["contents"].apply(len)
df.head()

fig,ax = plt.subplots(1,2,figsize=(12,4))

df[df["y"]==0].hist(column="length", bins=2500, ax=ax[0], color = "green")
ax[0].set(xlabel = "Length Class", ylabel = "Frequency", title = "not spam")
ax[0].set_xlim([0,12000])

df[df["y"]==1].hist(column="length", bins=50, ax=ax[1], color = "red")
ax[1].set(xlabel = "Length Class", ylabel = "Frequency", title = "spam")

plt.show()

# 文章の長さの差を数値でも確認
df.groupby("y")["length"].mean()

df["words"] = df["contents"].str.split().map(lambda x: len(x))
df.head()

fig,ax = plt.subplots(1,2,figsize=(12,4))

df[df["y"]==0].hist(column="words", bins=2500, ax=ax[0], color = "green")
ax[0].set(xlabel = "words Class", ylabel = "Frequency", title = "not spam")
ax[0].set_xlim([0,1200])

df[df["y"]==1].hist(column="words", bins=50, ax=ax[1], color = "red")
ax[1].set(xlabel = "words Class", ylabel = "Frequency", title = "spam")

plt.show()

# 単語の多さの差を数値でも確認
df.groupby("y")["words"].mean()

# trainデータを分割
X = pd.DataFrame(df['contents'])
y = pd.DataFrame(df['y'])

seed = [i for i in range(5)]
preds = []

for i in seed:
    # アンダーサンプリングしながら訓練用データの作成
    sampler = RandomUnderSampler(random_state=i)
    X_resampled, y_resampled = sampler.fit_resample(X,y)
    
    # 検証用データの作成（random_stateを変えて取得）
    sampler = RandomUnderSampler(random_state=i+10)
    X_val, y_val = sampler.fit_resample(X,y)
    
    # 単語の出現回数取得
    vec_count = CountVectorizer(min_df=3)
    vec_count.fit(X_resampled['contents'])

    # 単語の種類の表示
    print('word size: ', len(vec_count.vocabulary_))
    # 先頭５件の単語を表示
    print('word content: ', dict(list(vec_count.vocabulary_.items())[0:5]))
    
    # 訓練データと検証データをベクトル化
    X_resampled_vec = vec_count.transform(X_resampled['contents'])
    X_val_vec = vec_count.transform(X_val["contents"])
    
    # ベルヌーイモデルで学習
    model = MultinomialNB()
    model.fit(X_resampled_vec, y_resampled['y'])
    
    # F1スコアのprint
    print("val_F1-score",f1_score(model.predict(X_val_vec),y_val))
    print("------------------------------------------------------------------------------------------------------------------")
    
    # testデータをベクトル化
    df_data = pd.DataFrame(test, columns=['contents'])
    input_vec = vec_count.transform(df_data['contents'])

    # testの予測→seed毎に空のリストに格納
    pred = model.predict(input_vec)
    preds.append(pred)

# 予測結果を多数決
prediction = stats.mode(preds)[0]

# spam判定数を確認
# test全体で17000とのこと
prediction.sum()

# 提出用DaraFrameの作成
sub = pd.DataFrame(test["id"])
sub["y"] = prediction.T

# 予測ファイルの作成
# LB:0.9688243693
sub.to_csv("submission.csv",index=False)