import pandas as pd
import numpy as np
import re
import matplotlib.pyplot as plt
import seaborn as sns
import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score
import warnings
warnings.filterwarnings('ignore')

# データの読込
train = pd.read_csv("./features/train_data.csv")
test  = pd.read_csv('./features/test_data.csv')

# 欠損値は、全て「－１」とする。
def fill_all_null(df):
    for col_name in df.columns[df.isnull().sum()!=0]:
        df[col_name] = df[col_name].fillna(-1)

# 訓練データ、テストデータの欠損値を補完
fill_all_null(train)
fill_all_null(test)

# ターゲットエンコーディングの関数定義
def change_to_target2(train_df,test_df,input_column_name,output_column_name):
    from sklearn.model_selection import KFold
    
    # nan埋め処理
    train[input_column_name] = train[input_column_name].fillna('-1').isnull().sum()
    test[input_column_name] = test[input_column_name].fillna('-1').isnull().sum()

    kf = KFold(n_splits=5, shuffle=True, random_state=71)
    #=========================================================#
    c=input_column_name
    # 学習データ全体で各カテゴリにおけるyの平均を計算
    data_tmp = pd.DataFrame({c: train_df[c],'target':train_df['y']})
    target_mean = data_tmp.groupby(c)['target'].mean()
    #テストデータのカテゴリを置換
    test_df[output_column_name] = test_df[c].map(target_mean)
    
    # 変換後の値を格納する配列を準備
    tmp = np.repeat(np.nan, train_df.shape[0])

    for i, (train_index, test_index) in enumerate(kf.split(train_df)): # NFOLDS回まわる
        #学習データについて、各カテゴリにおける目的変数の平均を計算
        target_mean = data_tmp.iloc[train_index].groupby(c)['target'].mean()
        #バリデーションデータについて、変換後の値を一時配列に格納
        tmp[test_index] = train_df[c].iloc[test_index].map(target_mean) 

    #変換後のデータで元の変数を置換
    train_df[output_column_name] = tmp
#========================================================#

# オブジェクトの列のリストを作成
object_col_list = train.select_dtypes(include=object).columns

# オブジェクトの列は全てターゲットエンコーディング実施
for col in object_col_list:
    change_to_target2(train,test,col,"enc_"+col)

#　変換前の列を削除
train = train.drop(object_col_list,axis=1)
test = test.drop(object_col_list,axis=1)

#　'id'の列を削除
train = train.drop('id',axis=1)
test  = test.drop('id',axis=1)

enc_obj_list = ['enc_'+s for s in object_col_list.tolist()]
print(train[enc_obj_list].head())