# ------------------------------------------------------------------------------
# 各種ライブラリのインポート
# ------------------------------------------------------------------------------
import pandas as pd
import numpy as np
import json
import os
import random
import string
import re

from pathlib import Path
from tqdm import tqdm

import lightgbm as lgb
from sklearn.model_selection import KFold,GroupKFold
from sklearn.metrics import f1_score
from sklearn import preprocessing
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from  sklearn.neural_network import MLPRegressor
from  sklearn.pipeline import make_pipeline
from tqdm import tqdm

#cd /XXXX/XXX/notebook   カレントディレクトリをこのファイルを保存したディレクトリに移動

# xfeatのインストール
!pip install git+https://github.com/pfnet-research/xfeat.git

# データ読み込み
#####################################
###### train ########################
#####################################

train = pd.read_csv('../data/train_data.csv')
target = train['y']
train = train.drop(['id','y'],axis=1)

#####################################
#### test ###########################
#####################################

test = pd.read_csv('../data/test_data.csv')
test = test.drop('id',axis=1)

#####################################
#### test ###########################
#####################################

game = pd.read_csv('../data/game_info.csv')
game = game.drop('Unnamed: 0',axis=1)

train.shape,test.shape,target.shape,game.shape

# 訓練データの球速を他の特徴量から予測できるように仮の目的変数とする。
# 欠損値は直前の値を入れて補完
target_speed = train['speed'].str.extract(r'(\d+)').fillna(method='ffill')

##############################
# チーム名を数字に置き換える
##############################
# 全チーム名のリストを作成する
TeamList = game['topTeam'].unique()
# チーム名のディクショナリを初期化
TeamDic = {}
#　チーム名毎に数字を割り当てたディクショナリを初期化
for i in range(len(TeamList)):
  TeamDic[TeamList[i]] = i

game['bottomTeam']=game['bottomTeam'].replace(TeamDic)
game['topTeam']=game['topTeam'].replace(TeamDic)

game.tail()

# 年月日、曜日、時分秒を追加
game['startDayTime'] = pd.to_datetime(game['startDayTime']) # 型を変換
game['year']=game["startDayTime"].dt.year
game['month']=game["startDayTime"].dt.month
game['day']=game["startDayTime"].dt.day
game['hour']=game["startDayTime"].dt.hour
game['dayofweek']=game["startDayTime"].dt.dayofweek
game['minute']=game["startDayTime"].dt.minute
game['second']=game["startDayTime"].dt.second
# 'startDayTime'を削除
game = game.drop(['startDayTime'],axis=1)

# 訓練データのみにある列名（テストデータにはない列名）のリストを作成
delcollist = []
for col in train.columns:
  if not col in test.columns:
    delcollist.append(col)
# 訓練データのみにある列名を削除
train = train.drop(delcollist,axis=1)

# inning を　数値に変換
train['inning_num'] =  train['inning'].apply(lambda x: re.sub("\\D", "", x))
test['inning_num'] =  test['inning'].apply(lambda x: re.sub("\\D", "", x))

# 表裏を判定する関数
def omote_ura(x):
  if '表' in x:
    return 0
  else:
    return 1

# 表裏の列を追加
train['inning_ForB'] =  train['inning'].apply(lambda x: omote_ura(x))
test['inning_ForB'] =  test['inning'].apply(lambda x: omote_ura(x))

# game_infoの追加
train = pd.merge(train, game, how='left')
test = pd.merge(test, game, how='left')

# inningの削除
train = train.drop('inning',axis=1)
test = test.drop('inning',axis=1)

# ボール、ストライク、アウトの合計値を追加
train['total_stat'] = train['B']+train['S']+train['O']
test['total_stat'] = test['B']+test['S']+test['O']
train['B_S'] = train['B']+train['S']
test['B_S'] = test['B']+test['S']

# ベース上のランナーの数を追加
train['total_base'] = train['b1'].astype('int')+train['b2'].astype('int')+train['b3'].astype('int')
test['total_base'] = test['b1'].astype('int')+test['b2'].astype('int')+test['b3'].astype('int')

# バッターのチームを追加
train['batterTeam'] = train['topTeam']
train['batterTeam'] = train['batterTeam'].where(train['inning_ForB']==1, train['bottomTeam'])
test['batterTeam'] = test['topTeam']
test['batterTeam'] = test['batterTeam'].where(test['inning_ForB']==1, test['bottomTeam'])

# ピッチャーのチームを追加
train['pitcherTeam'] = train['topTeam']
train['pitcherTeam'] = train['pitcherTeam'].where(train['inning_ForB']==0, train['bottomTeam'])
test['pitcherTeam'] = test['topTeam']
test['pitcherTeam'] = test['pitcherTeam'].where(test['inning_ForB']==0, test['bottomTeam'])

# カテゴリカル変数のカラムを抽出
categorical_columns = [x for x in train.columns if train[x].dtypes == 'object']

# カテゴリカル変数をカウントエンコードする
from xfeat import CountEncoder

encoder = CountEncoder(input_cols=categorical_columns)
train = encoder.fit_transform(train)
test = encoder.transform(test)

# 訓練データにターゲット列を追加する
train['target'] = target

# カテゴリカル変数をターゲットエンコーディングする
from sklearn.model_selection import KFold
from xfeat import TargetEncoder

fold = KFold(n_splits=5, shuffle=True, random_state=42)
encoder = TargetEncoder(input_cols=categorical_columns,
                        target_col='target',
                        fold=fold)
train = encoder.fit_transform(train)
test = encoder.transform(test)

# エンコーディング前の列を削除する
train = train.drop(categorical_columns,axis=1)
test = test.drop(categorical_columns,axis=1)

# ターゲット列を削除
train = train.drop('target',axis=1)

train.shape,test.shape,target.shape,target_speed.shape

# pivot tabel を用いた特徴量を追加する関数
def get_game_id_vecs_features(input_df):
    _input_df = input_df
    # pivot table
    stat_df = pd.pivot_table(_input_df, index="gameID", columns="batter_te", values="total_stat").add_prefix("total_stat=")
    base_df = pd.pivot_table(_input_df, index="gameID", columns="batter_te", values="total_base").add_prefix("total_base=")
    inning_df = pd.pivot_table(_input_df, index="gameID", columns="batter_te", values="inning_num_ce").add_prefix("inning=")
    all_df = pd.concat([stat_df, base_df, inning_df], axis=1)
    
    # PCA all 
    sc_all_df = StandardScaler().fit_transform(all_df.fillna(0))
    pca = PCA(n_components=59, random_state=2021)
    pca_all_df = pd.DataFrame(pca.fit_transform(sc_all_df), index=all_df.index).rename(columns=lambda x: f"gameID_all_PCA={x:03}")
    # PCA Stat
    sc_stat_df = StandardScaler().fit_transform(stat_df.fillna(0))
    pca = PCA(n_components=16, random_state=2021)
    pca_stat_df = pd.DataFrame(pca.fit_transform(sc_stat_df), index=all_df.index).rename(columns=lambda x: f"gameID_stat_PCA={x:03}")
    # PCA bace
    sc_base_df = StandardScaler().fit_transform(base_df.fillna(0))
    pca = PCA(n_components=16, random_state=2021)
    pca_base_df = pd.DataFrame(pca.fit_transform(sc_base_df), index=all_df.index).rename(columns=lambda x: f"gameID_base_PCA={x:03}")
    # PCA inning
    sc_inning_df = StandardScaler().fit_transform(inning_df.fillna(0))
    pca = PCA(n_components=16, random_state=2021)
    pca_inning_df = pd.DataFrame(pca.fit_transform(sc_inning_df), index=all_df.index).rename(columns=lambda x: f"gameID_inning_PCA={x:03}")
    
    df = pd.concat([all_df, pca_all_df, pca_stat_df, pca_base_df, pca_inning_df], axis=1)
    output_df = pd.merge(_input_df[["gameID"]], df, left_on="gameID", right_index=True, how="left")
    return output_df

# 訓練データとテストデータを結合する
input_df = pd.concat([train, test]).reset_index(drop=True)  # use concat data

#　ピボットデータを作成する
output_df = get_game_id_vecs_features(input_df)

# ピボットデータを訓練データとテストデータに分割する
train_x = output_df.iloc[:len(train)]
test_x = output_df.iloc[len(train):].reset_index(drop=True)

train_x.shape,test_x.shape,train.shape,test.shape,target.shape,target_speed.shape

# 元データとピボットデータを結合する
input_all_df = pd.concat([input_df,output_df],axis=1)
input_all_df.shape

# null のカラムの確認
nul_sum = input_all_df.isnull().sum()
null_cols = list(nul_sum[nul_sum > 0].index)

# null があるカラムの削除
input_all_df = input_all_df.drop(null_cols,axis=1)

from sklearn.model_selection import train_test_split
from sklearn.feature_selection import VarianceThreshold

# 分散が0（すべて同じ値）のカラムの探索
sel = VarianceThreshold(threshold=0)
sel.fit(input_all_df)

# get_supportで分散が0でないカラムのみをTrue値、分散が0のカラムはFalse値を返します
print(sum(sel.get_support()))

# 分散が0のカラムを削除
input_all_df =input_all_df.loc[:, sel.get_support()]
print(input_all_df.shape)

# indexとcolumnsを入れ替える
input_all_df_T = input_all_df.T

print(input_all_df_T.duplicated().sum())

# 同じ特徴量の名前を取得
duplicated_features = input_all_df_T[input_all_df_T.duplicated()].index.values

# 値が同じ特徴量の片方を削除
input_all_df = input_all_df.drop(duplicated_features,axis=1)

print(input_all_df.shape)

# テストデータと訓練データに分ける
X_train = input_all_df.iloc[:len(train)]
X_test = input_all_df.iloc[len(train):].reset_index(drop=True)

X_train.shape,X_test.shape

# 作成した特徴量のデータを保存しておく
X_train.to_csv('../features/preprocessed_train.csv',index=False)
X_test.to_csv('../features/preprocessed_test.csv',index=False)
target.to_csv('../features/preprocessed_target.csv',index=False)
target_speed.to_csv('../features/preprocessed_speed.csv',index=False)

SEED = 42
NFOLDS = 5

# speed のデータを１次元に変換
target_speed = target_speed.to_numpy().reshape(-1,)

#ニューラルネットを作成する関数定義
def create_model_NN(activation, n_layers, n_neurons, solver):
    hidden_layer_sizes=[]
    
    #与えられたパラメータのレイヤを作成
    for i in range(n_layers):
        hidden_layer_sizes.append(n_neurons[i])
    #print('hidden_layer_sizes -> ' + str(hidden_layer_sizes))
    
    #ニューラルネットのモデルを作成
    model = MLPRegressor(activation = activation,
                         hidden_layer_sizes=hidden_layer_sizes,
                         solver = solver,
                         random_state=42
                        )
    #標準化とニューラルネットのパイプラインを作成
    pipe = make_pipeline(StandardScaler(),model)
    return pipe

# テストデータの「Speed」を予測する関数
def pred_speed_of_test_data(train_x,test,target_speed,param):
    ###################################
    ### パラメータの設定
    ##################################
    activation = param['activation']
    n_layers = param['n_layers']
    n_neurons=[]
    for i in range(n_layers):
        n_neurons.append(param['neuron' + str(i).zfill(2)])
    solver = param['solver']
    
    ###################################
    ### CVの設定
    ##################################
    
    FOLD_NUM = 5
    kf = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)

    scores = []
    mlp_pred = 0

    for i, (tdx, vdx) in enumerate(kf.split(X=train_x)):
        X_train, X_valid, y_train, y_valid = train_x.iloc[tdx], train_x.iloc[vdx], target_speed[tdx], target_speed[vdx]
        #モデルを作成
        mlp  = create_model_NN(activation, n_layers, n_neurons, solver)
        # 学習
        mlp.fit(X_train,y_train)
        # 予測
        mlp_pred += mlp.predict(test) / FOLD_NUM

    print('#######################################################')
    print('### Seed was predicted #######')
    print('#######################################################')
    return mlp_pred

# Speed予測用のハイパーパラメータ
param = {
"activation": 'tanh',
"n_layers": 9,
"neuron00": 45,
"neuron01": 52,
"neuron02": 57,
"neuron03": 79,
"neuron04": 21,
"neuron05": 102,
"neuron06": 118,
"neuron07": 31,
"neuron08": 66,
"solver": 'sgd',
}

# テストデータの「Speed」を予測する
speed_pred = pred_speed_of_test_data(X_train,X_test,target_speed,param)

target.shape

target_speed

speed_pred

# テストデータの「ｙ」を予測する関数
#####################################################3
### LGBで学習、予測する関数の定義
########################################################
def pred_y_of_test_data(train,test,target,lgb_param,mlp_pred,select_col_list):
    # --------------------------------------
    # パラメータ定義
    # --------------------------------------
    lgb_params = {
                    'objective': 'multiclass',
                    'boosting_type': 'gbdt',
                    'n_estimators': 50000,
                    'colsample_bytree': 0.5,
                    'subsample': 0.5,
                    'subsample_freq': 3,
                    'reg_alpha': 8,
                    'reg_lambda': 2,
                    'random_state': SEED,
        'bagging_fraction': lgb_param['bagging_fraction'],
        'bagging_freq': lgb_param['bagging_freq'],        
        'feature_fraction': lgb_param['feature_fraction'],
        "learning_rate":lgb_param['learning_rate'],
        'min_child_samples': lgb_param['min_child_samples'],
        'num_leaves': lgb_param['num_leaves'],
        
                  }

    # --------------------------------------
    # 学習と予測
    # --------------------------------------
    kf = KFold(n_splits=NFOLDS, shuffle=True, random_state=SEED)
    lgb_oof = np.zeros(train.shape[0])
    lgb_pred = pd.DataFrame()

    train_x = train.loc[:][select_col_list]
    test_x = test.loc[:][select_col_list]

    train_x['speed'] = target_speed.astype('float')
    test_x['speed'] = mlp_pred
    
    target_y = target

    for fold, (trn_idx, val_idx) in enumerate(kf.split(X=train_x)):
        X_train, y_train = train_x.iloc[trn_idx], target_y[trn_idx]
        X_valid, y_valid = train_x.iloc[val_idx], target_y[val_idx]
        X_test = test_x

        # LightGBM
        model = lgb.LGBMClassifier(**lgb_params)
        model.fit(X_train, y_train,
                  eval_set=(X_valid, y_valid),
                  eval_metric='logloss',
                  verbose=False,
                  early_stopping_rounds=500
                  )

        lgb_oof[val_idx] = model.predict(X_valid)
        lgb_pred[f'fold_{fold}'] = model.predict(X_test)
        f1_macro = f1_score(y_valid, lgb_oof[val_idx], average='macro')
        print(f"fold {fold} lgb score: {f1_macro}")

    # 予測値の最頻値を求める（ご指摘をいただき修正）
    sub_pred = lgb_pred.mode(axis=1)[0]
    print("+-" * 40)
    print(f"score: {f1_macro}")
    
    return sub_pred

# 「ｙ」を予測するモデルのハイパーパラメータを設定
lgb_param = {
"bagging_fraction": 0.7537281209924886,
"bagging_freq": 5,
"feature_fraction": 0.7548131884427044,
"learning_rate": 0.00854494687558397,
"min_child_samples": 78,
"num_leaves": 209,
}

# 予測に使う特徴量を選択
select_col_list =['B', 'O', 'b1', 'b3', 'bottomTeam', 'topTeam', 'bgTop',
                  'month', 'dayofweek', 'total_stat', 'pitcherTeam',
                  'pitcherHand_ce', 'batter_ce', 'inning_num_ce',
                  'startTime_ce', 'pitcherHand_te', 'batter_te',
                  'inning_num_te', 'startTime_te', 'place_te',
                  'gameID_all_PCA=000', 'gameID_all_PCA=002',
                  'gameID_all_PCA=004', 'gameID_all_PCA=005',
                  'gameID_all_PCA=009', 'gameID_all_PCA=012',
                  'gameID_all_PCA=015', 'gameID_all_PCA=016',
                  'gameID_all_PCA=017', 'gameID_all_PCA=019',
                  'gameID_all_PCA=023', 'gameID_all_PCA=024',
                  'gameID_all_PCA=029', 'gameID_all_PCA=031',
                  'gameID_all_PCA=035', 'gameID_all_PCA=039',
                  'gameID_all_PCA=040', 'gameID_all_PCA=042',
                  'gameID_all_PCA=045', 'gameID_all_PCA=046',
                  'gameID_all_PCA=047', 'gameID_all_PCA=048',
                  'gameID_all_PCA=049', 'gameID_all_PCA=051',
                  'gameID_all_PCA=053', 'gameID_all_PCA=054',
                  'gameID_all_PCA=057', 'gameID_stat_PCA=000',
                  'gameID_stat_PCA=001', 'gameID_stat_PCA=003',
                  'gameID_stat_PCA=004', 'gameID_stat_PCA=005',
                  'gameID_stat_PCA=006', 'gameID_stat_PCA=008',
                  'gameID_stat_PCA=010', 'gameID_stat_PCA=012',
                  'gameID_stat_PCA=014', 'gameID_stat_PCA=015',
                  'gameID_base_PCA=001', 'gameID_base_PCA=005',
                  'gameID_base_PCA=007', 'gameID_base_PCA=008',
                  'gameID_base_PCA=009', 'gameID_base_PCA=011',
                  'gameID_base_PCA=012', 'gameID_base_PCA=013',
                  'gameID_base_PCA=014', 'gameID_base_PCA=015',
                  'gameID_inning_PCA=001', 'gameID_inning_PCA=002',
                  'gameID_inning_PCA=003', 'gameID_inning_PCA=004',
                  'gameID_inning_PCA=006', 'gameID_inning_PCA=008',
                  'gameID_inning_PCA=009', 'gameID_inning_PCA=010',
                  'gameID_inning_PCA=012', 'gameID_inning_PCA=013',
                  'gameID_inning_PCA=014']

#学習と予測の実行
sub_pred = pred_y_of_test_data(X_train,X_test,target,lgb_param,speed_pred,select_col_list)

# ------------------------------------------------------------------------------
# 提出ファイルの作成
# ------------------------------------------------------------------------------

#テスト結果の出力
submit_df = pd.DataFrame({'y': sub_pred.astype(int)})
submit_df.index.name = 'id'
submit_df.to_csv('../submission/submission.csv')