# Library
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns
from tqdm.auto import tqdm
import statsmodels
import statsmodels.api as sm

import lightgbm as lgb
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_pinball_loss
from lightgbm import LGBMRegressor

import warnings
warnings.simplefilter('ignore')

# mount
from google.colab import drive
if not os.path.isdir('/content/drive'):
    drive.mount('/content/drive')

# Config
DRIVE_PATH = "/content/drive/MyDrive/ML/PROBSPACE/weather_merchandising"
INPUT = os.path.join(DRIVE_PATH, "data")
OUTPUT = os.path.join(DRIVE_PATH, "output")

TRAIN_FILE = os.path.join(INPUT, "train_data.csv")
TEST_FILE = os.path.join(INPUT, "test_data.csv")
SUB_FILE = os.path.join(INPUT, "submission.csv")

seed =42

# plot style
pd.set_option('display.max_columns', 200)
plt.rcParams['axes.facecolor'] = 'EEFFFE'

# Data
train = pd.read_csv(TRAIN_FILE)
test = pd.read_csv(TEST_FILE)
sub = pd.read_csv(SUB_FILE)

# Target Columns
target_columns = ['ice1', 'ice2', 'ice3', 'oden1', 'oden2', 'oden3', 'oden4', 'hot1',
       'hot2', 'hot3', 'dessert1', 'dessert2', 'dessert3', 'dessert4',
       'dessert5', 'drink1', 'drink2', 'drink3', 'drink4', 'drink5', 'drink6',
       'alcol1', 'alcol2', 'alcol3', 'snack1', 'snack2', 'snack3', 'bento1',
       'bento2', 'bento3', 'bento4', 'tild1', 'tild2', 'men1', 'men2', 'men3',
       'men4', 'men5', 'men6']

def preprocessing(df, mode='train'):
        df_tmp = df.copy()
        input_year = 2018

        df_tmp['time'] = pd.to_datetime(df_tmp.date, format='%m/%d')
        df_tmp['year'] = df_tmp['time'].dt.year
        df_tmp['month'] = df_tmp['time'].dt.month
        df_tmp['day'] = df_tmp['time'].dt.day
        if mode=='train':
            df_tmp.loc[df_tmp['month']>3, 'year'] = input_year
            df_tmp.loc[df_tmp['month']<=3, 'year'] = input_year + 1
        else:
            df_tmp['year'] = input_year + 1
        df_tmp['time'] = pd.to_datetime({'year':df_tmp.year, 'month':df_tmp.month, 'day':df_tmp.day})
        df_tmp['weekday'] = df_tmp['time'].dt.weekday
        return df_tmp

train_df = preprocessing(train, mode='train')
test_df = preprocessing(test, mode='test')
all_df = pd.concat([train_df, test_df]).reset_index(drop=True)

display(train_df.head(3))
display(test_df.head(3))

plot_col = [c for c in train.columns if c not in ['id', 'date', 'time', 'year', 'month', 'day', 'weekday']]
ncols = len(plot_col) // 13
plt.subplots(14, ncols, sharey=True, sharex=True, figsize=(30, 80))
for i, col in enumerate(plot_col):
    plt.subplot(14, ncols, i+1)
    plt.plot(train_df.date, train_df[col], alpha=1, color='orange', label=col)
    for x in [20,51,81,112,143,173.204,234,265,296,324,350]: # beginning of month, last line is train/test split
        plt.axvline(x)
    plt.xlabel(col)
    plt.legend()
    plt.xticks([])
plt.show()

# アイスは夏の分布が大きく異なるものがある＞学習から除外
# おでんは7月から9月は提供していない＞学習から除外
# ホットは周期性と小さい下降トレンド
# アイスやドリンクは気温とかなり相関があるが、ドリンクは売り上げの下限値がありそう
# 横ばいのものにも曜日の周期性は見られる

train_df.groupby('weekday').mean()[target_columns].style.background_gradient()

# アイスは0で売れる
# おでんは2,3で売れない
# ホットは1,2,5,6で売れる
# デザートは2,6で売れる
# ドリンク１４５は0で、ドリンク２３は4で、ドリンク６は0,2,3で売れる
# アルコールは0から6にかけて売れ行きが増加
# スナックは3,4,0の順で売れている
# 弁当は1,2,5,6で大きく売れる
# チルドは2で大きく売れる
# 麺は1,2,5,6で大きく売れる

train_df.groupby('day').mean()[target_columns].style.background_gradient()
# 月初め、月末などに特徴があるのかもしれない

# Adversarial Validation
train = pd.read_csv(TRAIN_FILE)
test = pd.read_csv(TEST_FILE)
train = train.drop(target_columns, axis=1)

train = preprocessing(train, mode='train')
test = preprocessing(test, mode='test')

train['target'] = 0
test['target'] = 1
all_df = pd.concat([train, test], axis=0)
target = all_df['target'].values

train_, test_ = train_test_split(all_df, test_size=0.33, random_state=42, shuffle=True)
train_x = train_.drop(['id', 'date', 'time', 'year', 'month', 'target'],axis=1)
test_x = test_.drop(['id', 'date', 'time', 'year', 'month', 'target'],axis=1)
train_y = train_['target'].values
test_y = test_['target'].values

lgb_train = lgb.Dataset(train_x, label=train_y)
lgb_test = lgb.Dataset(test_x, label=test_y)

param = {'metric': 'auc',
         'seed': seed,
         'verbosity': -1}

num_round = 100
clf = lgb.train(param, lgb_train, num_round, valid_sets = [lgb_test], verbose_eval=50, early_stopping_rounds = 50)

feature_imp = pd.DataFrame(sorted(zip(clf.feature_importance(), train_x.columns)), columns=['Value','Feature'])
plt.figure(figsize=(6, 5))
sns.barplot(x="Value", y="Feature", data=feature_imp.sort_values(by="Value", ascending=False).head(500))
plt.title('LightGBM Features')
plt.tight_layout()
plt.show()

display(feature_imp.sort_values(by="Value", ascending=False))
print(np.sort(clf.predict(train_x))[::-1][:30])
print(np.argsort(clf.predict(train_x))[::-1][:30])

# 似ている期間も特になし

# Data
train = pd.read_csv(TRAIN_FILE)
test = pd.read_csv(TEST_FILE)
sub = pd.read_csv(SUB_FILE)

def preprocessing(df, mode='train'):
    df_tmp = df.copy()
    input_year = 2018

    df_tmp['time'] = pd.to_datetime(df_tmp.date, format='%m/%d')
    df_tmp['year'] = df_tmp['time'].dt.year
    df_tmp['month'] = df_tmp['time'].dt.month
    df_tmp['day'] = df_tmp['time'].dt.day
    if mode=='train':
        df_tmp.loc[df_tmp['month']>3, 'year'] = input_year
        df_tmp.loc[df_tmp['month']<=3, 'year'] = input_year + 1
    else:
        df_tmp['year'] = input_year + 1
    df_tmp['time'] = pd.to_datetime({'year':df_tmp.year, 'month':df_tmp.month, 'day':df_tmp.day})
    df_tmp['weekday'] = df_tmp['time'].dt.weekday
    return df_tmp

train = preprocessing(train, mode='train')
test = preprocessing(test, mode='test')

train.columns

# 検証データのindexを指定
valid_index = range(297,351) # month:2,3

# 予測結果を保存する辞書型データを作成
results = dict({})
all_lgb_score = []
# 商品毎の予測を作成する
for c in tqdm(target_columns):
    # 商品特有の性質を反映させた特徴量を生成
    train_tmp = train.copy()
    test_tmp = test.copy()
    # ice
    if c in target_columns[0:3]:
        train_tmp = train_tmp[~train_tmp['month'].isin([7,8,9])]
        train_tmp['is_wday0'] = train['weekday'].isin([0]).astype(int)
        test_tmp['is_wday0'] = test['weekday'].isin([0]).astype(int)
    # oden
    elif c in target_columns[3:7]:
        train_tmp = train_tmp[(train_tmp['month'].isin([10,11,12,1,2,3]))|(train_tmp['id'].isin(valid_index))]
        train_tmp['is_wday23'] = train['weekday'].isin([2,3]).astype(int)
        test_tmp['is_wday23'] = test['weekday'].isin([2,3]).astype(int)
    # hot
    elif c in target_columns[7:10]:
        train_tmp['is_wday034'] = train['weekday'].isin([0,3,4]).astype(int)
        test_tmp['is_wday034'] = test['weekday'].isin([0,3,4]).astype(int)
    # dessert
    elif c in target_columns[10:15]:
        train_tmp['is_wday26'] = train['weekday'].isin([2,6]).astype(int)
        test_tmp['is_wday26'] = test['weekday'].isin([2,6]).astype(int)
    # drink145
    elif c in [target_columns[15],target_columns[18],target_columns[19]]:
        train_tmp['is_wday0'] = train['weekday'].isin([0]).astype(int)
        test_tmp['is_wday0'] = test['weekday'].isin([0]).astype(int)
    # drink23
    elif c in target_columns[16:18]:
        train_tmp['is_wday4'] = train['weekday'].isin([4]).astype(int)
        test_tmp['is_wday4'] = test['weekday'].isin([4]).astype(int)
    # snack
    elif c in target_columns[24:27]:
        train_tmp['is_wday4'] = train['weekday'].isin([4]).astype(int)
        train_tmp['is_wday15'] = train['weekday'].isin([1,5]).astype(int)
        test_tmp['is_wday4'] = test['weekday'].isin([4]).astype(int)
        test_tmp['is_wday15'] = test['weekday'].isin([1,5]).astype(int)
    # bento
    elif c in target_columns[27:31]:
        train_tmp['is_wday034'] = train['weekday'].isin([0,3,4]).astype(int)
        test_tmp['is_wday034'] = test['weekday'].isin([0,3,4]).astype(int)
    # tild
    elif c in target_columns[31:33]:
        train_tmp['is_wday2'] = train['weekday'].isin([2]).astype(int)
        test_tmp['is_wday2'] = test['weekday'].isin([2]).astype(int)
    # men
    elif c in target_columns[33:39]:
        train_tmp['is_wday034'] = train['weekday'].isin([0,3,4]).astype(int)
        test_tmp['is_wday034'] = test['weekday'].isin([0,3,4]).astype(int)

    train_columns = [c for c in train_tmp.columns if c not in target_columns if c not in ['date', 'time']]

    x_train = train_tmp[~train_tmp['id'].isin(valid_index)][train_columns]
    y_train = train_tmp[~train_tmp['id'].isin(valid_index)][c]
    x_valid = train_tmp[train_tmp['id'].isin(valid_index)][train_columns]
    y_valid = train_tmp[train_tmp['id'].isin(valid_index)][c]
    x_test = test_tmp[train_columns]

    x_train = x_train.drop(['id'],axis=1)
    x_valid = x_valid.drop(['id'],axis=1) 
    x_test = x_test.drop(['id'],axis=1)

    # 分位点を設定
    qs = np.array([0.01, 0.1, 0.5, 0.9, 0.99])
    
    lgb_scores = []
    # 分位点毎に予測を作成する
    for q in qs:
        # モデルのインスタンスを作成
        lgb = LGBMRegressor(
            objective='quantile',
            alpha=q,
            n_estimators=10000,
            max_depth=2,
            colsample_bytree=0.9,
            random_state=seed)
        #学習を実施
        lgb.fit(x_train, y_train, eval_set=(x_valid, y_valid), early_stopping_rounds=100, verbose=False)
        #予測を実施
        lgb_scores.append(lgb.best_score_['valid_0']['quantile'])  # validationのbest_score
        pred_y = lgb.predict(x_test)
        #予測結果を格納
        results[(c, q)] = pred_y

    quantiles = [0.01, 0.1, 0.5, 0.9, 0.99]
    lgb_scores = []
    for i, q in enumerate(quantiles):
        lgb = LGBMRegressor(
            objective='quantile',
            alpha=q,
            n_estimators=10000,
            max_depth=2,
            colsample_bytree=0.9,
            random_state=seed)
        lgb.fit(x_train, y_train, eval_set=(x_valid, y_valid), early_stopping_rounds=100, verbose=False)
        lgb_scores.append(lgb.best_score_['valid_0']['quantile'])  # validationのbest_score
    all_lgb_score.append(lgb_scores)
# Score
print(np.array(all_lgb_score).mean())

# 商品_分位点毎のリストに変換
submit_rows = [[f'{k[0]}_{k[1]}']+ v.tolist() for k, v in results.items()]

# テスト結果の出力
submit_df = pd.DataFrame(np.array(submit_rows)[:, 1:22].astype(float).round(), index=np.array(submit_rows)[:, 0])
submit_df.columns = list(range(1, 22))
submit_df = submit_df.transpose()
submit_df.index.name = 'id'
submit_df.to_csv(os.path.join(OUTPUT, "sub_exp00.csv"))
display(submit_df.head())

pred_median_col = [c for c in submit_df.columns if '_0.5' in c]
test_add_pred = test.merge(submit_df[pred_median_col].reset_index(), on='id', how='left')
test_add_pred.columns = [c.replace('_0.5', '') if '_0.5' in c else c for c in test_add_pred.columns]
test_add_pred[target_columns] = test_add_pred[target_columns].astype(float)
test_add_pred.head(3)

# 移動平均
window=7
train_rolling = train.rolling(window, min_periods=1).mean()
test_rolling = test_add_pred.rolling(window, min_periods=1).mean()
all_rolling = pd.concat([train, test_add_pred], axis=0).reset_index(drop=True).rolling(window, min_periods=1).mean()

# 目視確認
plot_col = [c for c in train.columns if c not in ['id', 'date', 'time', 'year', 'month', 'day', 'weekday']]
ncols = len(plot_col) // 13
plt.subplots(14, ncols, sharey=True, sharex=True, figsize=(30, 80))
for i, col in enumerate(plot_col):
    plt.subplot(14, ncols, i+1)
    plt.plot(train_rolling.index[window:], train_rolling[col][window:], alpha=1, color='blue', label='train')
    plt.plot(all_rolling.index[-len(test_rolling):], all_rolling[col][-len(test_rolling):], alpha=1, color='red', label='test')
    for x in [20,51,81,112,143,173.204,234,265,296,324,350]:
        plt.axvline(x)
    plt.xlabel(col)
    plt.legend()
    plt.xticks([])
plt.show()