import gc
import datetime

#環境確認
import pandas as pd
import numpy as np
#import statsmodels
!python3 --version
print(pd.__version__)
print(np.__version__)
#print(statsmodels.__version__)
import matplotlib
print(matplotlib.__version__)
import matplotlib.pyplot as plt

from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler
import sklearn

from sklearn.model_selection import train_test_split
import tensorflow as tf
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense, LSTM, Dropout, Bidirectional, Flatten, BatchNormalization
from tensorflow.keras.utils import plot_model
from keras.callbacks import EarlyStopping
from tensorflow.keras import Input, Model

print(sklearn.__version__)
print(tf.__version__)

# google colab を使う場合

#from google.colab import drive
#drive.mount('/content/drive')

#ニューヨークの祝祭日を設定

holydays_data =( [
datetime.date(2017, 1, 1),  # "New Year's Day"
datetime.date(2017, 1, 16), # 'Martin Luther King Jr. Day'
datetime.date(2017, 2, 13), # "Lincoln's Birthday "
datetime.date(2017, 2, 20), # "Washington's Birthday"
datetime.date(2017, 5, 29), # 'Memorial Day'
datetime.date(2017, 7, 4),  # 'Independence Day'
datetime.date(2017, 9, 4),  # 'Labor Day'
datetime.date(2017, 10, 9), # 'Columbus Day'
datetime.date(2017, 11, 7), # 'Election Day'
datetime.date(2017, 11, 11),# 'Veterans Day'
datetime.date(2017, 11, 23),# 'Thanksgiving'
datetime.date(2017, 12, 25),# 'Christmas Day'
datetime.date(2018, 1, 1),  # "New Year's Day"
datetime.date(2018, 1, 15), # 'Martin Luther King Jr. Day'
datetime.date(2018, 2, 12), # "Lincoln's Birthday "
datetime.date(2018, 2, 19), # "Washington's Birthday"
datetime.date(2018, 5, 28), # 'Memorial Day'
datetime.date(2018, 7, 4),  # 'Independence Day'
datetime.date(2018, 9, 3),  # 'Labor Day'
datetime.date(2018, 10, 8), # 'Columbus Day'
datetime.date(2018, 11, 6), # 'Election Day'
datetime.date(2018, 11, 12),# 'Veterans Day'
datetime.date(2018, 11, 22),# 'Thanksgiving'
datetime.date(2018, 12, 25),# 'Christmas Day'
datetime.date(2019, 1, 1),  # "New Year's Day"
datetime.date(2019, 1, 21), # 'Martin Luther King Jr. Day'
datetime.date(2019, 2, 12), # "Lincoln's Birthday "
datetime.date(2019, 2, 18), # "Washington's Birthday"
datetime.date(2019, 5, 27), # 'Memorial Day'
datetime.date(2019, 7, 4),  # 'Independence Day'
datetime.date(2019, 9, 2),  # 'Labor Day'
datetime.date(2019, 10, 14),# 'Columbus Day'
datetime.date(2019, 11, 5), # 'Election Day'
datetime.date(2019, 11, 11),# 'Veterans Day'
datetime.date(2019, 11, 28),# 'Thanksgiving'
datetime.date(2019, 12, 25) # 'Christmas Day'
])

# GPUが認識されていることを確認
physical_devices = tf.config.list_physical_devices('GPU')
if len(physical_devices) > 0:
    for device in physical_devices:
        tf.config.experimental.set_memory_growth(device, True)
        print('{} memory growth: {}'.format(device, tf.config.experimental.get_memory_growth(device)))
else:
    print("Not enough GPU hardware devices available")

#データの読み込みと前処理
import pandas as pd

path = "/home/kodama3104/ws_011/data/" #データを置くフォルダ ( train , zone , test , weather  )
#path = "/content/drive/MyDrive/data/" #データを置くフォルダ

train_data = pd.read_csv(path + "train_data.csv", index_col='tpep_pickup_datetime')
weather_data = pd.read_csv(path + "nyc_weather_2017_2019.csv")
zone_data = pd.read_csv(path + "taxi_zones.csv")
print(train_data.shape)
print(weather_data.shape)
print(zone_data.shape)

#日付をdatetime型に変換

train_data.index = pd.to_datetime(train_data.index)
weather_data['DATE'] = pd.to_datetime(weather_data['DATE'])

x_col = train_data.columns
x_num = train_data.shape[1]

weather_data = weather_data.set_index('DATE')
weather_data = weather_data[weather_data['REPORT_TYPE']=='FM-15']

# 気象データの時間間隔を、タクシー乗車数のデータと時間間隔をそろえる

weather_30min = weather_data.resample('30min').nearest()
weather_30min.loc['2017-01-01 00:00:00'] = weather_30min.iloc[0]
weather_30min.index = pd.to_datetime(weather_30min.index)
weather_30min = weather_30min.sort_index(ascending=True)
print(weather_data.shape)
print(weather_30min.shape)

weather_30min = weather_30min.fillna(method = 'ffill')
train_data    = train_data.fillna(method = 'ffill')

# 気象データの中から影響の少なそうな変数を削除
weather_30min = weather_30min.drop(['REPORT_TYPE','SOURCE','HourlyPresentWeatherType',
                                    'HourlySkyConditions','HourlyWindGustSpeed','REM','HourlyDryBulbTemperature',
                                    'HourlyAltimeterSetting','HourlyPressureChange','HourlyPressureTendency',
                                    'HourlySeaLevelPressure','HourlyDewPointTemperature','HourlyWindDirection',
                                    'HourlyStationPressure','HourlyVisibility'],axis=1)

# 変数に、月、曜日、日付、時間、分を加える

weather_30min['month'] = weather_30min.index.month
weather_30min['week'] = weather_30min.index.weekday
weather_30min['day'] = weather_30min.index.day
weather_30min['hour'] = weather_30min.index.hour
weather_30min['minute'] = weather_30min.index.minute

train_data['month'] = train_data.index.month
train_data['week'] = train_data.index.weekday
train_data['day'] = train_data.index.day
train_data['hour'] = train_data.index.hour
train_data['minute'] = train_data.index.hour

# 休日の列を追加して、何の休日かを１－１３の数字で入れる

holydays_flag = []
holydays_count = 0
for i in range( len(weather_30min)):
    if(datetime.date( weather_30min.index[i].year, weather_30min.index[i].month, weather_30min.index[i].day ) in holydays_data):
        holydays_count += 1
        if(holydays_count == 13 ):
          holydays_count = 1
        holydays_flag.append(holydays_count)
    else:
        holydays_flag.append(0)

weather_30min['holydays'] = holydays_flag

holydays_flag = []
holydays_count = 0
for i in range( len(train_data)):
    if(datetime.date( train_data.index[i].year, train_data.index[i].month, train_data.index[i].day ) in holydays_data):
        holydays_count += 1
        if(holydays_count == 13 ):
          holydays_count = 1
        holydays_flag.append(holydays_count)
    else:
        holydays_flag.append(0)
train_data['holydays']    = holydays_flag

#for i in range(train_data.shape[1]):
#    plt.figure(figsize=(18,2))
#    plt.plot(train_data.iloc[:train_data.shape[0], i])
#    plt.show()
#    plt.clf()

# 気象データの中の非数値を、数値に置き換える

for elem in weather_30min.select_dtypes(include=object).columns :
    print(elem)
    weather_30min[elem] = weather_30min[elem].str.replace('s', '0')
    weather_30min[elem] = weather_30min[elem].str.replace('V', '0')
    weather_30min[elem] = weather_30min[elem].str.replace('RB', '0')
    weather_30min[elem] = weather_30min[elem].str.replace('T', '0.001')

weather_30min = weather_30min.astype('float32')

#for i in range(weather_30min.shape[1]):
#    plt.figure(figsize=(18,2))
#    plt.plot(weather_30min.iloc[:48*92, i])
#    plt.show()
#    plt.clf()

weather_30min.head(1)

weather_30min.tail(1)

lag_max = int(48*7)      # 過去何日分のデータを使って予測するか
predict_len = int(48*7)  # 予測日数

# 気象データを、学習機関(train)、予測対象期間(future)、テスト期間（test)に分ける
wea_30_train  = weather_30min[:-predict_len]
wea_30_future = weather_30min[lag_max + predict_len-1:-predict_len]
wea_30_test   = weather_30min[-predict_len:]
wea_30_train  = wea_30_train.astype('float32')
wea_30_future = wea_30_future.astype('float32')
wea_30_test   = wea_30_test.astype('float32')

print(train_data.shape)
print(wea_30_train.shape)
print(wea_30_future.shape)
print(wea_30_test.shape)

# タクシー乗車数データと気象データを結合

df_train = pd.concat([train_data, wea_30_train['HourlyPrecipitation'],wea_30_train['HourlyRelativeHumidity']
                       ,wea_30_train['HourlyWindSpeed'],wea_30_train['HourlyWetBulbTemperature']
                      ],axis='columns', ignore_index=True)

# df_train = pd.concat([train_data, wea_30_train],axis='columns', ignore_index=True)

df_train = df_train.astype('float32')

df_train.head(1)

df_train.tail(1)

df_train.shape

# 学習用の説明変数、目的変数、予測用の説明変数の生成
def gen_dataset(dataset, lag_max, predict_len):
    X, y = [], []
    test_X = []

    for i in range(len(dataset) - lag_max - predict_len + 1):
        a = i + lag_max
        X.append(dataset[i:a, :]) #説明変数
        y.append(dataset[a + predict_len - 1, :])   #目的変数

    for i in range(len(dataset),len(dataset)+predict_len):
        end = i - predict_len + 1
        start = end - lag_max
        test_X.append(dataset[start:end, :])
    return np.array(X), np.array(y), np.array(test_X)

X_train, y_train, X_test = gen_dataset(df_train.values, lag_max, predict_len)
print(X_train.shape)
print(y_train.shape)
print(X_test.shape)

X_train = X_train.astype(np.float32)
y_train = y_train.astype(np.float32)
X_test  = X_test.astype(np.float32)

del df_train,  wea_30_train, weather_30min
gc.collect()

y_train = y_train[:,:x_num]

validation_len = 48*1

X_valid = X_train[-validation_len:]
X_train = X_train[:-validation_len]

w_valid = wea_30_future[-validation_len:]
w_train = wea_30_future[:-validation_len]

y_valid = y_train[-validation_len:]
y_train = y_train[:-validation_len]

print(X_train.shape, w_train.shape)
print(X_valid.shape, w_valid.shape)

print(y_train.shape)
print(y_valid.shape)

# モデルの各パラメータ

# 隠れ層のノード数
hidden_num1 = 512

# 入力データ
input1 = Input(( X_train.shape[1], X_train.shape[2])) # タクシー乗車数データと気象データを結合したデータ
input2 = Input(  w_train.shape[1] ) # 予測対象時刻の気象データ

# モデル
x  = BatchNormalization()(input1)
x  = LSTM(hidden_num1, return_sequences = True)(x)
x  = BatchNormalization()(x)
x  = LSTM(hidden_num1, return_sequences = True)(x)
x  = BatchNormalization()(x)
x  = LSTM(hidden_num1, return_sequences = True)(x)
x  = BatchNormalization()(x)
x  = LSTM(hidden_num1, return_sequences = True)(x)

x1 = Flatten()(x)
x2 = input2 # 予測対象時刻の気象データ
x3 = tf.keras.layers.Concatenate(axis=1)([x1, x2])
x3 = Dense(128, activation="relu")(x3)
x3 = Dense(128, activation="relu")(x3)

# 出力層
outputs = Dense(y_train.shape[1], activation="linear")(x3)  #全結合層

model = Model(inputs=[input1,input2], outputs=[outputs])
model.summary()

# 損失関数：MSE， 最適化関数：Adam
opt = tf.keras.optimizers.Adam( learning_rate=0.0001)
model.compile(loss="mse", optimizer=opt)

# Lossをグラフ表示
def history_graph(history):
    loss = history.history['loss']
    val_loss = history.history["val_loss"]
    plt.plot(loss, label='train')
    plt.plot(val_loss, label='validation')
    plt.ylabel('loss')
    plt.xlabel('epochs')
    plt.show()

# 学習
early_stopping =  EarlyStopping(monitor='val_loss', min_delta=0.0, patience=10)

history = model.fit([X_train, w_train] , y_train,
                    validation_data = ([X_valid, w_valid], y_valid),
                    epochs = 300,
                    callbacks=[early_stopping],
                    batch_size = 64)

# Lossをグラフ表示
history_graph(history)

# 予測
def predict_test(model, data1, data2):
    dt = pd.date_range(start='2019-12-01 00:00:00', end='2019-12-07 23:30:00',freq='30T')
    pred = model.predict([data1, data2])

    df_pred = pd.DataFrame(pred, columns=x_col)
    df_pred['tpep_pickup_datetime'] = dt
    rnn_pred = df_pred.set_index("tpep_pickup_datetime")
    return rnn_pred

predict_df = predict_test(model, X_test, wea_30_test)

# 予測結果の表示
predict_df.head(5)

submit_df = predict_df
submit_df = submit_df.where(submit_df > 0.0, 0.0)

submit_df = submit_df.round()

submit_df.to_csv('./submit/submission_LSTM_BEST.csv')
#submit_df.to_csv('/content/drive/MyDrive/submit/submission_180.csv')

for i in range(submit_df.shape[1]):
    plt.figure(figsize=(18,2))
    plt.plot(train_data.iloc[-48*21:, i])
    plt.plot(submit_df.iloc[:, i])
    plt.show()
    plt.clf()