import pandas as pd
pd.options.display.max_columns = 100
pd.options.display.max_rows = 999
pd.options.display.float_format = '{:.6f}'.format
import numpy as np

import datetime

import warnings
warnings.filterwarnings('ignore')

%matplotlib inline
# %matplotlib notebook
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import cm

from google.colab import drive
drive.mount('/content/drive')

path = "/content/drive/MyDrive/tmp/7_probspace/3_taxi/"

weather = pd.read_csv(path+'data/nyc_weather_2017_2019.csv', parse_dates=["DATE"]).rename(columns={'DATE':'ds'})
train = pd.read_csv(path+'data/train_data.csv', parse_dates=["tpep_pickup_datetime"]).rename(columns={'tpep_pickup_datetime':'ds'})

train[train["ds"]>="2019-11-30"].head(5)

weather[weather["ds"]<="2019-11-30 02:00:00"].tail(5)

# 縦持ち
data_df = train.copy()
data_df = pd.melt(
    data_df, id_vars=["ds"], value_vars=list(map(str, range(79))),
    var_name="id", value_name="y"
)
data_df.loc[:,"id"] = data_df["id"].astype("int64")

# お天気情報merge用の列追加
data_df.loc[:,"ds_h"] = pd.to_datetime(data_df["ds"].dt.strftime('%Y-%m-%d %H')+":00:00")
data_df

weather.loc[:,"year"] = weather["ds"].dt.year
weather.loc[:,"month"] = weather["ds"].dt.month
weather.loc[:,"day"] = weather["ds"].dt.day
weather.loc[:,"hour"] = weather["ds"].dt.hour
weather.loc[:,"minute"] = weather["ds"].dt.minute

weather.loc[:,"ds_h"] = pd.to_datetime(
    weather["year"].astype(str) +"-"+ weather["month"].astype(str).str.zfill(2) +"-"+ weather["day"].astype(str).str.zfill(2) +
    " "+ weather["hour"].astype(str).str.zfill(2) + ":00:00"
)
# 全部一時間後に参照できるようにする
weather.loc[:,"ds_h"] = weather["ds_h"] + datetime.timedelta(hours=+1)
weather[["ds","year","month","day","hour","minute","ds_h"]].head(10)

col_weather = [
  "HourlyDewPointTemperature","HourlyDryBulbTemperature","HourlyPrecipitation","HourlyPressureChange","HourlyPressureTendency","HourlyRelativeHumidity",
  "HourlySkyConditions","HourlySeaLevelPressure","HourlyStationPressure","HourlyVisibility","HourlyWetBulbTemperature","HourlyWindDirection","HourlyWindGustSpeed","HourlyWindSpeed",
]
# 強引にfloat変換（変換できないところはNA）
for c in col_weather:
  weather.loc[:,c] = pd.to_numeric(weather[c], errors="coerce")

# 時間ごとに平均
weather_ = pd.DataFrame({
  "ds_h":pd.date_range(start='2017-01-01 00:00:00', end='2019-12-07 23:00:00', freq='60T')
})
weather_ = pd.merge(
    weather_,
    weather[["ds_h"]+col_weather].groupby(["ds_h"]).mean().fillna(0).reset_index(),
    how="left", on="ds_h"
).set_index("ds_h")
# NAは線形補完
weather_ = weather_.interpolate(method='linear', axis=0)
weather_ = weather_.reset_index()
weather_.head(5)
# 一番最初のデータがなくなったけど、まぁいいか

data_df_w = pd.merge(data_df, weather_, how="left", on="ds_h")
data_df_w.head(5)

# なんとなくこの期間を可視化
plot_d = data_df_w[(data_df_w["ds"]>="2018-11-08")&(data_df_w["ds"]<="2018-11-20")]

fig, ax = plt.subplots(len(col_weather), 1, figsize=(20, len(col_weather)*4))

i = 31 # なんとなく31地点を可視化（空港？）
for j, col in enumerate(col_weather):
  ax2 = ax[j].twinx()
  ax[j].plot(plot_d[plot_d["id"]==i]['ds'], plot_d[plot_d["id"]==i]["y"], color=cm.tab20.colors[0])
  ax2.plot(plot_d[plot_d["id"]==i]['ds'], plot_d[plot_d["id"]==i][col], color=cm.tab20.colors[j+1])
  ax[j].set_title(str(i)+"_"+col)

  # 色変更
  ax2.spines['left'].set_color(cm.tab20.colors[0])
  ax2.spines['right'].set_color(cm.tab20.colors[j+1])
  ax[j].tick_params(axis='y', colors=cm.tab20.colors[0])
  ax2.tick_params(axis='y', colors=cm.tab20.colors[j+1])

plt.show()
plt.close()
plt.clf()