# Library
import os
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
!pip install japanize_matplotlib
import japanize_matplotlib
%matplotlib inline
import seaborn as sns
from tqdm.auto import tqdm

import warnings
warnings.simplefilter('ignore')

# mount
from google.colab import drive
if not os.path.isdir('/content/drive'):
    drive.mount('/content/drive')

# Config
DRIVE_PATH = "/content/drive/MyDrive/ML/PROBSPACE/pollen_counts"
INPUT = os.path.join(DRIVE_PATH, "input")
OUTPUT = os.path.join(DRIVE_PATH, "output")

TRAIN_FILE = os.path.join(INPUT, "train.csv")
TEST_FILE = os.path.join(INPUT, "test.csv")
SUB_FILE = os.path.join(INPUT, "submission.csv")

seed =42

# plot style
pd.set_option('display.max_columns', 200)
plt.rcParams['axes.facecolor'] = 'EEFFFE'

# Data
train = pd.read_csv(TRAIN_FILE)
test = pd.read_csv(TEST_FILE)
sub = pd.read_csv(SUB_FILE)

display(train.head(3))
display(test.head(3))
display(sub.head(3))

train.info()

test.info()

train.select_dtypes(object)

train.precipitation_tokyo.value_counts()

for col in train.select_dtypes(object).columns:
    print(train[train[col].isin(['欠測'])].datetime)

# object(欠測) -> float
import lightgbm as lgb
from lightgbm import LGBMRegressor
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer

train_df = train.replace('欠測', np.nan)
lgb_imp = IterativeImputer(
                       estimator=LGBMRegressor(num_boost_round=100, random_state=seed),
                       max_iter=10, 
                       initial_strategy='mean',
                       imputation_order='ascending',
                       verbose=1,
                       random_state=seed)

train_df = pd.DataFrame(lgb_imp.fit_transform(train_df), columns=train_df.columns)
train_df[['winddirection_chiba', 'winddirection_tokyo']] = train_df[['winddirection_chiba', 'winddirection_tokyo']].round().astype(int)
train_df[['precipitation_tokyo', 'temperature_chiba', 'temperature_tokyo', 'windspeed_chiba', 'windspeed_tokyo']] = train_df[['precipitation_tokyo', 'temperature_chiba', 'temperature_tokyo', 'windspeed_chiba', 'windspeed_tokyo']].round(1)
train[train.select_dtypes(object).columns] = train_df[train.select_dtypes(object).columns]
train

df = pd.concat([train, test]).reset_index(drop=True)
df['time'] = pd.to_datetime(df.datetime.astype(str).str[:-2])
df['year'] = df['time'].dt.year
df['month'] = df['time'].dt.month
df['day'] = df['time'].dt.day
df['hour'] = df.datetime.astype(str).str[-2:].astype(int)
df['weekday'] = df['time'].dt.weekday
df.head(3)

plot_col = ['pollen_utsunomiya', 'pollen_chiba', 'pollen_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, col in enumerate(plot_col):
    plt.subplot(1, ncols, i+1)
    plt.plot(df.time, df[col], alpha=1, color=color[i], label=col)
    plt.xlabel(col)
    plt.legend()
    plt.grid()
plt.show()

df[df.month.isin([7,8,9,10,11,12,1])]

target_col = ['pollen_utsunomiya', 'pollen_chiba', 'pollen_tokyo']
for col in target_col:
    print('='*10+col+'='*10)
    print(df[col].describe())
    print('<75%より多く花粉を含む(月)>')
    df[df[col] > df[col].quantile(0.75)].month.value_counts().plot(kind='barh', figsize=(5,3))
    plt.show()
    print('<99%より多く花粉を含む(月)>')
    df[df[col] > df[col].quantile(0.99)].month.value_counts().plot(kind='barh', figsize=(5,3))
    plt.show()
    print()

plot_col = ['pollen_utsunomiya', 'pollen_chiba', 'pollen_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, col in enumerate(plot_col):
    plt.subplot(1, ncols, i+1)
    plt.hist(df[col], bins=200, alpha=1, color=color[i], label=col)
    plt.xlabel(col)
    plt.legend()
    plt.grid()
    plt.xlim(-1,500)
plt.show()

df[df.month.isin([2,3])].groupby('year').mean()[target_col].style.background_gradient()

df[df.datetime <= 2020033124].groupby('month').mean()[target_col].style.background_gradient()

df[df.datetime <= 2020033124].groupby('hour').mean()[target_col].style.background_gradient()

df[df.datetime <= 2020033124].groupby('weekday').mean()[target_col].style.background_gradient()

fig, ax = plt.subplots(figsize=(12, 10)) 
sns.heatmap(df[df.datetime <= 2020033124].drop(['datetime', 'time'], axis=1).corr(), vmax=1, vmin=-1, center=0)

plot_col = ['precipitation_utsunomiya', 'precipitation_chiba', 'precipitation_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, col in enumerate(plot_col):
    plt.subplot(1, ncols, i+1)
    for y in df.year.unique():
        plt.plot(df[df.year==y].time, df[df.year==y][col], alpha=1, color=color[i])
    plt.xlabel(col)
    plt.grid()
plt.show()

plot_col = ['precipitation_utsunomiya', 'precipitation_chiba', 'precipitation_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, (tcol, col) in enumerate(zip(target_col, plot_col)):
    plt.subplot(1, ncols, i+1)
    plt.scatter(df[col], df[tcol], alpha=1, color=color[i])
    plt.xlabel(col)
    plt.ylabel(tcol)
    plt.grid()
    plt.xlim(-1,11)
plt.show()

plot_col = ['temperature_utsunomiya', 'temperature_chiba', 'temperature_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, col in enumerate(plot_col):
    plt.subplot(1, ncols, i+1)
    for j, y in enumerate(df.year.unique()):
        plt.plot(df[df.year==y].index, df[df.year==y][col], alpha=0.25*(j+1), color=color[i])
        plt.plot(df[df.year==y].index, df[df.year==y][col].rolling(100).mean(), alpha=0.25*(j+1), color='black')
    plt.xlabel(col)
    plt.grid()
plt.show()

plot_col = ['temperature_utsunomiya', 'temperature_chiba', 'temperature_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, (tcol, col) in enumerate(zip(target_col, plot_col)):
    plt.subplot(1, ncols, i+1)
    plt.scatter(df[col], df[tcol], alpha=1, color=color[i])
    plt.xlabel(col)
    plt.ylabel(tcol)
    plt.grid()
plt.show()

# 指定地域の該当日の風向
winddirection = {
    0:'静穏',
    1:'北北東',
    2:'北東',
    3:'東北東',
    4:'東',
    5:'東南東',
    6:'南東',
    7:'南南東',
    8:'南',
    9:'南南西',
    10:'南西',
    11:'西南西',
    12:'西',
    13:'西北西',
    14:'北西',
    15:'北北西',
    16:'北',
}

df_wd = df.copy()
for col in ['winddirection_utsunomiya', 'winddirection_chiba', 'winddirection_tokyo']:
    df_wd[col] = df_wd[col].map(winddirection)
df_wd.head(3)

plot_col = ['winddirection_utsunomiya', 'winddirection_chiba', 'winddirection_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, (tcol, col) in enumerate(zip(target_col, plot_col)):
    plt.subplot(1, ncols, i+1)
    df_wd[col].value_counts().plot(kind='barh')
    plt.xlabel(tcol)
    plt.ylabel(col)
    plt.grid()
plt.show()

plot_col = ['winddirection_utsunomiya', 'winddirection_chiba', 'winddirection_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, (tcol, col) in enumerate(zip(target_col, plot_col)):
    plt.subplot(1, ncols, i+1)
    df_wd[df_wd.datetime <= 2020033124].groupby(col).mean()[tcol].plot(kind='barh')
    plt.xlabel(tcol)
    plt.ylabel(col)
    plt.grid()
plt.show()

plot_col = ['windspeed_utsunomiya', 'windspeed_chiba', 'windspeed_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, col in enumerate(plot_col):
    plt.subplot(1, ncols, i+1)
    for j, y in enumerate(df.year.unique()):
        plt.plot(df[df.year==y].index, df[df.year==y][col], alpha=0.25*(j+1), color=color[i])
        plt.plot(df[df.year==y].index, df[df.year==y][col].rolling(100).mean(), alpha=0.25*(j+1), color='black')
    plt.xlabel(col)
    plt.grid()
plt.show()

plot_col = ['windspeed_utsunomiya', 'windspeed_chiba', 'windspeed_tokyo']
color = ['red','green','blue']
ncols = len(plot_col)
plt.subplots(1, ncols, sharey=True, sharex=True, figsize=(30, 5))
for i, (tcol, col) in enumerate(zip(target_col, plot_col)):
    plt.subplot(1, ncols, i+1)
    plt.scatter(df[col], df[tcol], alpha=1, color=color[i])
    plt.xlabel(col)
    plt.ylabel(tcol)
    plt.grid()
plt.show()

!pip install folium

import folium

cities = pd.DataFrame({
    'train': ['宇都宮（宇都宮市中央生涯学習センター）', '千葉（千葉県環境研究センター）', '府中（東京都多摩小平保健所）'],
    'latitude': [36.559444, 35.525312, 35.730062],
    'longtude': [139.882689, 140.068465, 139.516689],
})
map = folium.Map(width='75%', height='75%',location=[36,140], zoom_start=8)
 
for i, r in cities.iterrows():
    folium.Marker(location=[r['latitude'], r['longtude']], popup=r['train']).add_to(map)
map