!python3 -m pip install -q slackweb
!echo "$(pip freeze | grep slackweb) is successfully installed"

import pandas as pd
import numpy as np
import datetime
import os
import warnings
warnings.filterwarnings("ignore")

import matplotlib
import matplotlib.pylab as plt

import sklearn
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_absolute_error
from sklearn import preprocessing

import lightgbm as lgb

import slackweb

from google.colab import drive
drive.mount("/content/drive")

SEED = 42

DRIVE = "/content/drive/My Drive/ML/kiva/"
INPUT = os.path.join(DRIVE, "input")
SUBMIT = os.path.join(DRIVE, "submit")

train_df = pd.read_csv(f"{INPUT}/train.csv")
display(train_df.info())

test_df = pd.read_csv(f"{INPUT}/test.csv")
display(test_df.info())

submit_df = pd.read_csv(f"{INPUT}/sample_submission.csv")

# !cp "/content/drive/My Drive/ML/kiva/input/test_images.zip" .
# !unzip test_images.zip
# print(len(os.listdir("test_images"))) #should be 91819

train_df.columns

train_columns = ["ORIGINAL_LANGUAGE", "ACTIVITY_NAME", "SECTOR_NAME", "COUNTRY_CODE","CURRENCY_POLICY",
       "CURRENCY","REPAYMENT_INTERVAL", "DISTRIBUTION_MODEL"]
target = "LOAN_AMOUNT"

X = train_df[train_columns]
y = train_df[target]

for column in X.columns:
  le = preprocessing.LabelEncoder()
  target_column = X[column]
  le.fit(target_column)
  label_encoded_column = le.transform(target_column)
  X[column] = pd.Series(label_encoded_column).astype("category")

display(X)

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state = SEED)

params = {
    "application" : "regression_l1",
    "metric" : "l1" #mae
}

lgb_train = lgb.Dataset(X_train, y_train)
lgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train) 

%%time
evaluation_results = {}
model = lgb.train(params,
                  train_set=lgb_train,
                  valid_sets=lgb_eval,
                  valid_names=["train"],  
                  evals_result=evaluation_results
                  )

plt.plot(evaluation_results["train"]["l1"], label="train")
plt.ylabel("MAE")
plt.xlabel("Num Boosting round")
plt.title("Training performance")
plt.legend()
plt.show()

y_pred = model.predict(X_test)

display(pd.DataFrame({"y_test":y_test,"y_pred":y_pred}))

MAE = mean_absolute_error(y_test, y_pred)
print("MAE = " + str(MAE))

lgb.plot_importance(model)
plt.show()

test_X = test_df[train_columns]

for column in test_X.columns:
  le = preprocessing.LabelEncoder()
  target_column = test_X[column]
  le.fit(target_column)
  label_encoded_column = le.transform(target_column)
  test_X[column] = pd.Series(label_encoded_column).astype("category")

test_predicted = model.predict(test_X)

test_predicted = (np.round(test_predicted/25)*25).astype(int)
submit_df["LOAN_AMOUNT"]=test_predicted
print(submit_df)

display(pd.DataFrame({"trainファイル":train_df["LOAN_AMOUNT"].describe(),"submitファイル":submit_df["LOAN_AMOUNT"].describe()}))

now = datetime.datetime.now(datetime.timezone(datetime.timedelta(hours=1)))
date_time = "{0:%Y%m%d_%H%M}".format(now)

submit_df.to_csv(os.path.join(SUBMIT, f"submit_{date_time}_{MAE:.3f}.csv"), index=False)

SLACK_CHANNEL="#general"
SLACK_USER="Colab"
SLACK_TEXT="Colab完了通知"

slack = slackweb.Slack(url="ご自身のIncoming Webhook URL")
attachments=[]
attachment={
            "color": "#2eb886",
            "fields":[
                  {
                  "title": "ファイル作成時刻","value": date_time,
                  },
                  {
                  "title": "MAE","value": MAE,
                  }
              ]
            }
attachments.append(attachment)
slack.notify(text=SLACK_TEXT, channel=SLACK_CHANNEL, username=SLACK_USER, attachments=attachments)