from google.colab import drive
drive.mount('/content/drive/')

# ご自身の環境に合わせてください
%cd /content/drive/My\ Drive/予測コンペ/ProbSpace/YouTube動画視聴回数予測
!ls

import featuretools as ft
import lightgbm as lgb
#import optuna
import numpy as np
import sklearn.datasets
import sklearn.metrics
import pandas as pd
import matplotlib.pyplot as plt
import tensorflow as tf
import xgboost as xgb
import re
import seaborn as sns
from tensorflow import keras
import keras.layers as L
import seaborn as sns
from datetime import datetime, timezone, timedelta
from keras.models import Model
from sklearn.decomposition import PCA
from keras import losses
from sklearn import preprocessing
from sklearn.metrics import log_loss
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.metrics import mean_squared_log_error
import unicodedata

!pip install japanize-matplotlib
import japanize_matplotlib
!pip install jeraconv
from jeraconv import jeraconv

train = pd.read_csv("train_data.csv")
print("train shape is " + str(train.shape))
train.head(1)

test = pd.read_csv("test_data.csv")
iddf = test[["id"]]
print("test shape is " + str(test.shape))
test.head(1)

train = pd.read_csv("train_data.csv")
test = pd.read_csv("test_data.csv")

train["y"] = np.log(train["y"])

mean_ = train[["categoryId", "y"]].groupby("categoryId").mean().reset_index().rename({"y":"mean"}, axis=1)
max_ = train[["categoryId", "y"]].groupby("categoryId").max().reset_index().rename({"y":"max"}, axis=1)
min_ = train[["categoryId", "y"]].groupby("categoryId").min().reset_index().rename({"y":"min"}, axis=1)
std_ = train[["categoryId", "y"]].groupby("categoryId").std().reset_index().rename({"y":"std"}, axis=1)
count_ = train[["categoryId", "y"]].groupby("categoryId").count().reset_index().rename({"y":"count"}, axis=1)
q1_ = train[["categoryId", "y"]].groupby("categoryId").quantile(0.1).reset_index().rename({"y":"q1"}, axis=1)
q25_ = train[["categoryId", "y"]].groupby("categoryId").quantile(0.25).reset_index().rename({"y":"q25"}, axis=1)
q5_ = train[["categoryId", "y"]].groupby("categoryId").quantile(0.5).reset_index().rename({"y":"q5"}, axis=1)
q75_ = train[["categoryId", "y"]].groupby("categoryId").quantile(0.75).reset_index().rename({"y":"q75"}, axis=1)
q9_ = train[["categoryId", "y"]].groupby("categoryId").quantile(0.9).reset_index().rename({"y":"q9"}, axis=1)

def is_japanese(string):
    for ch in string:
        try:
            name = unicodedata.name(ch) 
            if "CJK UNIFIED" in name \
            or "HIRAGANA" in name \
            or "KATAKANA" in name:
                return True
        except:
          continue
    return False

y = train["y"]
del train["y"]

df = pd.concat([train, test])

df["tags"].fillna("[none]", inplace=True)
tagdic = dict(pd.Series("|".join(list(df["tags"])).split("|")).value_counts().sort_values())


def bool_to_int(df):
  df["comments_disabled"] = df["comments_disabled"].astype(np.int16)
  df["ratings_disabled"] = df["ratings_disabled"].astype(np.int16)
  return df

def create_features(df):
  # like dislike comment
  #df["likes2"] = df["likes"]**2
  df["loglikes"] = np.log(df["likes"]+1)
  #df["dislikes2"] = df["dislikes"]**2
  df["logdislikes"] = np.log(df["dislikes"]+1)
  df["logcomment_count"] = np.log(df["comment_count"]+1)
  df["sqrtlikes"] = np.sqrt(df["likes"])
  df["like_dislike_ratio"] = df["likes"]/(df["dislikes"]+1)
  df["comments_like_ratio"] = df["comment_count"]/(df["likes"]+1)
  df["comments_dislike_ratio"] = df["comment_count"]/(df["dislikes"]+1)

  # likes comments diable
  df["likes_com"] = df["likes"] * df["comments_disabled"]
  df["dislikes_com"] = df["dislikes"] * df["comments_disabled"]
  df["comments_likes"] = df["comment_count"] * df["ratings_disabled"]

  # tags
  df["num_tags"] = df["tags"].astype(str).apply(lambda x: len(x.split("|")))
  df["length_tags"] = df["tags"].astype(str).apply(lambda x: len(x))
  df["tags_point"] = df["tags"].apply(lambda tags: sum([tagdic[tag] for tag in tags.split("|")]))
  df["count_en_tag"] = df["tags"].apply(lambda x: sum([bool(re.search(r'[a-zA-Z0-9]', x_)) for x_ in x.split("|")]))
  df["count_ja_tag"] = df["tags"].apply(lambda x: sum([is_japanese(x_) for x_ in x.split("|")]))

  # publishedAt
  df["publishedAt"] = pd.to_datetime(df["publishedAt"], utc=True)
  df["publishedAt_year"] = df["publishedAt"].apply(lambda x: x.year)
  df["publishedAt_month"] = df["publishedAt"].apply(lambda x: x.month)
  df["publishedAt_day"] = df["publishedAt"].apply(lambda x: x.day)
  df["publishedAt_hour"] = df["publishedAt"].apply(lambda x: x.hour)
  df["publishedAt_minute"] = df["publishedAt"].apply(lambda x: x.minute)
  #df["publishedAt_second"] = df["publishedAt"].apply(lambda x: x.second)
  df["publishedAt_dayofweek"] = df["publishedAt"].apply(lambda x: x.dayofweek)

  # collection_date
  #df["collection_date_year"] = df["collection_date"].apply(lambda x: int(x[0:2]))
  df["collection_date_month"] = df["collection_date"].apply(lambda x: int(x[3:5]))
  df["collection_date_day"] = df["collection_date"].apply(lambda x: int(x[6:8]))
  df["collection_date"] = pd.to_datetime("20"+df["collection_date"], format="%Y.%d.%m", utc=True)

  # delta
  df["delta"] = (df["collection_date"] - df["publishedAt"]).apply(lambda x: x.days)
  df["logdelta"] = np.log(df["delta"])
  df["sqrtdelta"] = np.sqrt(df["delta"])
  df["published_delta"] = (df["publishedAt"] - df["publishedAt"].min()).apply(lambda x: x.days)
  df["collection_delta"] = (df["collection_date"] - df["collection_date"].min()).apply(lambda x: x.days)

  df["description"].fillna(" ", inplace=True)
  df["ishttp_in_dis"] = df["description"].apply(lambda x: x.lower().count("http"))
  df["len_description"] = df["description"].apply(lambda x: len(x))

  df["title"].fillna(" ", inplace=True)
  df["len_title"] = df["title"].apply(lambda x: len(x))

  # is japanese
  df["isJa_title"] = df["title"].apply(lambda x: is_japanese(x))
  df["isJa_tags"] = df["tags"].apply(lambda x: is_japanese(x))
  df["isJa_description"] = df["description"].apply(lambda x: is_japanese(x))

  # is englosh
  #df["onEn_title"] = df["title"].apply(lambda x: x.encode('utf-8').isalnum())
  df["onEn_tags"] = df["tags"].apply(lambda x: x.encode('utf-8').isalnum())
  df["onEn_description"] = df["description"].apply(lambda x: x.encode('utf-8').isalnum())

  # cotain englosh
  df["conEn_title"] = df["title"].apply(lambda x: len(re.findall(r'[a-zA-Z0-9]', x.lower())))
  df["conEn_tags"] = df["tags"].apply(lambda x: len(re.findall(r'[a-zA-Z0-9]', x.lower())))
  df["conEn_description"] = df["description"].apply(lambda x: len(re.findall(r'[a-zA-Z0-9]', x.lower())))

  # Music
  df["music_title"] = df["title"].apply(lambda x: "music" in x.lower())
  df["music_tags"] = df["tags"].apply(lambda x: "music" in x.lower())
  df["music_description"] = df["description"].apply(lambda x: "music" in x.lower())

  # Official
  df["isOff"] = df["title"].apply(lambda x: "fficial" in x.lower())
  df["isOffChannell"] = df["channelTitle"].apply(lambda x: "fficial" in x.lower())
  df["isOffJa"] = df["title"].apply(lambda x: "公式" in x.lower())
  df["isOffChannellJa"] = df["channelTitle"].apply(lambda x: "公式" in x.lower())
  
  # Music
  df["cm_title"] = df["title"].apply(lambda x: "cm" in x.lower())
  df["cm_tags"] = df["tags"].apply(lambda x: "cm" in x.lower())
  df["cm_description"] = df["description"].apply(lambda x: "cm" in x.lower())

  
  df = df.merge(mean_, how='left', on=["categoryId"])
  df = df.merge(max_, how='left', on=["categoryId"])
  df = df.merge(min_, how='left', on=["categoryId"])
  df = df.merge(std_, how='left', on=["categoryId"])
  #df = df.merge(count_, how='left', on=["categoryId"])
  df = df.merge(q1_, how='left', on=["categoryId"])
  df = df.merge(q25_, how='left', on=["categoryId"])
  df = df.merge(q5_, how='left', on=["categoryId"])
  df = df.merge(q75_, how='left', on=["categoryId"])
  df = df.merge(q9_, how='left', on=["categoryId"])

  # 出現頻度
  for col in ["categoryId", "channelTitle"]:
    freq = df[col].value_counts()
    df["freq_"+col] = df[col].map(freq)

  return df

  #df['categoryId'] = df['categoryId'].astype('category')

df = bool_to_int(df)
df = create_features(df)

del df["channelId"]
del df["video_id"]
del df["title"]
del df["description"]
del df["thumbnail_link"]
del df["channelTitle"]
del df["tags"]
del df["publishedAt"]
del df["collection_date"]
del df["id"]

scalar = StandardScaler()
scalar.fit(df)
df = pd.DataFrame(scalar.transform(df), columns=df.columns)

X = df.iloc[:len(y), :]
test = df.iloc[len(y):, :]

def rmsle(preds, data):
    y_true = data.get_label()
    y_pred = preds
    y_pred[y_pred<0] = 0
    y_true[y_true<0] = 0
    acc = np.sqrt(mean_squared_log_error(np.exp(y_true), np.exp(y_pred)))
    # name, result, is_higher_better
    return 'accuracy', acc, False

# Optunaの最適化パラメータを代入する
light_params = {'task': 'train',
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'metric': 'rmse',
        'verbosity': -1,
        "seed":42,
        'learning_rate': 0.01,}
best_params =  {'lambda_l1': 0.019918875912078603, 'lambda_l2': 0.002616688073257713, 'num_leaves': 219, 'feature_fraction': 0.6641013611124621, 'bagging_fraction': 0.7024199018549259, 'bagging_freq': 5, 'min_child_samples': 5}
#best_params =  {}
light_params.update(best_params)

xgb_params = {'learning_rate': 0.1,
              'objective': 'reg:squarederror',
              'eval_metric': 'rmse',
              'seed': 42,
              'tree_method': 'hist'}
best_params = {'learning_rate': 0.01665914389764044, 'lambda_l1': 4.406831762257336, 'num_leaves': 39}
#best_params = {}
xgb_params.update(best_params)


FOLD_NUM = 11
kf = KFold(n_splits=FOLD_NUM,
           shuffle=True,
           random_state=42)
scores = []
feature_importance_df = pd.DataFrame()

pred_cv = np.zeros(len(test.index))
num_round = 10000


for i, (tdx, vdx) in enumerate(kf.split(X, y)):
    print(f'Fold : {i}')
    ######LGB
    X_train, X_valid, y_train, y_valid = X.iloc[tdx], X.iloc[vdx], y.values[tdx], y.values[vdx]

    # LGB
    lgb_train = lgb.Dataset(X_train, y_train)
    lgb_valid = lgb.Dataset(X_valid, y_valid)
    gbc = lgb.train(light_params, lgb_train, num_boost_round=num_round,
                  valid_names=["train", "valid"], valid_sets=[lgb_train, lgb_valid],
                  #feval=rmsle,
                  early_stopping_rounds=100, verbose_eval=500)
    if i ==0:
        importance_df = pd.DataFrame(gbc.feature_importance(), index=X.columns, columns=['importance'])
    else:
        importance_df += pd.DataFrame(gbc.feature_importance(), index=X.columns, columns=['importance'])
    gbc_va_pred = np.exp(gbc.predict(X_valid, num_iteration=gbc.best_iteration))
    gbc_va_pred[gbc_va_pred<0] = 0

    # XGB
    xgb_dataset = xgb.DMatrix(X_train, label=y_train)
    xgb_test_dataset = xgb.DMatrix(X_valid, label=y_valid)
    xgbm = xgb.train(xgb_params, xgb_dataset, 10000, evals=[(xgb_dataset, 'train'),(xgb_test_dataset, 'eval')],
                      early_stopping_rounds=100, verbose_eval=500)
    xgbm_va_pred = np.exp(xgbm.predict(xgb.DMatrix(X_valid)))
    xgbm_va_pred[xgbm_va_pred<0] = 0
    

    # ENS
    # lists for keep results
    lgb_xgb_rmsle = []
    lgb_xgb_alphas = []

    for alpha in np.linspace(0,1,101):
        y_pred = alpha*gbc_va_pred + (1 - alpha)*xgbm_va_pred
        rmsle_score = np.sqrt(mean_squared_log_error(np.exp(y_valid), y_pred))
        lgb_xgb_rmsle.append(rmsle_score)
        lgb_xgb_alphas.append(alpha)
    
    lgb_xgb_rmsle = np.array(lgb_xgb_rmsle)
    lgb_xgb_alphas = np.array(lgb_xgb_alphas)

    lgb_xgb_best_alpha = lgb_xgb_alphas[np.argmin(lgb_xgb_rmsle)]

    print('best_rmsle=', lgb_xgb_rmsle.min())
    print('best_alpha=', lgb_xgb_best_alpha)
    plt.plot(lgb_xgb_alphas, lgb_xgb_rmsle)
    plt.title('f1_score for ensemble')
    plt.xlabel('alpha')
    plt.ylabel('f1_score')

    score_ = lgb_xgb_rmsle.min()
    scores.append(score_)

    lgb_submission = np.exp(gbc.predict((test), num_iteration=gbc.best_iteration))
    lgb_submission[lgb_submission<0] = 0

    xgbm_submission = np.exp(xgbm.predict(xgb.DMatrix(test)))
    xgbm_submission[xgbm_submission<0] = 0

    submission = lgb_xgb_best_alpha*lgb_submission + (1 - lgb_xgb_best_alpha)*xgbm_submission

    pred_cv += submission/FOLD_NUM

print("##########")
print(np.mean(scores))

# 0.765394813186907
pd.set_option('display.max_rows', None)
importance_df.sort_values("importance")

light_submission_df = pd.concat([iddf, pd.DataFrame(pred_cv)], axis=1)
light_submission_df.columns = ["id", "y"]
light_submission_df.to_csv("submission_lgb.csv", index=False)
print("end")

from keras import losses
from keras.models import Sequential
from keras.layers import Dense, Dropout
from keras.layers.convolutional import Conv1D, UpSampling1D
from keras.layers.pooling import MaxPool1D
from keras.layers.core import Dense, Activation, Dropout, Flatten
from keras.layers import BatchNormalization
from keras.layers.pooling import MaxPooling1D
from keras.callbacks import LearningRateScheduler
from sklearn.preprocessing import OneHotEncoder
from keras.optimizers import Adam

# 学習率
def step_decay(epoch):
    x = 0.01
    if epoch >= 120: x = 0.001
    return x
lr_decay = LearningRateScheduler(step_decay)

# Optunaの最適化パラメータを代入する
def create_mlp(shape):
    '''
    Returns a keras model
    '''
    print(f"shape: {shape}")
    model = Sequential()
    model.add(Conv1D(32, 3, activation='relu', kernel_initializer='he_uniform', padding='same', input_shape=shape))
    model.add(BatchNormalization())
    model.add(Conv1D(32, 3, activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(BatchNormalization())
    model.add(MaxPooling1D(2))
    model.add(Dropout(0.2))
    model.add(Conv1D(64, 3, activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(BatchNormalization())
    model.add(Conv1D(64, 3, activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(BatchNormalization())
    model.add(MaxPooling1D(2))
    model.add(Dropout(0.3))
    model.add(Conv1D(128, 3, activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(BatchNormalization())
    model.add(Conv1D(128, 3, activation='relu', kernel_initializer='he_uniform', padding='same'))
    model.add(BatchNormalization())
    model.add(MaxPooling1D(2))
    model.add(Dropout(0.4))
    model.add(Flatten())
    model.add(Dense(128, activation='relu', kernel_initializer='he_uniform'))
    model.add(BatchNormalization())
    model.add(Dropout(0.5))
    model.add(Dense(1))
    return model


FOLD_NUM = 4
kf = KFold(n_splits=FOLD_NUM,
           shuffle=True,
           random_state=42)
scores = []
feature_importance_df = pd.DataFrame()

pred_cv = np.zeros(len(test.index))


for i, (tdx, vdx) in enumerate(kf.split(X, y)):
    print(f'Fold : {i}')
    X_train, X_valid, y_train, y_valid = X.iloc[tdx], X.iloc[vdx], y.values[tdx], y.values[vdx]

    mlp = create_mlp((X_train.values.shape[1], 1))
    optimizer = Adam(lr=0.001)
    mlp.compile(optimizer=optimizer, loss=losses.mean_squared_error)
    mlp.fit(x=np.reshape(X_train.values, (-1, X_train.shape[1], 1)), y=y_train.reshape(len(y_train),1),
           epochs=150, batch_size=493,
           validation_data=(np.reshape(X_valid.values, (-1, X_valid.shape[1], 1)), y_valid),
            callbacks=[lr_decay])#, verbose=0)

    mlp_pred = mlp.predict(np.reshape(X_valid.values, (-1, X_train.shape[1], 1)))

    plt.plot(mlp.history.history['loss'][3:], 'r', label='loss', alpha=0.7)
    plt.plot(mlp.history.history['val_loss'][3:], label='val_loss', alpha=0.7)
    plt.show()

    rmsle_score = np.sqrt(mean_squared_log_error(np.exp(y_valid), np.exp(mlp_pred)))
    print(rmsle_score)

    break

# memo
#Epoch 230/400
#14790/14790 [==============================] - 4s 301us/step - loss: 1.9379 - val_loss: 0.7369
np.sqrt(mean_squared_log_error(np.exp(y_valid), np.exp(mlp_pred)))

!pip install optuna
import optuna

def objective(trial):
    params = {
        'task': 'train',
        'boosting_type': 'gbdt',
        'objective': 'regression',
        'metric': 'rmse',
        'verbosity': -1,
        "seed":42,
        "learning_rate":trial.suggest_loguniform('learning_rate', 0.005, 0.03),
        'lambda_l1': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
        'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
        'num_leaves': trial.suggest_int('num_leaves', 2, 256),
        'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
        'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
        'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
        'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
    }
    FOLD_NUM = 4
    kf = KFold(n_splits=FOLD_NUM,
              #shuffle=True,
              random_state=42)
    scores = []
    feature_importance_df = pd.DataFrame()

    pred_cv = np.zeros(len(test.index))
    num_round = 10000


    for i, (tdx, vdx) in enumerate(kf.split(X, y)):
        print(f'Fold : {i}')
        X_train, X_valid, y_train, y_valid = X.iloc[tdx], X.iloc[vdx], y.values[tdx], y.values[vdx]
        lgb_train = lgb.Dataset(X_train, y_train)
        lgb_valid = lgb.Dataset(X_valid, y_valid)
        model = lgb.train(params, lgb_train, num_boost_round=num_round,
                      valid_names=["train", "valid"], valid_sets=[lgb_train, lgb_valid],
                      early_stopping_rounds=10, verbose_eval=10000)
        va_pred = model.predict(X_valid)
        score_ = np.sqrt(mean_squared_error(y_valid, va_pred))
        scores.append(score_)

    return np.mean(scores)

study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=100)

# 結果の確認
print('Best trial:')
light_trial = study.best_trial

print('  Value: {}'.format(light_trial.value))

print('  Params: ')

with open("lightgbmparams.txt", "w") as file:
    for key, value in light_trial.params.items():
       print('    "{}": {},'.format(key, value))
       file.write('"{}": {},'.format(key, value))

#0.7894605792171627

def objective(trial):
    params = {
        'objective': 'reg:squarederror',
        'eval_metric': 'rmse',
        'seed': 42,
        'tree_method': 'hist',
        "learning_rate":trial.suggest_loguniform('learning_rate', 0.005, 0.03),
        'lambda_': trial.suggest_loguniform('lambda_l1', 1e-8, 10.0),
        #'lambda_l2': trial.suggest_loguniform('lambda_l2', 1e-8, 10.0),
        'num_leaves': trial.suggest_int('num_leaves', 2, 256),
        #'feature_fraction': trial.suggest_uniform('feature_fraction', 0.4, 1.0),
        #'bagging_fraction': trial.suggest_uniform('bagging_fraction', 0.4, 1.0),
        #'bagging_freq': trial.suggest_int('bagging_freq', 1, 7),
        #'min_child_samples': trial.suggest_int('min_child_samples', 5, 100),
    }
    FOLD_NUM = 4
    kf = KFold(n_splits=FOLD_NUM,
              #shuffle=True,
              random_state=42)
    scores = []
    feature_importance_df = pd.DataFrame()

    pred_cv = np.zeros(len(test.index))
    num_round = 10000


    for i, (tdx, vdx) in enumerate(kf.split(X, y)):
        print(f'Fold : {i}')
        X_train, X_valid, y_train, y_valid = X.iloc[tdx], X.iloc[vdx], y.values[tdx], y.values[vdx]
        # XGB
        xgb_dataset = xgb.DMatrix(X_train, label=y_train)
        xgb_test_dataset = xgb.DMatrix(X_valid, label=y_valid)
        xgbm = xgb.train(params, xgb_dataset, 10000, evals=[(xgb_dataset, 'train'),(xgb_test_dataset, 'eval')],
                          early_stopping_rounds=100, verbose_eval=5000)
        xgbm_va_pred = xgbm.predict(xgb.DMatrix(X_valid))
        xgbm_va_pred[xgbm_va_pred<0] = 0
        score_ = np.sqrt(mean_squared_error(y_valid, xgbm_va_pred))
        scores.append(score_)

    return np.mean(scores)

study = optuna.create_study(direction='minimize')
study.optimize(objective, n_trials=100)

# 結果の確認
print('Best trial:')
light_trial = study.best_trial

print('  Value: {}'.format(light_trial.value))

print('  Params: ')

with open("lightgbmparams.txt", "w") as file:
    for key, value in light_trial.params.items():
       print('    "{}": {},'.format(key, value))
       file.write('"{}": {},'.format(key, value))

#non turning params 0.7897379694106698