lightgbm实战-二分类问题（贝叶斯优化下调参方法）

# use bayes_opt from sklearn.datasets import make_classification from sklearn.ensemble import RandomForestClassifier from sklearn.model_selection import cross_val_score,KFold from bayes_opt import BayesianOptimization import numpy as np # 产生随机分类数据集，10个特征， 2个类别 x, y = make_classification(n_samples=1000,n_features=10,n_classes=2) # 尝试一下用未调参的随机森林模型进行交叉验证 rf = RandomForestClassifier() # 这里会输出5个值，取得均值 cv_score = cross_val_score(rf, x, y, scoring="f1", cv=5).mean() cv_score # 定义一个函数，输入一些超参数，这些超参数就是需要进行调整的参数 def rf_cv(n_estimators, min_samples_split, max_features, max_depth): cv_score = cross_val_score( RandomForestClassifier(n_estimators=int(n_estimators), min_samples_split=int(min_samples_split), max_features=float(max_features), max_depth=int(max_depth), random_state=2), x, y, scoring="f1", cv=5).mean() # 必须返回一个值，如果像误差的话（回归算法）这里是需要加上一个负号的 return cv_score rf_bo = BayesianOptimization( rf_cv, {'n_estimators': (10, 250), 'min_samples_split': (2, 25), 'max_features': (0.1, 0.999), 'max_depth': (5, 15)} ) # 输出不同迭代参数组合下的得分 rf_bo.maximize() # 输出最高得分下的参数组合 rf_bo.max

# 带入最佳参数进行计算 rf = RandomForestClassifier(random_state=2,max_depth=12,max_features=0.2694,min_samples_split=6,n_estimators=103) np.mean(cross_val_score(rf, x, y, cv=4, scoring='f1'))

使用lightgbm尝试一波

# 设置几个参数 def lgb_cv(colsample_bytree, min_child_samples, num_leaves, subsample, max_depth): model = lgb.LGBMClassifier(boosting_type='gbdt',objective='binary', colsample_bytree=float(colsample_bytree), learning_rate=0.01, min_child_samples=int(min_child_samples), min_child_weight=0.001, n_estimators=800, n_jobs=-1, num_leaves=int(num_leaves), random_state=None, reg_alpha=0.0, reg_lambda=0.0,max_depth=int(max_depth), subsample=float(subsample)) cv_score = cross_val_score(model, x, y, scoring="f1", cv=5).mean() return cv_score # 使用贝叶斯优化 lgb_bo = BayesianOptimization( lgb_cv, {'colsample_bytree': (0.7,1), 'min_child_samples': (2, 25), 'num_leaves': (5, 250), 'subsample': (0.7, 1), 'max_depth':(2,10)} ) lgb_bo.maximize() lgb_bo.max # 将优化好的参数带入进行使用 model = lgb.LGBMClassifier(boosting_type='gbdt',objective='binary', colsample_bytree=0.7111, learning_rate=0.01, min_child_samples=9, min_child_weight=0.001, n_estimators=800, n_jobs=-1, num_leaves=188, random_state=None, reg_alpha=0.0, reg_lambda=0.0,max_depth=2, subsample=0.91) cv_score = cross_val_score(model, x, y, scoring="f1", cv=5).mean() cv_score # 结果就不放出来了，也就是一个值，大家可以模仿一下，跑跑代码。

接下里是采用lightgbm的自带的train方法配合交叉验证

def LGB_CV( max_depth, num_leaves, min_data_in_leaf, feature_fraction, bagging_fraction, lambda_l1 ): # 这里就不采用交叉验证的cv_score = cross_val_score(model, x, y, scoring="f1", cv=5) kf = KFold(n_splits=5,shuffle=True) # f是准备存储预测值的，交叉验证下，用五份数据作为验证集，最后将这五份数据放回f里 f = np.zeros(x.shape[0]) for index, (train_index, val_index) in enumerate(kf.split(x)): print("fold--{}".format(index)) train_data = lgb.Dataset(x[train_index], label=y[train_index]) val_data = lgb.Dataset(x[val_index], label=y[val_index]) params = {'num_leaves': int(num_leaves), 'min_data_in_leaf': int(min_data_in_leaf), 'objective':'binary', 'max_depth': int(max_depth), 'learning_rate': 0.01, "boosting": "gbdt", "feature_fraction": feature_fraction, "bagging_fraction": bagging_fraction , "metric": 'auc', "lambda_l1": lambda_l1, } # 因为是交叉验证的算法，这里直接使用train，valid_sets就是要评估的数据集 model = lgb.train(params, train_data, valid_sets=[train_data, val_data],num_boost_round=700, verbose_eval=500,early_stopping_rounds = 20) # 返回迭代中最好的数据,这里的predict里面的数据（不需要经过dataset）不需要再进行转化，如果是xgboost就需要，需要把x_test进行转化DMatrix(x_test),这里x_test不包含类别特征 f[val_index] = model.predict(x[val_index], num_iteration=model.best_iteration) # predict里面的验证集不需要进行dataset，但是xgboost算法时需要dmatrix，并且只需要DMatrix(x_test),这里x_test不包含类别特征,很多地方这里都会出错，直接带着类别就去预测 del model, train_index, val_index # 由于输出的是概率值，转化为0，1的整型值 f = np.array([1 if i>0.5 else 0 for i in oof]) return metrics.f1_score(f, y) # 最后进行调参 LGB_CV( max_depth=5, num_leaves=32, min_data_in_leaf=15, feature_fraction=0.8, bagging_fraction=0.8, lambda_l1=None ) # 采用贝叶斯优化算法 lgb_ba = BayesianOptimization(LGB_CV, {"max_depth":(2,12), "num_leaves":(5,130), "min_data_in_leaf":(5,30), "feature_fraction":(0.7,1), "bagging_fraction":(0.7,1), "lambda_l1":(0,6)}) lgb_ba.maximize() lgb_ba.max["params"]

# 把设置好的参数带入 kf = KFold(n_splits=5,shuffle=True) f = np.zeros(x.shape[0]) # 设置测试集数据 x_test = x[:200] y_test = y[:200] prediction = np.zeros(x_test.shape[0]) for index, (train_index, val_index) in enumerate(kf.split(x)): print("fold--{}".format(index)) train_data = lgb.Dataset(x[train_index], label=y[train_index]) val_data = lgb.Dataset(x[val_index], label=y[val_index]) params = {'num_leaves': 44, 'min_data_in_leaf': 19, 'objective':'binary', 'max_depth': 11, 'learning_rate': 0.01, "boosting": "gbdt", "feature_fraction": 0.81, "bagging_fraction": 0.84 , "metric": 'auc', "lambda_l1": 1.8, } # 因为是交叉验证的算法，这里直接使用train，valid_sets就是要评估的数据集 model = lgb.train(params, train_data, valid_sets=[train_data, val_data],num_boost_round=700, verbose_eval=500,early_stopping_rounds=20) f[val_index] = model.predict(x[val_index], num_iteration=model.best_iteration) # predict里面的数据不需要进行dataset prediction +=model.predict(x_test)num_iteration=model.best_iteration)/kf.n_splits metrics.f1_score(np.array([1 if i>0.5 else 0 for i in prediction]), y_test)

题外话

这是kaggle上面的一种做法，也是非常高效的