基于AutoML的航班延迟起飞预测模型搭建

import pandas as pd


import warnings
warnings.filterwarnings('ignore')


import logging
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.ERROR)


from ads.dataset.factory import DatasetFactory
复制

flights = DatasetFactory.open("/home/datascience/ads-examples/3P_data/flights.csv")flights.head()
复制

flights = flights.set_target('DepDelay')
复制

flights.show_in_notebook()
复制

flights_nocheat = flights.drop_columns(["ArrDelay", "ArrTime", "DepTime", 
                                            "AirTime", "ActualElapsedTime"]).sample(frac=0.1)
复制

flights_nocheat.target.show_in_notebook(feature_names=["DayOfWeek", "DayofMonth", 
                                                           "UniqueCarrier"])
复制

type(flights_nocheat)
# RegressionDataset
复制

# 转换前show_in_notebook() 默认显示为 Histogram Graph
flights_nocheat.target.show_in_notebook()
复制

flights_binary = flights_nocheat.assign_column("DepDelay", lambda x:  x is None or x>15)
type(flights_binary)
# BinaryClassificationDataset
复制

# 数据转换后, show_in_notebook() 默认显示为Count Plot Graph
flights_binary.target.show_in_notebook()
复制

flights_binary.get_recommendations()
复制

flights_clean = flights_binary.auto_transform(fix_imbalance=False)
复制

flights_clean.show_in_notebook()
复制

[len(flights_binary.columns), len(flights_clean.columns)]
复制

# 相关性Top12变量选择
modeling_dataset = flights_clean.select_best_features(k=12)
print(modeling_dataset.columns.values)
复制

import h2o
import os
from h2o.automl import H2OAutoML
复制

# H2O Proxy 初始化
http_proxy = os.environ.get("http_proxy", "")
HTTP_PROXY = os.environ.get("HTTP_PROXY", "")
os.environ["HTTP_PROXY"] = ""
os.environ["http_proxy"] = ""
h2o.init()
复制

h2o_df = modeling_dataset.to_h2o_dataframe()
h2o_df.head(5)
复制

train, valid, test = h2o_df.split_frame(ratios=list([.7, .15]), seed = 1)
[train.shape, valid.shape, test.shape]
复制

x = train.columns
y = "DepDelay"
x.remove(y)
复制

%%time
h2o_aml = H2OAutoML(max_models=20, seed=1, max_runtime_secs=100)
h2o_aml.train(x=x, y=y, training_frame=train, validation_frame=valid)
h2o_model = h2o_aml.leader
复制

print("\nH2O Accuracy: ", (h2o_model.predict(test)[0] == test["DepDelay"]).sum() / test.shape[0])
复制

os.environ["http_proxy"] = http_proxy
os.environ["HTTP_PROXY"] = HTTP_PROXY
复制

from ads.automl.provider import OracleAutoMLProvider
复制

train, valid, test = modeling_dataset.train_validation_test_split(test_size=0.1, validation_size=0.2, random_state=42)
[train.X.shape, valid.X.shape, test.X.shape]
复制

olabs_automl = OracleAutoMLProvider()
复制

import lightgbm
复制

%%time
from ads.automl.driver import AutoML
automl = AutoML(train, validation_data=valid, 
                provider=olabs_automl)
olabs_model, baseline = automl.train(min_features=["CRSDepTime", "TaxiOut"], model_list=["LogisticRegression", "LGBMClassifier", "XGBClassifier", "RandomForestClassifier"], random_state = 42, score_metric = 'roc_auc')
复制

print("\nOLabs Score: ", olabs_model.score(test.X, test.y))
复制

olabs_model.selected_model_params_
复制

olabs_model.num_total_trials_
复制

olabs_model.summary()
复制

automl.visualize_feature_selection_trials()
复制

automl.visualize_tuning_trials()
复制

olabs_model.visualize_transforms()
复制

from ads.evaluations.evaluator import ADSEvaluator
from ads.common.data import MLData


evaluator = ADSEvaluator(test, models=[baseline, olabs_model])
evaluator.show_in_notebook()
复制

from ads.explanations.explainer import ADSExplainer
from ads.explanations.mlx_global_explainer import MLXGlobalExplainer
复制

olabs_explainer = ADSExplainer(test, olabs_model)
global_explainer = olabs_explainer.global_explanation(provider=MLXGlobalExplainer())
复制

global_explainer.feature_importance_summary()
复制

feature_importance = global_explainer.compute_feature_importance()
复制

feature_importance.get_global_explanation()
复制