
from sklearn.datasets import make_classification
from sklearn.model_selection import GridSearchCV
import nose.tools as test_# For testing
import numpy as np
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score
from sklearn.model_selection import train_test_split


# Useful in beautifying numpy arrays.
from IPython.display import HTML, display
import tabulate
def pp(a, show_head=True): 
    '''
    args: show_head -> if True print only first 5 rows.
    return: None
    '''
    if a.ndim < 2:
        a = [a]
    if show_head:
        display(HTML(tabulate.tabulate(a[:5], tablefmt='html')))
        return
    display(HTML(tabulate.tabulate(a, tablefmt='html')))


def generate_dataset(n_samples_, n_classes_, n_features_, \
                           n_informative_, n_redundant_, random_state_, shuffle_):
    '''
    args: n_samples -> int => number of samples to generate
          n_classes  -> int => number of classes in your dataset
          n_features -> int => total number of features (inforamtive + redundant)
          n_informative  -> int => number of informative features
          n_informative  -> int => number of informative features
          n_redundant  -> int => number of redundant features
          random_state_ -> int => random state (for reproduciable results)
          shuffle_ -> Bool => whether to shuffle data.
          
    return: tuple (X, y) => X is ndarray of features (m, 7)   
                         => y is ndarray of labels (m,)
    '''
    ### BEGIN SOLUTION
    X, y = make_classification(n_samples=n_samples_, n_classes=n_classes_, \
                               n_features=n_features_, n_informative=n_informative_, \
                               n_redundant=n_redundant_, random_state=random_state_, \
                               shuffle=shuffle_)
    return (X, y)
    ### END SOLUTION


X_for_test = generate_dataset(10000, 16, 7, 5, 2, 42, True)[0]
y_for_test = generate_dataset(10000, 16, 7, 5, 2, 42, True)[1]
test_.eq_ (X_for_test.shape, (10000, 7))
test_.eq_ (y_for_test.shape, (10000,))
### BEGIN HIDDEN TESTS
test_.ok_(np.isclose(X_for_test[:5][1], np.asarray([ 2.24443806,  1.7409484 , -1.50799445, -1.87367918,  4.70342764,
        0.72546393, -3.27004638])).all())
### END HIDDEN TESTS


X = generate_dataset(10000, 16, 7, 5, 2, 42, True)[0]
y = generate_dataset(10000, 16, 7, 5, 2, 42, True)[1]


print('Dataset with 5 informative features and 2 redundant features:')
pp(X)
print('Labels (Total Classes-16):')
pp(y)

Dataset with 5 informative features and 2 redundance features:

Labels (Total Classes-16):


X_train, X_test, y_train, y_test = train_test_split(X, y, \
                                                    test_size=0.33, random_state=42)


svc = SVC()
svc.fit(X_train, y_train)

SVC()


y_pred = svc.predict(X_test)


svc.score(X_test, y_test)

0.5663636363636364


accuracy_score(y_test, y_pred)

0.5663636363636364


# Set up any libarary imports here
### BEGIN SOLUTION
from sklearn.linear_model import LogisticRegression
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
### END SOLUTION SOLUTION


def instantiate_classifiers_for_voting(rs, n_estimators_):
    '''
    Return 4 instantiated (not fitted) classifiers.
    args: rs -> int => random state for all classifiers which accept it.
          n_estimators_ -> int => number of estimators for RandomForestClassifier
    
    return: tuple  of length 4 -> (clf1, clf2, clf3, clf4) => Each element is 
                an instantiated classifier in this order logistic regression, Random Forest Classifier,
                Gaussian NB, SVC.
    
    Other:
        > In LogisticRegression use 'multinomial' for 'multi_class' arg.
        > For all classifiers, leave other parameters with their default values.
    '''
    ### BEGIN SOLUTION
    clf1 = LogisticRegression(multi_class='multinomial', random_state=rs)
    clf2 = RandomForestClassifier(n_estimators=n_estimators_, random_state=rs)
    clf3 = GaussianNB()
    clf4 = SVC(random_state=rs)
    return (clf1, clf2, clf3, clf4)
    ### END SOLUTION


test_.eq_ (len(instantiate_classifiers_for_voting(34, 10)), 4)
logr = instantiate_classifiers_for_voting(34, 10)[0]
test_.eq_(logr.random_state, 34)

### BEGIN HIDDEN TESTS
try:
    logr.score(X, y)
except:
    pass
else:
    raise AssertionError("Fitted classifier when you only instantiation was required") 
### END HIDDEN TESTS


_classifiers_voting = instantiate_classifiers_for_voting(42, 100)


_classifiers_voting

(LogisticRegression(multi_class='multinomial', random_state=42),
 RandomForestClassifier(random_state=42),
 GaussianNB(),
 SVC(random_state=42))


### BEGIN SOLUTION
from sklearn.ensemble import VotingClassifier
### END SOLUTION


def fit_voting_classifier(classifiers_, voting_):
    '''
    Fit a Voting classifier which uses classifiers_ as estimators. Note that in sklearn
    fitting a voting classifier will automatically fit its estimators. Read relevant documentation
    for more information. 
    
    
    args: classifiers_ -> tuple  of length 4 -> (clf1, clf2, clf3, clf4) => Each element is 
                                                              an instantiated classifier.
          voting_ -> string -> 'hard', 'soft'. We'll only do 'hard' voting here. Do not 
                                               worry about soft voting at this point. Read relevant documentation
                                               for more information. 
    
    return: fitted voting classifier.
    
    '''
    ### BEGIN SOLUTION
    clf1, clf2, clf3, clf4 = classifiers_[0], classifiers_[1],\
                            classifiers_[2], classifiers_[3]
    estimators_ = [
        ('lr', clf1), 
        ('rf', clf2), 
        ('gnb', clf3), 
        ('svc', clf4)
    ]
    vot_clf = VotingClassifier(estimators=estimators_, voting=voting_)
    vot_clf = vot_clf.fit(X, y)
    return vot_clf
    ### END SOLUTION


_vot_clf = fit_voting_classifier(_classifiers_voting, 'hard')


estimators_list = _vot_clf.estimators_


_vot_clf = fit_voting_classifier(_classifiers_voting, 'hard')
estimators_list = _vot_clf.estimators_
test_.eq_ (
    [str(est_name) for est_name in estimators_list], [
        "LogisticRegression(multi_class='multinomial', random_state=42)",
        'RandomForestClassifier(random_state=42)',
        'GaussianNB()',
        'SVC(random_state=42)'
    ]
)
### BEGIN HIDDEN TESTS
test_.eq_(_vot_clf.n_features_in_ , 7)
try:
    _vot_clf.score(X_test, y_test)
except:
    raise AssertionError('Unfitted voting classifier?')
### END HIDDEN TESTS


_vot_clf = fit_voting_classifier(_classifiers_voting, 'hard')


_vot_clf = fit_voting_classifier(_classifiers_voting, 'hard')


y_pred_vot_clf = _vot_clf.predict(X_test)


print('R2 score:', _vot_clf.score(X_test, y_test))
print('Accuracy:', accuracy_score(y_pred_vot_clf, y_test))

R2 score: 0.6193939393939394
Accuracy: 0.6193939393939394


# Set up any libarary imports here
### BEGIN SOLUTION
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
### END SOLUTION SOLUTION


def instantiate_classifiers_for_stacking(rs, n_estimators_, max_iter_):
    '''
    Return 5 instantiated (not fitted) classifiers.
    args: rs -> int => random state for all classifiers which accept it.
          n_estimators_ -> int => number of estimators for RandomForestClassifier
          max_iter_ -> int => pass this to SVC and both Logistic Regressions to silence warnings. 
                              Read relevant docs for more information.
    
    return: tuple  of length 4 -> (clf1, clf2, clf3, clf4, final_clf) => Each element is 
                an instantiated classifier in this order Random Forest Classifier,
                SVC, Logistic Regression, Gaussian NB. The final_clf is also a 
                Logistic Regression.
    
    Other:
        > In both LogisticRegressions use 'multinomial' for 'multi_class' arg.
        > For all classifiers, leave other parameters with their default values.
    '''
    ### BEGIN SOLUTION
    clf1 = RandomForestClassifier(n_estimators=n_estimators_, random_state=rs)
    clf2 = make_pipeline(StandardScaler(), 
                          SVC(max_iter=max_iter_, random_state=rs))
    clf3 = LogisticRegression(multi_class='multinomial', random_state=rs, \
                              max_iter=max_iter_)
    clf4 = GaussianNB()
    
    final_clf = final_estimator=LogisticRegression(multi_class='multinomial',
                                       max_iter=max_iter_)
    return (clf1, clf2, clf3, clf4, final_clf)
    ### END SOLUTION


test_.eq_ (len(instantiate_classifiers_for_stacking(34, 10, 10000)), 5)
logr = instantiate_classifiers_for_stacking(34, 10, 10000)[0]
test_.eq_(logr.random_state, 34)
### BEGIN HIDDEN TESTS
try:
    logr.score(X, y)
except:
    pass
else:
    raise AssertionError("Fitted classifier when you only instantiation was required") 
### END HIDDEN TESTS


classifiers_stacking = instantiate_classifiers_for_stacking(42, 100, 100000)


# Set up any libarary imports here
### BEGIN SOLUTION
from sklearn.ensemble import StackingClassifier
### END SOLUTION SOLUTION


def fit_stacking_classifier(classifiers_):
    '''
    Fit a Voting classifier which uses classifiers_ as estimators. Note that in sklearn
    fitting a voting classifier will automatically fit its estimators. Read relevant documentation
    for more information. 
    
    
    args: classifiers_ -> tuple  of length 4 -> (clf1, clf2, clf3, clf4) => Each element is 
                                                              an instantiated classifier. 
    
    return: fitted voting classifier.
    
    The solution should be very similar to that of fit_voting_classifier.
    '''
    
    ### BEGIN SOLUTION
    clf1, clf2, clf3, clf4, final_clf = classifiers_[0], classifiers_[1],\
                            classifiers_[2], classifiers_[3], classifiers_[4]
   
    estimators_ = [
        ('rf', clf1), 
        ('svc', clf2), 
        ('lr', clf3), 
        ('gb', clf4)
    ]
    stack_clf = StackingClassifier(estimators=estimators_, final_estimator=final_clf)
    stack_clf = stack_clf.fit(X, y)
    return stack_clf
    ### END SOLUTION


_stack_clf = fit_stacking_classifier(classifiers_stacking)
estimators_list = _stack_clf.estimators_
test_.eq_ (
    ["".join(str(est_name).split()) for est_name in estimators_list], # remove white spaces then compare
    [
        'RandomForestClassifier(random_state=42)',
        "Pipeline(steps=[('standardscaler',StandardScaler()),('svc',SVC(max_iter=100000,random_state=42))])",
        "LogisticRegression(max_iter=100000,multi_class='multinomial',random_state=42)",
        'GaussianNB()'
    ]
)
### BEGIN HIDDEN TESTS
test_.eq_(_stack_clf.n_features_in_ , 7)
try:
    _stack_clf.score(X_test, y_test)
except:
    raise AssertionError('Unfitted voting classifier?')
### END HIDDEN TESTS


_stack_clf = fit_stacking_classifier(classifiers_stacking)
y_pred_stack_clf = _stack_clf.predict(X_test)
print('R2 score:', _stack_clf.score(X_test, y_test))
print('Accuracy:', accuracy_score(y_pred_vot_clf, y_test))

R2 score: 0.9215151515151515
Accuracy: 0.6193939393939394


# Set up any library imports here
### BEGIN SOLUTION
import xgboost as xgb
### END SOLUTION


# You may use this cell (or create other cells) 
# for scratch work e.g. GridSearch() for hyperparameters tuning.


def train_XGBoost(X_train_, y_train_):
    '''
    Train a booster object on 
    return: xgboost.core.Booster object 
    
    args: X_train_ -> ndarray -> shape (m, 7)
          y_train_ -> ndarray -> shape (m,)
          
    Returned Booster should be trained (hyperparameter tuned etc) which can predict
    accuracy score on test data. Accuracy should be at least 56 to earn credit.
    '''
    bst = None
    ### BEGIN SOLUTION
    dtrain = xgb.DMatrix(X_train, y_train)
    dtest = xgb.DMatrix(X_test, y_test)
    param = {'max_depth': 32, 'eta': .4, 'objective': 'multi:softmax',\
        'subsample':.93}
    param['nthread'] = 5
    param['num_class'] = 16
    param['n_estimators'] = 400
    num_round = 20
    bst = xgb.train(param, dtrain, num_round)
    ### END SOLUTION
    return bst # xgboost.core.Booster object


bst = train_XGBoost(X_train, y_train)
test_.eq_ (str(type(bst)), "<class 'xgboost.core.Booster'>")
### BEGIN HIDDEN TESTS
dtest = xgb.DMatrix(X_test)
y_pred_test = bst.predict(dtest)
acc = accuracy_score(y_test, y_pred_test)
test_.ok_ (acc >= 0.56)
### END HIDDEN TESTS

[15:18:53] WARNING: /Users/travis/build/dmlc/xgboost/src/learner.cc:480: 
Parameters: { n_estimators } might not be used.

  This may not be accurate due to some parameters are only used in language bindings but
  passed down to XGBoost core.  Or some parameters are not used but slip through this
  verification. Please open an issue if you find above cases.


xgb.plot_importance(bst) # feature immportance

<matplotlib.axes._subplots.AxesSubplot at 0x7f9697e934a8>


# !pip install graphviz


xgb.to_graphviz(bst, num_trees=2) # second tree. Change this number to display the corresponding tree.

0.913171	0.83296	1.19702	0.293289	-0.573523	0.97669	0.926118
2.24444	1.74095	-1.50799	-1.87368	4.70343	0.725464	-3.27005
2.64499	0.132772	3.39987	-1.80951	-1.39368	-1.52042	0.228421
0.0373674	0.880855	0.256795	-1.93653	2.12171	-2.14082	-2.64057
0.938134	-2.22023	2.08695	-1.89748	-2.10017	-2.00003	0.607856

Test 8 Solution

Ensemble Learning¶

Question 1¶

Train Test Split¶

Train SVC¶

Bagging¶

Question 2¶

Question 3¶

Train¶

Predict¶

Accuracy¶

Stacking¶

Question 4¶

Boosting¶

Question 5¶

Other Insights From XGBoost¶