We'll now build a single model with the Keras library:

1. Build a linear stack of layers with the sequential model:

# building a linear stack of layers with the sequential model
model = Sequential()
model.add(Dense(512, input_shape=(1024,)))
model.add(Activation('relu')) 

model.add(Dense(512))
model.add(Activation('relu'))

model.add(Dense(10))
model.add(Activation('softmax'))

2. Compile the model:

# compiling the sequential model
model.compile(loss='categorical_crossentropy', metrics=['accuracy'], optimizer='adam')

3. Fit the model to the train data and validate it with the test data:

# training the model and saving metrics in history
svhn_model = model.fit(x_train, y_train,
          batch_size=128, epochs=100,
          verbose=2,
          validation_data=(x_test, y_test))

4. Plot the model's accuracy at every epoch:

# plotting the metrics
fig = plt.figure(figsize=(12,4))

#plt.subplot(2,1,1)
plt.plot(svhn_model.history['acc'])
plt.plot(svhn_model.history['val_acc'])
plt.title('Model Accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epochs')
plt.legend(['Train', 'Test'], loc='uppper left')

plt.tight_layout()

We see the following model accuracy plot:

5. Plot the loss at every epoch:

# plotting the metrics
fig = plt.figure(figsize=(12,4))

plt.plot(svhn_model.history['loss'])
plt.plot(svhn_model.history['val_loss'])
plt.title('Model Loss')
plt.ylabel('Loss')
plt.xlabel('Epochs')
plt.legend(['Train', 'Test'], loc='upper right')

plt.tight_layout()

We see the following model loss plot:

6. Reuse the code from the scikit-learn website to plot the confusion matrix:

# code from http://scikit-learn.org
def plot_confusion_matrix(cm, classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
"""
This function prints and plots the confusion matrix.
"""
plt.imshow(cm, cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)

thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j, i, cm[i, j],
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")

plt.ylabel('Actuals')
plt.xlabel('Predicted')

7. Plot the confusion matrix both numerically and graphically:

target_names = [ '0', '1', '2', '3', '4', '5', '6', '7', '8', '9']

# Formulating the Confusion Matrix
import itertools
from sklearn.metrics import confusion_matrix

cm = confusion_matrix(y_test_actuals, predicted_classes)
print(cm)

plt.figure(figsize=(10,10))
plot_confusion_matrix(cm, classes=target_names, normalize=False)
plt.show()

The confusion matrix appears as follows:

8. We'll now look at how to ensemble the results of multiple homogeneous models. Define a function to fit the model to the training data:

# fit model on dataset
def train_models(x_train, y_train):
  # building a linear stack of layers with the sequential model
  model = Sequential()
  model.add(Dense(512, input_shape=(1024,)))
  model.add(Activation('relu')) 
  model.add(Dropout(0.2))

  model.add(Dense(512))
  model.add(Activation('relu'))
  model.add(Dropout(0.2))

  model.add(Dense(10))
  model.add(Activation('softmax'))
  
  # compiling the sequential model
  model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
  
  # training the model and saving metrics in history
  svhn_model = model.fit(x_train, y_train, batch_size=32, epochs=25)
  
  return model

9. Write a function to ensemble the predictions of all the models:

# make an ensemble prediction for multi-class classification
def ensemble_predictions(models, x_test):
  # make predictions
  y_predicted = [model.predict(x_test) for model in models]
  y_predicted = np.array(y_predicted)
  
  # sum predictions from all ensemble models
  predicted_total = np.sum(y_predicted, axis=0)
  
  # argmax across classes
  result = np.argmax(predicted_total, axis=1)
  
  return result

10. Write a function to evaluate the models and get the accuracy score of each model:

# evaluate a specific number of members in an ensemble
def evaluate_models(models, no_of_models, x_test, y_test):
 # select a subset of members
 subset = models[:no_of_models]
 
 # make prediction
 y_predicted_ensemble = ensemble_predictions(subset, x_test)
 
 # calculate accuracy
 return accuracy_score(y_test_actuals, y_predicted_ensemble)

11. Fit all the models:

# fit all models
no_of_models = 50

models = [train_models(x_train, y_train) for _ in range(no_of_models)]

# evaluate different numbers of ensembles
all_scores = list()
for i in range(1, no_of_models+1):
  score = evaluate_models(models, i, x_test, y_test)
  print("Accuracy Score of model ", i, " ", score)
  all_scores.append(score)

12. Plot the accuracy score against each epoch:

# plot score vs number of ensemble members
x_axis = [i for i in range(1, no_of_models+1)]
plt.plot(x_axis, all_scores)
plt.show()