Author - Mohit Rathore mrmohitrathoremr@gmail.com - markroxor.in
Licensed under The MIT License - https://opensource.org/licenses/MIT

In [1]:
import fromscratchtoml
fromscratchtoml.use("numpy")
# fromscratchtoml.use("cupy")

from fromscratchtoml.neural_network.models import Sequential
from fromscratchtoml.neural_network.optimizers import StochasticGradientDescent, Adagrad, RMSprop, Adam
from fromscratchtoml.neural_network.layers import Dense, Activation, Dropout
from fromscratchtoml.neural_network.regularizers import l1, l2, l1_l2

import numpy as np
# import cupy as np

from sklearn.model_selection import train_test_split

from fromscratchtoml.toolbox.random import Distribution
from fromscratchtoml.toolbox.preprocess import to_onehot
In [2]:
from sklearn import datasets
from sklearn.utils import shuffle

from fromscratchtoml.toolbox import binary_visualize

%matplotlib inline

Neural Networks

MNIST Digit Classification

In [3]:
from sklearn.datasets import fetch_mldata
mnist = fetch_mldata('MNIST original', data_home=".")
print(len(mnist.data))
X2 = mnist.data/256.
mnist.target.__class__
y2 = to_onehot(mnist.target)

70000
In [4]:
X_train2, X_test2, y_train2, y_test2 = train_test_split(X2, y2, test_size=10000, random_state=42)
X_train2.shape, y_train2.shape, X_train2.dtype
Out[4]:
((60000, 784), (60000, 10), dtype('float64'))
In [5]:
model2 = Sequential(verbose=1,vis_loss=1)

model2.add(Dense(30, input_dim=784, seed=1))
model2.add(Activation('relu'))

model2.add(Dense(10, seed=2))
model2.add(Activation('softmax'))


opt = StochasticGradientDescent(learning_rate=0.08, momentum=0.8, nesterov=1)
model2.compile(optimizer=opt, loss="cross_entropy")
In [6]:
%time model2.fit(X_train2, y_train2, batch_size=256, epochs=10)

epoch: 1/10  loss: 0.447  acc: 92.38 
[████                                    ] 10% 
epoch: 2/10  loss: 0.357  acc: 94.07 
[████████                                ] 20% 
epoch: 3/10  loss: 0.306  acc: 94.93 
[████████████                            ] 30% 
epoch: 4/10  loss: 0.271  acc: 95.51 
[████████████████                        ] 40% 
epoch: 5/10  loss: 0.245  acc: 95.99 
[████████████████████                    ] 50% 
epoch: 6/10  loss: 0.226  acc: 96.36 
[████████████████████████                ] 60% 
epoch: 7/10  loss: 0.209  acc: 96.62 
[████████████████████████████            ] 70% 
epoch: 8/10  loss: 0.197  acc: 96.82 
[████████████████████████████████        ] 80% 
epoch: 9/10  loss: 0.186  acc: 96.98 
[████████████████████████████████████    ] 90% 
epoch: 10/10  loss: 0.176  acc: 97.14 
[████████████████████████████████████████] 100% 
CPU times: user 50.4 s, sys: 38.9 s, total: 1min 29s
Wall time: 12.2 s
In [7]:
model2.accuracy(X_test2, y_test2)
Out[7]:
96.04

Regression

In [8]:
x_train = np.random.rand(100)
x_train = np.expand_dims(x_train, axis=1)
y_train=x_train**4

x_test=np.linspace(0,1,9)
y_test=x_test**4
In [9]:
model3 = Sequential(verbose=0, vis_loss=1)
model3.add(Dense(units=100, input_dim=1, seed=10))
model3.add(Activation('relu'))
model3.add(Dense(units=45, seed=2))
model3.add(Activation('relu'))
model3.add(Dense(units=1, seed=2))

sgd = StochasticGradientDescent(learning_rate=0.05)
model3.compile(optimizer=sgd, loss="mean_squared_error", accuracy_metric="regression")

%time model3.fit(x_train, y_train, epochs=1000)

[███████████████████████████████████████ ] 99% 
epoch: 1000/1000  loss: 0.000  acc: 100.00 
[████████████████████████████████████████] 100% 
CPU times: user 2min 17s, sys: 3min 42s, total: 6min
Wall time: 52.2 s
In [10]:
y_pred = model3.predict(x_test, prob=1)
y_pred[:5]
Out[10]:
array([[-0.00370739],
       [ 0.00037773],
       [ 0.00414967],
       [ 0.01897731],
       [ 0.067524  ]])
In [11]:
y_test[:5]
Out[11]:
array([0.        , 0.00024414, 0.00390625, 0.01977539, 0.0625    ])

Radial

In [12]:
X11 = Distribution.radial_binary(pts=300,
               mean=[0, 0],
               st=1,
               ed=2, seed=20)
X22 = Distribution.radial_binary(pts=300,
               mean=[0, 0],
               st=4,
               ed=5, seed=20)

Y11 = np.ones(X11.shape[0])
Y22 = np.zeros(X11.shape[0])

X5 = np.vstack((X11, X22))
y5 = np.hstack((Y11, Y22))
In [ ]:
In [13]:
y5 = to_onehot(y5)
In [14]:
X_train5, X_test5, y_train5, y_test5 = train_test_split(X5, y5, test_size=50, random_state=42)
In [15]:
y_train5.shape
Out[15]:
(550, 2)
In [16]:
model5 = Sequential(verbose=1, vis_each_epoch=True)
model5.add(Dense(10, input_dim=2, seed=1))
model5.add(Activation('sigmoid'))
model5.add(Dense(2, seed=2))
model5.add(Activation('sigmoid'))
model5.add(Dense(2, seed=3))
model5.add(Activation('sigmoid'))
sgd = StochasticGradientDescent(learning_rate=0.9)
model5.compile(optimizer=sgd, loss="mean_squared_error")
In [17]:
X_train5[0]
Out[17]:
array([-0.81275208,  1.01547833])
In [18]:
%time model5.fit(X_train5, y_train5, batch_size=4, epochs=10)

epoch: 1/10  loss: 0.250  acc: 46.00 

[████                                    ] 10% 
epoch: 2/10  loss: 0.249  acc: 53.64 

[████████                                ] 20% 
epoch: 3/10  loss: 0.248  acc: 52.73 

[████████████                            ] 30% 
epoch: 4/10  loss: 0.246  acc: 56.18 

[████████████████                        ] 40% 
epoch: 5/10  loss: 0.241  acc: 58.18 

[████████████████████                    ] 50% 
epoch: 6/10  loss: 0.232  acc: 62.91 

[████████████████████████                ] 60% 
epoch: 7/10  loss: 0.220  acc: 66.91 

[████████████████████████████            ] 70% 
epoch: 8/10  loss: 0.206  acc: 71.27 

[████████████████████████████████        ] 80% 
epoch: 9/10  loss: 0.172  acc: 78.55 

[████████████████████████████████████    ] 90% 
epoch: 10/10  loss: 0.078  acc: 98.73 

[████████████████████████████████████████] 100% CPU times: user 9.96 s, sys: 12.3 s, total: 22.3 s
Wall time: 4.51 s
In [19]:
binary_visualize(X_test5, clf=model5, draw_contour=True)