Author - Mohit Rathore mrmohitrathoremr@gmail.com - markroxor.in
Licensed under The MIT License - https://opensource.org/licenses/MIT

In [1]:
from fromscratchtoml import svm
import numpy as np

from fromscratchtoml.toolbox import binary_visualize
from fromscratchtoml.toolbox.random import Distribution

%matplotlib inline

Introduction to support vector machines

In [2]:
import sys
print(sys.version)

3.6.5 (default, Apr  1 2018, 05:46:30) 
[GCC 7.3.0]

Linearly seperable data

In [3]:
X1 = Distribution.linear(pts=100,
               mean=[8, 10],
               covr=[[1.5, 1], [1, 1.5]])
X2 = Distribution.linear(pts=100,
               mean=[9, 5],
               covr=[[1.5, 1], [1, 1.5]])

Y1 = np.ones(X1.shape[0])
Y2 = 2*np.ones(X2.shape[0])

X = np.vstack((X1, X2))
y = np.hstack((Y1, Y2))
In [4]:
y.shape, X.shape
Out[4]:
((200,), (200, 2))
In [5]:
clf_lin = svm.SVC(kernel='linear')
clf_lin.fit(X, y)
In [6]:
X1 = Distribution.linear(pts=10,
               mean=[8, 10],
               covr=[[1.5, 1], [1, 1.5]])
X2 = Distribution.linear(pts=10,
               mean=[9, 5],
               covr=[[1.5, 1], [1, 1.5]])
X = np.vstack((X1, X2))
In [7]:
binary_visualize(X, clf=clf_lin, draw_contour=True)

Soft margin on overlapping data

In [8]:
X1 = Distribution.linear(pts=100,
               mean=[8, 10],
               covr=[[1.5, 1], [1, 1.5]])
X2 = Distribution.linear(pts=100,
               mean=[9, 8],
               covr=[[1.5, 1], [1, 1.5]])


Y1 = np.ones(X1.shape[0])
Y2 = 2*np.ones(X2.shape[0])

X = np.vstack((X1, X2))
y = np.hstack((Y1, Y2))
In [10]:
clf_lin = svm.SVC(kernel='linear')
clf_lin.fit(X, y)
In [11]:
X1 = Distribution.linear(pts=100,
               mean=[8, 10],
               covr=[[1.5, 1], [1, 1.5]])
X2 = Distribution.linear(pts=100,
               mean=[9, 8],
               covr=[[1.5, 1], [1, 1.5]])
X = np.vstack((X1, X2))
In [12]:
binary_visualize(X, clf=clf_lin, draw_contour=True)

Non linearly seperable data

In [18]:
X1 = np.array([[10,10],[6,6],[6,11],[3,15],[12,6],[9,5],[16,3],[11,5]])
X2 = np.array([[3,6],[6,3],[2,9],[9,2],[18,1],[1,18],[1,13],[13,1]])

Y1 = np.ones(X1.shape[0])
Y2 = 2*np.ones(X1.shape[0])

X = np.vstack((X1, X2))
y = np.hstack((Y1, Y2))
In [ ]:
X
y
In [14]:
clf = svm.SVC(kernel='polynomial', const=0, degree=2)
clf.fit(X, y)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-14-ed53973aad82> in <module>()
      1 clf = svm.SVC(kernel='polynomial', const=0, degree=2)
----> 2 clf.fit(X, y)

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/fromscratchtoml-0.0.2-py3.6.egg/fromscratchtoml/svm/svc.py in fit(self, X, y, multiplier_threshold)
    114                 clf.C = self.C
    115 
--> 116                 self.classifiers.append(clf.fit(X, y_i))
    117             return
    118 

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/fromscratchtoml-0.0.2-py3.6.egg/fromscratchtoml/svm/svc.py in fit(self, X, y, multiplier_threshold)
    137 
    138         lagrange_multipliers = np.array(list(cvxopt.solvers.qp(P, q, G, h, A,
--> 139                                                                 b)['x']))
    140 
    141         lagrange_multiplier_indices = np.greater_equal(lagrange_multipliers, multiplier_threshold)

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/cvxopt/coneprog.py in qp(P, q, G, h, A, b, solver, kktsolver, initvals, **kwargs)
   4485             'residual as dual infeasibility certificate': dinfres}
   4486 
-> 4487     return coneqp(P, q, G, h, None, A,  b, initvals, kktsolver = kktsolver, options = options)

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/cvxopt/coneprog.py in coneqp(P, q, G, h, dims, A, b, initvals, kktsolver, xnewcopy, xdot, xaxpy, xscal, ynewcopy, ydot, yaxpy, yscal, **kwargs)
   1840         if P.typecode != 'd' or P.size != (q.size[0], q.size[0]):
   1841             raise TypeError("'P' must be a 'd' matrix of size (%d, %d)"\
-> 1842                 %(q.size[0], q.size[0]))
   1843         def fP(x, y, alpha = 1.0, beta = 0.0):
   1844             base.symv(P, x, y, alpha = alpha, beta = beta)

TypeError: 'P' must be a 'd' matrix of size (16, 16)
In [ ]:
binary_visualize(X, clf=clf, draw_contour=True)

RBF

In [24]:
X1 = Distribution.radial_binary(pts=30,
               mean=[0, 0],
               st=1,
               ed=2, seed=20)
X2 = Distribution.radial_binary(pts=30,
               mean=[0, 0],
               st=4,
               ed=5, seed=20)

Y1 = np.ones(X1.shape[0])
Y2 = 2*np.ones(X1.shape[0])

X = np.vstack((X1, X2))
y = np.hstack((Y1, Y2))
In [26]:
clf = svm.SVC(kernel='rbf', gamma=0.2)
clf.fit(X, y)

binary_visualize(X, clf=clf, draw_contour=True)
In [27]:
clf = svm.SVC(kernel='rbf', gamma=10)
clf.fit(X, y)

binary_visualize(X, clf=clf, draw_contour=True)

Multiclass SVC

In [28]:
X1 = Distribution.linear(pts=100,
               mean=[6, 10],
               covr=[[1.5, 1], [1, 1.5]])
X2 = Distribution.linear(pts=100,
               mean=[9, 5],
               covr=[[1.5, 1], [1, 1.5]])
X3 = Distribution.linear(pts=100,
               mean=[-9, -5],
               covr=[[1.5, 1], [1, 1.5]])
X4 = Distribution.linear(pts=100,
               mean=[-6, -10],
               covr=[[1.5, 1], [1, 1.5]])

Y1 = -1*np.ones(X1.shape[0])
Y2 = 1*np.ones(X2.shape[0])
Y3 = 2*np.ones(X3.shape[0])
Y4 = 30*np.ones(X4.shape[0])

X = np.vstack((X1, X2, X3 ,X4))
y = np.hstack((Y1, Y2, Y3, Y4))
In [29]:
clf = svm.SVC(kernel='linear')
classifiers=clf.fit(X, y)
In [30]:
clf.predict(np.array([0, 0]))
Out[30]:
array([0])
In [31]:
X1 = Distribution.linear(pts=30,
               mean=[6, 10],
               covr=[[1.5, 1], [1, 1.5]])
X2 = Distribution.linear(pts=30,
               mean=[9, 5],
               covr=[[1.5, 1], [1, 1.5]])
X3 = Distribution.linear(pts=30,
               mean=[-9, -5],
               covr=[[1.5, 1], [1, 1.5]])
X4 = Distribution.linear(pts=30,
               mean=[-6, -10],
               covr=[[1.5, 1], [1, 1.5]])

Y1 = -np.ones(X1.shape[0])
Y2 = 1*np.ones(X2.shape[0])
Y3 = 2*np.ones(X3.shape[0])
Y4 = 30*np.ones(X4.shape[0])

X = np.vstack((X1, X2, X3 ,X4))
y = np.hstack((Y1, Y2, Y3, Y4))
In [32]:
clf.predict(np.array([0,0]))
Out[32]:
array([0])
In [33]:
binary_visualize(X, clf=clf, draw_contour=True)

Multi class radial classification

In [34]:
X1 = Distribution.radial_binary(pts=50,
               mean=[0, 0],
               st=1,
               ed=2,seed=100)
X3 = Distribution.radial_binary(pts=50,
               mean=[0, 0],
               st=5,
               ed=10,seed=100)
X4 = Distribution.radial_binary(pts=50,
               mean=[0, 0],
               st=8,
               ed=9,seed=100)


Y1 = -np.ones(X1.shape[0])
Y3 = 2*np.ones(X3.shape[0])
Y4 = 3000*np.ones(X4.shape[0])

X = np.vstack([X1, X3, X4])
y = np.hstack((Y1, Y3, Y4))
In [35]:
clf = svm.SVC(kernel='rbf', gamma=10)
clf.fit(X, y)
In [ ]:
In [36]:
X1 = Distribution.radial_binary(pts=30,
               mean=[0, 0],
               st=1,
               ed=2,seed=100)
X3 = Distribution.radial_binary(pts=30,
               mean=[0, 0],
               st=5,
               ed=10,seed=100)
X4 = Distribution.radial_binary(pts=30,
               mean=[0, 0],
               st=8,
               ed=9,seed=100)


Y1 = -np.ones(X1.shape[0])
Y2 = np.ones(X2.shape[0])
Y3 = 2*np.ones(X3.shape[0])
Y4 = 3000*np.ones(X4.shape[0])

X = np.vstack((X1, X2, X3, X4))
y = np.hstack((Y1, Y2, Y3, Y4))
In [37]:
binary_visualize(X, clf=clf, draw_contour=True)

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-37-54657139d9c3> in <module>()
----> 1 binary_visualize(X, clf=clf, draw_contour=True)

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/fromscratchtoml-0.0.2-py3.6.egg/fromscratchtoml/toolbox/__init__.py in binary_visualize(X, y, clf, coarse, xlabel, ylabel, title, draw_contour, color_seed)
    105 
    106         Z = np.c_[_X.ravel(), _Y.ravel()]
--> 107         Z = clf.predict(Z)
    108 
    109         unq, Z = np.unique(Z, return_inverse=True)

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/fromscratchtoml-0.0.2-py3.6.egg/fromscratchtoml/svm/svc.py in predict(self, X, return_projection)
    199             for j, x in enumerate(X):
    200                 for i, clas in enumerate(self.classifiers):
--> 201                     prediction, projection = self.classifiers[i].predict(x, return_projection=True)
    202 
    203                     if int(prediction) == 1:

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/fromscratchtoml-0.0.2-py3.6.egg/fromscratchtoml/svm/svc.py in predict(self, X, return_projection)
    215                 for i in range(self.n_support_vectors):
    216                     projection += self.support_lagrange_multipliers[i] * self.support_vectors_y[i] *\
--> 217                                   self.kernel(self.support_vectors[i], x)
    218                 projections[j] = projection
    219                 predictions[j] = np.sign(projection)

~/Documents/jellAIfish/fromscratchtoml/env/lib/python3.6/site-packages/fromscratchtoml-0.0.2-py3.6.egg/fromscratchtoml/toolbox/kernels.py in rbf(x, y, gamma, **kwargs)
     77     """
     78     euclidean_dist = pow(np.linalg.norm(x - y), 2)
---> 79     return np.exp(- euclidean_dist / (2 * pow(gamma, 2)))

KeyboardInterrupt:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]:
In [ ]: