Logistic Regression
Implementation of classic logistic regression for binary class labels.
Imports
%matplotlib inline
import [Link] as plt
import numpy as np
import torch
import [Link] as F
Preparing a toy dataset
##########################
### DATASET
##########################
data = [Link]('data/[Link]', delimiter='\t')
x = data[:, :2].astype(np.float32)
y = data[:, 2].astype(np.int64)
[Link](123)
idx = [Link]([Link][0])
[Link](idx)
X_test, y_test = x[idx[:25]], y[idx[:25]]
X_train, y_train = x[idx[25:]], y[idx[25:]]
mu, std = [Link](X_train, axis=0), [Link](X_train, axis=0)
X_train, X_test = (X_train - mu) / std, (X_test - mu) / std
fig, ax = [Link](1, 2, figsize=(7, 2.5))
ax[0].scatter(X_train[y_train == 1, 0], X_train[y_train == 1, 1])
ax[0].scatter(X_train[y_train == 0, 0], X_train[y_train == 0, 1])
ax[1].scatter(X_test[y_test == 1, 0], X_test[y_test == 1, 1])
ax[1].scatter(X_test[y_test == 0, 0], X_test[y_test == 0, 1])
[Link]([x[:, 0].min()-0.5, x[:, 0].max()+0.5])
[Link]([x[:, 1].min()-0.5, x[:, 1].max()+0.5])
[Link]()
Low-level implementation with manual gradients
device = [Link]("cuda:0" if [Link].is_available() else "cpu")
class LogisticRegression1():
def __init__(self, num_features):
self.num_features = num_features
[Link] = [Link](1, num_features,
dtype=torch.float32, device=device)
[Link] = [Link](1, dtype=torch.float32, device=device)
def forward(self, x):
linear = [Link]([Link](x, [Link].t()), [Link]).view(-1) # net input
probas = self._sigmoid(linear)
return probas
def backward(self, x, y, probas):
grad_loss_wrt_z = [Link](-1) - y
grad_loss_wrt_w = [Link](x.t(), grad_loss_wrt_z.view(-1, 1)).t()
grad_loss_wrt_b = [Link](grad_loss_wrt_z)
return grad_loss_wrt_w, grad_loss_wrt_b
def predict_labels(self, x):
probas = [Link](x)
labels = [Link](probas >= .5, 1, 0) # threshold function
return labels
def evaluate(self, x, y):
labels = self.predict_labels(x).float()
accuracy = [Link]([Link](-1) == [Link]()).item() / [Link](0)
return accuracy
def _sigmoid(self, z):
return 1. / (1. + [Link](-z))
def _logit_cost(self, y, proba):
tmp1 = [Link](-[Link](1, -1), [Link]([Link](-1, 1)))
tmp2 = [Link]((1 - y).view(1, -1), [Link](1 - [Link](-1, 1)))
return tmp1 - tmp2
def train(self, x, y, num_epochs, learning_rate=0.01):
epoch_cost = []
for e in range(num_epochs):
#### Compute outputs ####
probas = [Link](x)
#### Compute gradients ####
grad_w, grad_b = [Link](x, y, probas)
#### Update weights ####
[Link] -= learning_rate * grad_w
[Link] -= learning_rate * grad_b
#### Logging ####
cost = self._logit_cost(y, [Link](x)) / [Link](0)
print('Epoch: %03d' % (e+1), end="")
print(' | Train ACC: %.3f' % [Link](x, y), end="")
print(' | Cost: %.3f' % cost)
epoch_cost.append(cost)
return epoch_cost
X_train_tensor = [Link](X_train, dtype=torch.float32, device=device)
y_train_tensor = [Link](y_train, dtype=torch.float32, device=device)
model1 = LogisticRegression1(num_features=2)
epoch_cost = [Link](X_train_tensor, y_train_tensor, num_epochs=30, learning_rate=0.1)
print('\nModel parameters:')
print(' Weights: %s' % [Link])
print(' Bias: %s' % [Link])
Evaluating the Model
[Link](epoch_cost)
[Link]('Neg. Log Likelihood Loss')
[Link]('Epoch')
[Link]()
X_test_tensor = [Link](X_test, dtype=torch.float32, device=device)
y_test_tensor = [Link](y_test, dtype=torch.float32, device=device)
test_acc = [Link](X_test_tensor, y_test_tensor)
print('Test set accuracy: %.2f%%' % (test_acc*100))
Test set accuracy: 96.00%
##########################
### 2D Decision Boundary
##########################
w, b = [Link](-1), [Link]
x_min = -2
y_min = ( (-(w[0] * x_min) - b[0])
/ w[1] )
x_max = 2
y_max = ( (-(w[0] * x_max) - b[0])
/ w[1] )
fig, ax = [Link](1, 2, sharex=True, figsize=(7, 3))
ax[0].plot([x_min, x_max], [y_min, y_max])
ax[1].plot([x_min, x_max], [y_min, y_max])
ax[0].scatter(X_train[y_train==0, 0], X_train[y_train==0, 1], label='class 0', marker='o')
ax[0].scatter(X_train[y_train==1, 0], X_train[y_train==1, 1], label='class 1', marker='s')
ax[1].scatter(X_test[y_test==0, 0], X_test[y_test==0, 1], label='class 0', marker='o')
ax[1].scatter(X_test[y_test==1, 0], X_test[y_test==1, 1], label='class 1', marker='s')
ax[1].legend(loc='upper left')
[Link]()
High-level implementation using the [Link] API
class LogisticRegression2([Link]):
def __init__(self, num_features):
super(LogisticRegression2, self).__init__()
[Link] = [Link](num_features, 1)
# initialize weights to zeros here,
# since we used zero weights in the
# manual approach
[Link]().zero_()
[Link]().zero_()
# Note: the trailing underscore
# means "in-place operation" in the context
# of PyTorch
def forward(self, x):
logits = [Link](x)
probas = [Link](logits)
return probas
model2 = LogisticRegression2(num_features=2).to(device)
optimizer = [Link]([Link](), lr=0.1)
def comp_accuracy(label_var, pred_probas):
pred_labels = [Link]((pred_probas > 0.5), 1, 0).view(-1)
acc = [Link](pred_labels == label_var.view(-1)).float() / label_var.size(0)
return acc
num_epochs = 30
X_train_tensor = [Link](X_train, dtype=torch.float32, device=device)
y_train_tensor = [Link](y_train, dtype=torch.float32, device=device).view(-1, 1)
for epoch in range(num_epochs):
#### Compute outputs ####
out = model2(X_train_tensor)
#### Compute gradients ####
loss = F.binary_cross_entropy(out, y_train_tensor, reduction='sum')
optimizer.zero_grad()
[Link]()
#### Update weights ####
[Link]()
#### Logging ####
pred_probas = model2(X_train_tensor)
acc = comp_accuracy(y_train_tensor, pred_probas)
print('Epoch: %03d' % (epoch + 1), end="")
print(' | Train ACC: %.3f' % acc, end="")
print(' | Cost: %.3f' % F.binary_cross_entropy(pred_probas, y_train_tensor))
print('\nModel parameters:')
print(' Weights: %s' % [Link])
print(' Bias: %s' % [Link])
Evaluating the Model
X_test_tensor = [Link](X_test, dtype=torch.float32, device=device)
y_test_tensor = [Link](y_test, dtype=torch.float32, device=device)
pred_probas = model2(X_test_tensor)
test_acc = comp_accuracy(y_test_tensor, pred_probas)
print('Test set accuracy: %.2f%%' % (test_acc*100))
Test set accuracy: 96.00%
##########################
### 2D Decision Boundary
##########################
w, b = [Link]().view(-1), [Link]()
x_min = -2
y_min = ( (-(w[0] * x_min) - b[0])
/ w[1] )
x_max = 2
y_max = ( (-(w[0] * x_max) - b[0])
/ w[1] )
fig, ax = [Link](1, 2, sharex=True, figsize=(7, 3))
ax[0].plot([x_min, x_max], [y_min, y_max])
ax[1].plot([x_min, x_max], [y_min, y_max])
ax[0].scatter(X_train[y_train==0, 0], X_train[y_train==0, 1], label='class 0', marker='o')
ax[0].scatter(X_train[y_train==1, 0], X_train[y_train==1, 1], label='class 1', marker='s')
ax[1].scatter(X_test[y_test==0, 0], X_test[y_test==0, 1], label='class 0', marker='o')
ax[1].scatter(X_test[y_test==1, 0], X_test[y_test==1, 1], label='class 1', marker='s')
ax[1].legend(loc='upper left')
[Link]()
Loading [MathJax]/jax/output/CommonHTML/fonts/TeX/[Link]
