# Homework 3 - main file
# COMP.4220 Machine Learning
import itertools, functools
import numpy as np
import matplotlib.pyplot as plt
from regression import LinearRegression, RidgeRegression
from sklearn.datasets import fetch_california_housing
from sklearn.linear_model import Ridge as skRidge
from sklearn.linear_model import LinearRegression as skLinearRegression
from sklearn.metrics import root_mean_squared_error, r2_score
from sklearn.preprocessing import StandardScaler as skStandardScalar
from sklearn.preprocessing import StandardScaler
def train_test_split(X, t, test_size=0.2, random_state=None):
"""Splits data into training and testing sets using only NumPy."""
if random_state:
np.random.seed(random_state)
# ---- Part (d) ---- #
# 1. Shuffle the data
indices = np.arange(1)
X = X
t = t
# 2. Split the data
#split_index = 1
X_train = X
X_test = []
t_train = t
t_test = []
return X_train, X_test, t_train, t_test
def standardscalar(x: np.ndarray):
# ---- Part (b) ---- #
Xs = X
Xs = standardscalar(X)
return (x - np.mean(x, axis=0)) / np.std(x, axis=0)
class PolynomialFeature(object):
def __init__(self, degree=2):
assert isinstance(degree, int)
self.degree = degree
def transform(self, x):
if x.ndim == 1:
x = x[:, None]
x_t = x.transpose()
features = [np.ones(len(x))]
for degree in range(1, self.degree + 1):
for items in itertools.combinations_with_replacement(x_t, degree):
features.append(functools.reduce(lambda x, y: x * y, items))
return np.asarray(features).transpose()
def main():
# ---- Part (a) ---- #
housing = fetch_california_housing()
X = housing.data
t = housing.target
print(X.shape, t.shape)
print(housing.feature_names[:6])
print(housing.DESCR)
# ---- Part (b) ---- #
def standardscalar(x):
mean = np.mean(x, axis=0)
std = np.std(x, axis=0)
return (x - mean) / std
X_scaled = standardscalar(X)
# Standardize the data using sklearn's StandardScaler
scaler = StandardScaler()
X_sklearn_scaled = scaler.fit_transform(X)
print(np.allclose(X_scaled, X_sklearn_scaled)) # Should be True
print(np.allclose(np.mean(X_scaled, axis=0), np.mean(X_sklearn_scaled,
axis=0))) # Should be True
print(np.allclose(np.std(X_scaled, axis=0), np.std(X_sklearn_scaled, axis=0)))
# Should be True
# ---- Part (c) ---- #
X_train, X_test, t_train, t_test = train_test_split(X, t, test_size=0.2,
random_state=42)
scaler = StandardScaler()
X_sk_standardized = scaler.fit_transform(X)
print((Xs - Xss))
# ---- Part (d) ---- #
class RidgeRegression:
def __init__(self, lambda_):
self.lambda_ = lambda_
def fit(self, X, t):
# Add the regularization term (lambda * I)
I = np.identity(X.shape[1])
self.w = np.linalg.inv(X.T @ X + self.lambda_ * I) @ X.T @ t
def predict(self, X):
return X @ self.w
X_train, X_test, t_train, t_test = [], [], [], []
# ---- Part (k) ---- #
linreg = LinearRegression()
linreg.fit(X_train, t_train)
# Predictions and evaluation for Linear Regression
t_pred_linreg = linreg.predict(X_test)
rmse_linreg = np.sqrt(mean_squared_error(t_test, t_pred_linreg))
r2_linreg = r2_score(t_test, t_pred_linreg)
print(f'Linear Regression RMSE: {rmse_linreg}')
print(f'Linear Regression R²: {r2_linreg}')
# Compare the performance with the best Ridge model (with lambda = 1.0)
best_ridge_model = ridge_models[lambdas.index(1.0)]
t_pred_best_ridge = best_ridge_model.predict(X_test)
rmse_best_ridge = np.sqrt(mean_squared_error(t_test, t_pred_best_ridge))
r2_best_ridge = r2_score(t_test, t_pred_best_ridge)
print(f'Best Ridge Regression (lambda=1.0) RMSE: {rmse_best_ridge}')
print(f'Best Ridge Regression (lambda=1.0) R²: {r2_best_ridge}')
# Comparing the results
print("\nComparison of Linear Regression and Ridge Regression (lambda=1.0):")
print(f"Linear Regression RMSE: {rmse_linreg}, R²: {r2_linreg}")
print(f"Ridge Regression (lambda=1.0) RMSE: {rmse_best_ridge}, R²:
{r2_best_ridge}")
# ---- Part (g, h) ---- #
# (g) Plotting true vs predicted for both models
plt.figure(figsize=(12, 6))
# Plot for Ridge Regression
plt.subplot(1, 2, 1)
plt.scatter(t_test, t_pred, color='blue')
plt.plot([t_test.min(), t_test.max()], [t_test.min(), t_test.max()],
color='red')
plt.title('Ridge Regression: True vs Predicted')
# Plot for Linear Regression
plt.subplot(1, 2, 2)
plt.scatter(t_test, t_pred_linreg, color='green')
plt.plot([t_test.min(), t_test.max()], [t_test.min(), t_test.max()],
color='red')
plt.title('Linear Regression: True vs Predicted')
plt.show()
# (h) Try different values of regularization lambda for Ridge regression
ridge_2 = RidgeRegression(lambda_=10.0)
ridge_2.fit(X_train, t_train)
t_pred_ridge_2 = ridge_2.predict(X_test)
# Evaluate new Ridge regression
rmse_ridge_2 = np.sqrt(mean_squared_error(t_test, t_pred_ridge_2))
r2_ridge_2 = r2_score(t_test, t_pred_ridge_2)
print(f'Ridge Regression (lambda=10) RMSE: {rmse_ridge_2}')
print(f'Ridge Regression (lambda=10) R²: {r2_ridge_2}')
# Model building
lr = LinearRegression()
y_lr = []
print('Linear Regression results')
print(f'RMSE: {np.inf}')
print(f'R2: {np.inf}')
rr = RidgeRegression(lambd=1.0)
y_rr = []
print('Ridge Regression results')
print(f'RMSE: {np.inf}')
print(f'R2: {np.inf}')
# ---- Part (i) ---- #
# (i) Ridge Regression for different lambda values
lambdas = [0.1, 1.0, 10.0, 100.0]
ridge_models = []
rmse_values = []
r2_values = []
for lambda_val in lambdas:
ridge_model = RidgeRegression(lambda_=lambda_val)
ridge_model.fit(X_train, t_train)
t_pred_ridge = ridge_model.predict(X_test)
rmse_ridge = np.sqrt(mean_squared_error(t_test, t_pred_ridge))
r2_ridge = r2_score(t_test, t_pred_ridge)
ridge_models.append(ridge_model)
rmse_values.append(rmse_ridge)
r2_values.append(r2_ridge)
lr_sk = skLinearRegression()
y_lr_sk = []
print('Sklearn Linear Regression results')
print(f'RMSE: {np.inf}')
print(f'R2: {np.inf}')
rr_sk = skRidge(alpha=1.0)
y_rr_sk = []
print('Sklearn Ridge Regression results')
print(f'RMSE: {np.inf}')
print(f'R2: {np.inf}')
# ---- Part (j) ---- #
plt.figure(figsize=(12, 6))
# Plot RMSE
plt.subplot(1, 2, 1)
plt.plot(lambdas, rmse_values, marker='o', linestyle='-', color='b')
plt.xscale('log')
plt.title('RMSE for Ridge Regression with different lambda values')
plt.xlabel('Lambda')
plt.ylabel('RMSE')
# Plot R²
plt.subplot(1, 2, 2)
plt.plot(lambdas, r2_values, marker='o', linestyle='-', color='r')
plt.xscale('log')
plt.title('R² for Ridge Regression with different lambda values')
plt.xlabel('Lambda')
plt.ylabel('R²')
plt.show()
# Plot the results
plt.figure(figsize=(12, 6))
plt.subplot(2, 2, 1)
# use scatter and plot to show the results
plt.xlabel('add a proper label')
plt.ylabel('add a proper label')
plt.title('add a proper title')
plt.subplot(2, 2, 2)
# use scatter and plot to show the results
plt.xlabel('add a proper label')
plt.ylabel('add a proper label')
plt.title('add a proper title')
plt.subplot(2, 2, 3)
# use scatter and plot to show the results
plt.xlabel('add a proper label')
plt.ylabel('add a proper label')
plt.title('add a proper title')
plt.subplot(2, 2, 4)
# use scatter and plot to show the results
plt.xlabel('add a proper label')
plt.ylabel('add a proper label')
plt.title('add a proper title')
plt.tight_layout()
plt.show()
if __name__=='__main__':
main()
