recommendationsystem.py

# -*- coding: utf-8 -*-
"""RecommendationSystem.ipynb

Automatically generated by Colab.

Original file is located at
    https://colab.research.google.com/drive/14usHdw3xrcVZ7UR6vjw_9-QY8-6V1sL5
"""

# install
!pip install pmdarima
!pip install surprise

# Import Packages
import pandas as pd
import numpy as np

# Reading the dataset
from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
csvFile = "/content/gdrive/MyDrive/Colab Notebooks/kz.csv"

# reading the csv file to different variables
df_data = pd.read_csv(csvFile)
data2 = pd.read_csv(csvFile)
data = pd.read_csv(csvFile)
data3 = pd.read_csv(csvFile)

# Loading the dataset
df_data['event_time'] = df_data['event_time'].replace(" UTC","", regex=True)
df_data['event_time'] = pd.to_datetime(df_data['event_time'])
df_data

# Data transaformation
# Split the categories into sub categories
df_data[['l1_cat', 'l2_cat', 'l3_cat']] = df_data['category_code'].str.split('.',expand=True)

df_data

# Describe Data + some data cleaning
df_data.info()

df_data.describe()

# calculate percentage of missing value
df_data.isnull().sum()/len(df_data)

print("Min Date: ", df_data['event_time'].min())
print("Max Date: ", df_data['event_time'].max())

df_data['year'] = df_data['event_time'].dt.year
df_data.groupby(['year'])['year'].count()

df_data = df_data[df_data['year'] != 1970].reset_index(drop=True)

df_data = df_data[df_data['user_id'].notna()].reset_index(drop=True)

# Handling missing price values
products_median_prices = df_data.groupby(['product_id'])['price'].median().reset_index()
products_median_prices = pd.Series(df_data['price'].values, index=df_data['product_id']).to_dict()

# fill in missing prices with the median
df_data['price'] = df_data['price'].fillna(df_data['product_id'].map(products_median_prices))
df_data['price'].isna().sum()



# Import libraries - 3
import os
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib
import matplotlib.pyplot as plt

import warnings
warnings.filterwarnings('ignore')

# suppress scientific notation
np.set_printoptions(suppress=True)
pd.options.display.float_format = '{:.2f}'.format

data.head()

data.info()

data.describe(include='all').T

# Data preprocessing
data = data.drop_duplicates()

data.isnull().sum()

# Filter rows where 'event_time' contains '1970'
filtered_df = data[data['event_time'].str.contains('1970')]

filtered_df

# exclude rows where 'event_time' contains '1970'
data = data[~data['event_time'].str.contains('1970')]
data.describe(include='all').T

# remove empty rows in brand and user_id columns
data = data.dropna(subset=['brand', 'user_id']).reset_index(drop=True)
data.isnull().sum()

# split the category_code column into category and product columns
data[['category', 'product']] = data['category_code'].str.split('.', n=1, expand=True)

# fill empty cells in the new category and product columns with unknown
data['category'].fillna('unknown', inplace=True)
data['product'].fillna('unknown', inplace=True)

# drop the category_code column
data.drop('category_code', axis=1, inplace=True)

data.tail()

data['category_id'] = data['category_id'].astype('int64')
data['user_id'] = data['user_id'].astype('int64')

# remove UTC from event_time
data['event_time'] = data['event_time'].str.replace('UTC', '')

# create date column
data['date'] = data.event_time.apply(lambda x: x.split(' ')[0])

# convert to datetime object
data['date'] = pd.to_datetime(data['date'])

data.info()

data.head()

# create new columns - date, month, hour and day_of_week
data['month'] = data.date.dt.strftime('%b')
data['month_num'] = data.date.dt.month

data['week_day'] = data.date.dt.strftime('%a')
data['week_day_num'] = data.date.apply(lambda x: x.strftime('%w')).astype('int64')

data['hour'] = data.event_time.apply(lambda x: x.split(' ')[1].split(':')[0]).astype('int64')

del data['event_time']

data.head()

data.describe().T

# General Analysis
# total unique users
total_users = data['user_id'].nunique()
print(f'The total unique users in the dataset are: {total_users: 0,}')

# total unique orders
total_orders = data['order_id'].nunique()
print(f'The total unique orders in the dataset are: {total_orders: 0,}')

# total sales
total_sales = round(data['price'].sum())
print(f'The total sales in $ is: {total_sales: 0,}')

# Time Analysis
df_month = data.groupby(['month_num', 'month']).agg(
    total_users=('user_id', 'nunique'),
    total_orders=('order_id', 'nunique'),
    total_sales= ('price', 'sum')
).sort_values(by='month_num', ascending=True).reset_index(level='month_num', drop=True)

df_month = df_month.reset_index()
df_month

plt.figure(figsize=(18, 6))

for i, col in enumerate(df_month.columns):
  if col != 'month':
    fig = plt.subplot(1, 3, i)
    sns.lineplot(data=df_month, x='month', y=col, )
    plt.title(f'{col}')

    if col == 'total_sales':
      fig.get_yaxis().set_major_formatter(matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))

      # after plotting the data, format the labels
      current_values = plt.gca().get_yticks()

      plt.gca().set_yticklabels(['{:,.0f}'.format(x/1000000) for x in current_values])
      plt.title(f'{col} (in Millions)')

plt.show()

# Weekly Analysis
df_week = data.groupby(['week_day_num', 'week_day']).agg(
    total_users=('user_id', 'nunique'),
    total_orders=('order_id', 'nunique'),
    total_sales=('price', 'sum')
).sort_values(by='week_day_num', ascending=True).reset_index(level='week_day_num', drop=True)

df_week = df_week.reset_index()
df_week

plt.figure(figsize=(18, 6))

for i, col in enumerate(df_week.columns):

    if col != 'week_day':

        fig = plt.subplot(1, 3, i)
        sns.lineplot(data=df_week, x='week_day', y=col, )
        plt.title(f'{col}')

        if col == 'total_sales':

            fig.get_yaxis().set_major_formatter(
            matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))

            # after plotting the data, format the labels
            current_values = plt.gca().get_yticks()

            plt.gca().set_yticklabels(['{:,.0f}'.format(x/1000000) for x in current_values])
            plt.title(f'{col} (in Millions)')

plt.show()

# Daily analysis
df_date = data.groupby('date').agg(
    total_users=('user_id', 'nunique'),
    total_orders=('order_id', 'nunique'),
    total_sales=('price', 'sum')
).reset_index()

df_date

fig, ax = plt.subplots(figsize=(18, 6))
xticks = df_date.date

ax.plot(df_date.date, df_date['total_users'])

ax.set_xlabel('Date')
ax.set_ylabel('Total Users')

ax.legend(['Total Users'])
plt.title('Users by Date', fontsize=15)

plt.tight_layout()
plt.show()

fig, ax = plt.subplots(figsize=(18, 6))
plt.ticklabel_format(style='plain')
xticks = df_date.date

ax.plot(df_date.date, df_date['total_sales'])

ax.set_xlabel('Date')
ax.set_ylabel('Total Sales')

ax.legend(['Total Sales'])
plt.title('Sales by Date', fontsize=15)

plt.tight_layout()
plt.show()

# Add a second y-axis
fig, ax1 = plt.subplots(figsize=(18, 6))
xticks = df_date.date

ax2 = ax1.twinx()
ax1.plot(df_date.date, df_date['total_orders'], color='b')
ax2.plot(df_date.date, df_date['total_sales'], color='r')

# format the second y-axis labels
ax2.get_yaxis().set_major_formatter(
matplotlib.ticker.FuncFormatter(lambda x, p: format(int(x), ',')))

current_values = plt.gca().get_yticks()
plt.gca().set_yticklabels(['{:,.1f}'.format(x/1000000) for x in current_values])

ax1.set_xlabel('Date')
ax1.set_ylabel('Total Orders', color='b')
ax2.set_ylabel('Total Sales', color='r')

ax1.legend(['Total Orders'], loc='upper left')
ax2.legend(['Total Sales (in Millions)'], loc='upper right')

plt.title('Total Orders vs Total Sales By Date', fontsize=15)

plt.show()

# Customer Analysis
# average customer spending
avg_user_rev = round(data['price'].sum() / data['user_id'].nunique())
print(f'The Average Customer Sales in $: {avg_user_rev: 0,}')

# average customer orders
avg_user_orders = round(data['order_id'].nunique() / data['user_id'].nunique(), 0)
print(f'The average customer orders are: {avg_user_orders}')

# average order value
avg_order_value = round(data['price'].sum() / data['order_id'].nunique())
print(f'The average order value in $: {avg_order_value}')

# top 20 customers
top_20_customers = data.groupby('user_id').agg(
    total_orders=('order_id', 'count'),
    total_sales=('price', 'sum')
).reset_index().sort_values('total_orders', ascending=False).head(20)
top_20_customers

# Product Analysis
# top 10 categories
print('Unique Categories', data['category'].nunique())

# by total orders
orders_category = data[data['category'] != 'unknown'].groupby('category').agg(
    total_orders=('order_id', 'count')
).reset_index().sort_values('total_orders', ascending=False).reset_index(drop=True)
orders_category.head(10)

# plot the pie plot for the top 5 categories by orders
plt.rcParams.update({'font.size': 12, 'figure.facecolor': 'white'})
# extract the top 5 categories as labels
labels = orders_category['category'][:5]
#only explode the 1st slice i.e. electronics
explode = (0.1, 0, 0, 0, 0)

fig, ax = plt.subplots(figsize=(8, 8))
ax.pie(x=orders_category.total_orders[:5], explode=explode, labels=labels, autopct='%1.1f%%', startangle=150)

plt.title('Top 5 Categories by Total Orders')
plt.tight_layout()
plt.show()

# by total sales
sales_category = data[data['category'] != 'unknown'].groupby('category').agg(
    total_sales=('price', 'sum')
).reset_index().sort_values('total_sales', ascending=False).reset_index(drop=True)
sales_category.head(10)

# Plot the pie plot for the top 5 categories by sales
# extract the top 5 categories as labels
labels = sales_category['category'][:5]
# only "explode" the 1st slice (i.e. 'electronics')
explode = (0.1, 0, 0, 0, 0)

fig, ax = plt.subplots(figsize=(8, 8))
ax.pie(x=sales_category.total_sales[:5], explode=explode, labels=labels, autopct='%1.1f%%', startangle=150)

plt.title('Top 5 Categories by Total Sales')
plt.tight_layout()
plt.show()

# top 10 brands
print('The total number of brands sold: ', data['brand'].nunique())

df_brand = data.groupby('brand').agg(
    total_users=('user_id', 'nunique'),
    total_orders = ('order_id', 'nunique'),
    total_sales = ('price', 'sum')
).reset_index().sort_values(by='total_sales', ascending=False).reset_index(drop=True)
df_brand.head(10)

# Plot the pie plot for top 5 brands by sales
# extract the top 5 catories as labels
labels = df_brand['brand'][:5]
# only "explode" the 1st slice (i.e. 'Samsung')
explode = (0.1, 0, 0, 0, 0)

fig, ax = plt.subplots(figsize=(8, 8))
ax.pie(x=df_brand.total_sales[:5], explode=explode,labels=labels, autopct='%1.1f%%', startangle=90)

plt.title('Top 5 Brands by Total Sales')
plt.tight_layout()
plt.show()

# top 10 products
print('The total number of unique products: ', data['product'].nunique())
print('The total number of product sold: ', data['product_id'].nunique())

df_product = data.groupby(['product', 'brand']).agg(
    total_users=('user_id', 'nunique'),
    total_orders=('order_id', 'nunique'),
    total_sales=('price', 'sum')
).reset_index().sort_values(by='total_sales', ascending=False).reset_index(drop=True)
df_product.head(10)

df_product_1 = data[data['product'] != 'unknown'].groupby('product').agg(
    total_users=('user_id', 'nunique'),
    total_orders=('order_id', 'nunique'),
    total_sales=('price', 'sum')
).reset_index().sort_values(by='total_sales', ascending=False). reset_index(drop=True)
df_product_1.head(10)

# Plot the pie plot for top 5 products by sales
# extract the top 5 catories as labels
labels = df_product_1['product'][:5]
# only "explode" the 1st slice (i.e. 'smartphone')
explode = (0.1, 0, 0, 0, 0)

fig, ax = plt.subplots(figsize=(8, 8))
ax.pie(x=df_product_1.total_sales[:5], explode=explode, labels=labels, autopct='%1.1f%%', startangle=90)

plt.title('Top 5 Products by Total Sales')
plt.tight_layout()
plt.show()

# Top 5 selling smart phone brands
df_smartphone = data[data['product'] == 'smartphone'].groupby('brand').agg(
    total_users=('user_id', 'nunique'),
    total_orders=('order_id', 'nunique'),
    total_sales=('price', 'sum')
).reset_index().sort_values(by='total_sales', ascending=False).reset_index(drop=True)
df_smartphone.head()

# Plot the top 5 smart phone brands by total sales
# Extract the top 10 category codes as labels
labels = df_smartphone.brand[:5]
# only "explode" the 1st slice (i.e. 'Apple')
explode = (0.1, 0, 0, 0, 0)

fig, ax = plt.subplots(figsize=(8, 8))
ax.pie(x=df_smartphone.total_sales[:5], explode=explode,labels=labels, autopct='%1.1f%%', startangle=90)

plt.title('Top 5 Smart Phone Brands by Total Sales')
plt.tight_layout()
plt.show()

# Plot the top 5 smart phones by total orders
# Extract the top 10 category codes as labels
labels = df_smartphone.brand[:5]
# only "explode" the 2nd slice (i.e. 'Samsung')
explode = (0, 0.1, 0, 0, 0)

fig, ax = plt.subplots(figsize=(8, 8))
ax.pie(x=df_smartphone.total_orders[:5], explode=explode,labels=labels, autopct='%1.1f%%', startangle=90)

plt.title('Top 5 Smart Phone Brands by Total Orders')
plt.tight_layout()
plt.show()



# Import libraries - 2
import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt

from pmdarima.arima import auto_arima
from sklearn.metrics import mean_squared_error
from pandas.plotting import autocorrelation_plot

from statsmodels.tsa.arima.model import ARIMA
from statsmodels.tsa.stattools import adfuller, kpss
from statsmodels.tsa.seasonal import seasonal_decompose
from statsmodels.graphics.tsaplots import plot_acf, plot_pacf
from statsmodels.nonparametric.smoothers_lowess import lowess

import warnings
warnings.filterwarnings('ignore')

# drop all other columns except the two below
data2 = data2[['event_time', 'price']]
data2.info()

# data preprocessing
data2 = data2.drop_duplicates()
data2.isnull().sum()

# remove rows where 'price' contains NaNs
data2 = data2[~data2['price'].isnull()]
data2

data2.isnull().sum()

data2.describe(include='all')

# exclude event_time rows that contain 1970
data2 = data2[~data2['event_time'].str.contains('1970')]
#remove UTC from event_time
data2['event_time'] = data2['event_time'].str.replace('UTC','')
data2.describe(include='all')

# create date column
data2['date'] = data2.event_time.apply(lambda x: x.split(' ')[0])

# convert to datetime object
data2['date'] = pd.to_datetime(data2['date'])
# delete event_time column
del data2['event_time']
data2.head()

# create a new time series
df = data2.groupby('date').agg(sales=('price', 'sum'))
df.head()

# visualize the time series data
fig, ax = plt.subplots()
plt.ticklabel_format(style='plain')
fig = df.plot(figsize=(12,5), ax=ax)
plt.title('Daily Sales')
plt.show()

# Define RFM Dataset - 1
from pandas.tseries.offsets import MonthEnd

df_data['month_key'] = df_data['event_time'].dt.month
df_data[['event_time', 'month_key']]

# creating a new dataframe
df_month_keys = pd.DataFrame({"month_key":df_data['month_key'].unique(), 'key':0})
df_user_ids = pd.DataFrame({"user_id":df_data['user_id'].unique(), 'key':0})

df_rfm = df_month_keys.merge(df_user_ids, on='key', how='outer')
df_rfm = df_rfm.drop(columns=['key'])
df_rfm = df_rfm.sort_values(by=['user_id', 'month_key']).reset_index(drop=True)
df_rfm

# Recency
# User last month purchase
df_user_month_purchases = df_data[['month_key', 'user_id']].drop_duplicates()
df_user_month_purchases['last_purchase'] = df_user_month_purchases['month_key']

df_rfm = df_rfm.merge(df_user_month_purchases, how='left', on=['month_key', 'user_id'])

# filling the last_purchase month
user_ids = df_rfm[['user_id']]
df_rfm = df_rfm.groupby('user_id').ffill()
df_rfm['R_months_since_last_purchase'] = df_rfm['month_key'] - df_rfm['last_purchase']
df_rfm['user_id'] = user_ids
df_rfm.head(20)

# Frequency
# user last month purchase order count
df_user_month_purchases = df_data.groupby(['month_key', 'user_id'])['order_id'].nunique().reset_index()
df_rfm = df_rfm.merge(df_user_month_purchases, how='left', on=['month_key', 'user_id'])

# filling the last_purchase month
user_ids = df_rfm[['user_id']]
df_rfm = df_rfm.groupby('user_id').ffill()
df_rfm['user_id'] = user_ids
df_rfm = df_rfm.rename(columns={"order_id":"F_last_monthly_purchases_count"})
df_rfm.head(20)

# Monetary
# user last monthly purchase value
df_user_month_purchases = df_data.groupby(['month_key','user_id'])['price'].sum().reset_index()
df_rfm=df_rfm.merge(df_user_month_purchases, how='left', on=['month_key','user_id'])

# fill in  last purchase month
user_ids = df_rfm[['user_id']]
df_rfm = df_rfm.groupby('user_id').ffill()
df_rfm['user_id'] = user_ids
df_rfm = df_rfm.rename(columns={"price":"M_last_monthly_purchases_value"})
df_rfm.head(20)

# define RFM dataframe
# drop all missing values
df_rfm = df_rfm.dropna()
df_rfm = df_rfm[['user_id','month_key','R_months_since_last_purchase','F_last_monthly_purchases_count', 'M_last_monthly_purchases_value']]
df_rfm

# visualizing PCA - Prinicpal Component Analysis
from sklearn.decomposition import PCA
X = df_rfm[['R_months_since_last_purchase','F_last_monthly_purchases_count','M_last_monthly_purchases_value']]
pca = PCA(n_components=2)
X_pca = pca.fit_transform(X)
df_pca = pd.DataFrame(X_pca, columns=['pca_1','pca_2'])
df_pca

import matplotlib.pyplot as plt
plt.figure()
plt.scatter(df_pca['pca_1'], df_pca['pca_2'])
plt.show()

# K means clustering
from sklearn.cluster import KMeans
inertias = []
for i in range(1,11):
  kmeans = KMeans(n_clusters=i)
  kmeans.fit(X)
  inertias.append(kmeans.inertia_)
plt.plot(range(1,11), inertias, marker='o')
plt.title('Elbow Method')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()

kmeans = KMeans(n_clusters=4)
kmeans.fit(df_pca)
plt.scatter(df_pca['pca_1'], df_pca['pca_2'], c=kmeans.labels_)
plt.show()

# Analyze results of clustering
df_rfm['kmeans_labels'] = kmeans.labels_
df_rfm.groupby('kmeans_labels')[['R_months_since_last_purchase','F_last_monthly_purchases_count','M_last_monthly_purchases_value']].agg({"mean","count"})



# Machine Learning
# limiting the data in the dataset to perform machine learning operation with limited resources used
df = data3.head(1000)
df

# Inspect the dataset
print(df.info())
print(df.head())

# Preprocess data
df = df.dropna(subset=['user_id', 'product_id', 'price'])  # Drop rows with missing user_id, product_id, or price
df['user_id'] = df['user_id'].astype(int)
df['product_id'] = df['product_id'].astype(int)

# added code
df.head()
df.info()

df.describe(include='all').T

# Data preprocessing
df = df.drop_duplicates()
df.isnull().sum()

# exclude rows where 'event_time' contains '1970'
df = df[~df['event_time'].str.contains('1970')]
df.describe(include='all').T

# remove empty rows in brand and user_id columns
df = df.dropna(subset=['brand', 'user_id']).reset_index(drop=True)
df.isnull().sum()

from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from surprise import SVD, Dataset, Reader, KNNBasic
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import linear_kernel
# Create train-test split for collaborative filtering
train_data, test_data = train_test_split(df, test_size=0.2, random_state=42)

# Collaborative Filtering
# Convert to Surprise format
reader = Reader(rating_scale=(df['price'].min(), df['price'].max()))
data = Dataset.load_from_df(df[['user_id', 'product_id', 'price']], reader)
trainset = data.build_full_trainset()

# Use SVD for collaborative filtering
collab_model = SVD()
collab_model.fit(trainset)

print("Unique product IDs:", df['product_id'].unique())

print("Unique users IDs:", df['user_id'].unique())

# Test on a sample user-product pair
pred = collab_model.predict(uid=1515915625441990000, iid=1515966223509089906)
print(f"Collaborative Filtering Prediction: {pred.est}")

testset = [tuple(x) for x in test_data[['user_id', 'product_id', 'price']].to_numpy()]

# Generate predictions for the testset
predictions = collab_model.test(testset)
true_ratings = [pred.r_ui for pred in predictions]  # True ratings
predicted_ratings = [pred.est for pred in predictions]  # Predicted ratings
rmse = np.sqrt(mean_squared_error(true_ratings, predicted_ratings))
print(f"RMSE: {rmse}")

# Content-Based Filtering
# Combine category_code and brand for content-based features
df['content'] = df['category_code'].fillna('') + " " + df['brand'].fillna('')

# Use TF-IDF to vectorize content
vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(df['content'])

# Function to compute similarity on-the-fly
def content_based_recommendations_on_the_fly(product_id, top_n=5):
    # idx = df[df['product_id'] == product_id].index[0]
    if product_id not in df['product_id'].values:
        return []
    indices = df.index[df['product_id'] == product_id].tolist()
    if not indices:
        return []
    idx = indices[0]
    product_vector = tfidf_matrix[idx]
    sim_scores = linear_kernel(product_vector, tfidf_matrix).flatten()
    sim_scores = [(i, score) for i, score in enumerate(sim_scores)]
    sim_scores = sorted(sim_scores, key=lambda x: x[1], reverse=True)
    sim_scores = sim_scores[1:top_n + 1]
    product_indices = [i[0] for i in sim_scores]
    return df.iloc[product_indices]['product_id'].tolist()

# Test content-based filtering
print("Content-Based Recommendations:", content_based_recommendations_on_the_fly(product_id=1515966223509089906))

from sklearn.neighbors import NearestNeighbors

# Train a nearest neighbor model
nn = NearestNeighbors(metric='cosine', algorithm='brute')
nn.fit(tfidf_matrix)

# Function to get recommendations
def ann_recommendations(product_id, top_n=5):
    idx = df[df['product_id'] == product_id].index[0]
    distances, indices = nn.kneighbors(tfidf_matrix[idx], n_neighbors=top_n + 1)
    return df.iloc[indices.flatten()[1:]]['product_id'].tolist()

# Test ANN recommendations
print("ANN Content-Based Recommendations:", ann_recommendations(product_id=1515966223509089906))

# RMSE Calculation for Content-Based Filtering
def predict_rating(user_id, product_id, tfidf_matrix, df):
    if product_id not in df['product_id'].values:
        return np.nan  # No prediction available
    indices = df.index[df['product_id'] == product_id].tolist()
    if not indices:
        return np.nan
    idx = indices[0]
    product_vector = tfidf_matrix[idx]
    sim_scores = linear_kernel(product_vector, tfidf_matrix).flatten()

    # User's rated products
    user_ratings = df[df['user_id'] == user_id]
    if user_ratings.empty:
        return np.nan  # No prediction available

    # Calculate weighted average rating
    weighted_sum = 0
    sim_sum = 0
    for _, row in user_ratings.iterrows():
        rated_product_idx = df.index[df['product_id'] == row['product_id']].tolist()
        if rated_product_idx:
            similarity = sim_scores[rated_product_idx[0]]
            weighted_sum += similarity * row['price']
            sim_sum += similarity

    return weighted_sum / sim_sum if sim_sum > 0 else np.nan

# Apply predictions for each user-product pair in the dataset
df['predicted_rating'] = df.apply(
    lambda x: predict_rating(x['user_id'], x['product_id'], tfidf_matrix, df), axis=1
)

# Drop rows with NaN predictions
df_filtered = df.dropna(subset=['predicted_rating'])

# Calculate RMSE
true_ratings = df_filtered['price'].to_numpy()
predicted_ratings = df_filtered['predicted_rating'].to_numpy()
rmse = np.sqrt(mean_squared_error(true_ratings, predicted_ratings))

print(f"RMSE for Content-Based Filtering: {rmse}")

# Hybrid Recommendation
# Weighted average of collaborative and content-based recommendations
def hybrid_recommendations(user_id, product_id, top_n=5):
    # Collaborative prediction
    collab_pred = collab_model.predict(uid=user_id, iid=product_id).est

    # Content-based recommendations
    content_recs = content_based_recommendations_on_the_fly(product_id, top_n=top_n)

    # Combine recommendations
    recommendations = [(prod_id, collab_pred) for prod_id in content_recs]
    return sorted(recommendations, key=lambda x: x[1], reverse=True)

# Test hybrid recommendation
print("Hybrid Recommendations:", hybrid_recommendations(user_id=1515915625441990000, product_id=1515966223509089906))

# Performance Metrics
def evaluate_models(test_data, collab_model, top_n=5):
    metrics = {"accuracy": {}, "diversity": {}, "engagement": {}}

    #Accuracy
    # Collaborative Filtering Evaluation
    collab_preds = []
    for _, row in test_data.iterrows():
        pred = collab_model.predict(uid=row['user_id'], iid=row['product_id']).est
        collab_preds.append(pred)
    metrics["accuracy"]["collaborative"] = np.sqrt(mean_squared_error(test_data['price'], collab_preds))

    # Content-Based Evaluation
    content_preds = []
    for _, row in test_data.iterrows():
        recs = content_based_recommendations_on_the_fly(row['product_id'], top_n=top_n)
        content_preds.append(row['product_id'] in recs)
    metrics["accuracy"]["content-based"] = np.mean(content_preds)

    # Hybrid Evaluation
    hybrid_preds = []
    for _, row in test_data.iterrows():
        recs = hybrid_recommendations(row['user_id'], row['product_id'], top_n=top_n)
        hybrid_preds.append(any(r[0] == row['product_id'] for r in recs))
    metrics["accuracy"]["hybrid"] = np.mean(hybrid_preds)

    # # Diversity (unique recommendations for a sample of users)
    sample_users = test_data['user_id'].sample(n=100, random_state=42).unique()
    diversity_scores = {
        "collaborative": len(set(collab_model.predict(uid=u, iid=i).iid for u in sample_users for i in range(top_n))),
        "content-based": len(set(i for user in sample_users for i in content_based_recommendations_on_the_fly(user, top_n=top_n))),
        "hybrid": 0,
        # "hybrid": len(set(i for user in sample_users for i in hybrid_recommendations(user, i, top_n=top_n))),
    }
    metrics["diversity"] = diversity_scores

    # Engagement (average number of recommendations per user)
    engagement_scores = {
        "collaborative": np.mean([len([collab_model.predict(uid=u, iid=i).iid for i in range(top_n)]) for u in sample_users]),
        "content-based": np.mean([len(content_based_recommendations_on_the_fly(i, top_n=top_n)) for i in sample_users]),
        "hybrid": np.mean([len(hybrid_recommendations(u, i, top_n=top_n)) for u in sample_users for i in range(top_n)]),
    }
    metrics["engagement"] = engagement_scores

    return metrics

# Evaluate and print metrics
metrics = evaluate_models(test_data, collab_model)
print("Performance Metrics:", metrics)