Upload folder using huggingface_hub

app.py

@@ -5,6 +5,11 @@ import altair as alt
 import matplotlib.pyplot as plt
 import seaborn as sns
 from scipy.stats import zscore
+from sklearn.preprocessing import LabelEncoder, StandardScaler
+from sklearn.decomposition import PCA
+from sklearn.cluster import KMeans, AgglomerativeClustering
+from scipy.spatial.distance import cdist
+from scipy.cluster.hierarchy import dendrogram, linkage
 
 # Function to load and clean data
 @st.cache_data
@@ -32,7 +37,7 @@ st.title('BDS24_Weekly_Assignment_Week 2 | Tryfonas Karmiris')
 
 # Sidebar for navigation
 st.sidebar.title("Navigation")
-page = st.sidebar.radio("Select a page:", ["Introduction", "Data Overview", "Top Values by Selected Variable", "Repayment Interval by Selected Variable", "Country Comparison Deepdive", "Sector Comparison Deepdive"])
+page = st.sidebar.radio("Select a page:", ["Introduction", "Data Overview", "Top Values by Selected Variable", "Repayment Interval by Selected Variable", "Country Comparison Deepdive", "Sector Comparison Deepdive", "KMeans Clustering & Recommendations","Hierarchical Clustering & Dendrogram"])
 
 # Introduction Page
 if page == "Introduction":
@@ -94,7 +99,7 @@ elif page == "Top Values by Selected Variable":
         top_values = df_kiva_loans_cleaned.groupby('sector')['funded_amount'].agg(['sum', 'count']).nlargest(num_columns, 'sum').reset_index()
         x_column = 'sector'
         count_column = 'count'
-        description = f"This chart illustrates the top {num_columns} sectors by total funded amount. The blue bars represent the total funded amount, while the red line indicates the count of loans. Most loans are funded to the Aggriculture Sector with Food and Retail completing the first three. Looks like that if the sector of the business is close to Primary production or its Basic Necessities(food) "
+        description = f"This chart illustrates the top {num_columns} sectors by total funded amount. The blue bars represent the total funded amount, while the red line indicates the count of loans. Most loans are funded to the Agriculture Sector with Food and Retail completing the first three. Looks like that if the sector of the business is close to Primary production or its Basic Necessities(food) "
 
     # Display description
     st.write(description)
@@ -151,10 +156,7 @@ elif page == "Top Values by Selected Variable":
     plt.xticks(rotation=90)
     st.pyplot(fig)
 
-    # Display description for boxplot
-    
-
-# Page 4: Other Plots
+# Remaining pages (Repayment Interval by Selected Variable, Country Comparison Deepdive, Sector Comparison Deepdive)
 elif page == "Repayment Interval by Selected Variable":
     st.subheader('Repayment Interval by Selected Variable')
 
@@ -178,7 +180,7 @@ elif page == "Repayment Interval by Selected Variable":
     elif plot_var == 'country':
         top_values_plot = df_kiva_loans_cleaned.groupby('country')['funded_amount'].agg('count').nlargest(num_top_values).index
         filtered_df_plot = df_kiva_loans_cleaned[df_kiva_loans_cleaned['country'].isin(top_values_plot)]
-        description = f"This countplot illustrates the distribution of repayment intervals for the top {num_top_values} countries based on the number of loans. In terms of countries the Phillipines had a great number of Irregular loans."
+        description = f"This countplot illustrates the distribution of repayment intervals for the top {num_top_values} countries based on the number of loans. In terms of countries the Philippines had a great number of Irregular loans."
 
     # Display description
     st.write(description)
@@ -205,15 +207,12 @@ elif page == "Repayment Interval by Selected Variable":
     plt.legend(title=plot_var.replace("_", " ").title(), bbox_to_anchor=(1.05, 1), loc='upper left')
     st.pyplot(fig)
 
-
-
-
-# Page 5: Country Comparison
+# Page 5: Country Comparison Deepdive
 elif page == "Country Comparison Deepdive":
     st.subheader("Country Comparison Deepdive")
 
     # Multi-select for countries
-    selected_countries = st.multiselect("Select Countries to Compare(Please select one or more)", options=df_kiva_loans_cleaned['country'].unique())
+    selected_countries = st.multiselect("Select Countries to Compare (Please select one or more)", options=df_kiva_loans_cleaned['country'].unique())
 
     # Option to choose between count or sum of funded amounts
     aggregation_option = st.radio("Select Aggregation Type:", ("Count of Loans", "Summary of Funded Amount"))
@@ -266,7 +265,7 @@ elif page == "Country Comparison Deepdive":
     else:
         st.write("Please select one or more countries to compare from the dropdown above.")
 
-# Page 6: Sector Comparison
+# Page 6: Sector Comparison Deepdive
 elif page == "Sector Comparison Deepdive":
     st.subheader("Sector Comparison Deepdive")
 
@@ -274,7 +273,7 @@ elif page == "Sector Comparison Deepdive":
     selected_sectors = st.multiselect("Select Sectors to Compare (Please select one or more)", options=df_kiva_loans_cleaned['sector'].unique())
 
     # Option to choose between count or sum of funded amounts
-    aggregation_option = st.radio("Select Aggregation Type:", ("Count of Loans", "Summmary of Funded Amount"))
+    aggregation_option = st.radio("Select Aggregation Type:", ("Count of Loans", "Summary of Funded Amount"))
 
     if selected_sectors:
         # Filter the data based on selected sectors
@@ -326,3 +325,199 @@ elif page == "Sector Comparison Deepdive":
     else:
         st.write("Please select one or more countries to compare from the dropdown above.")
 
+# Page 7: KMeans Clustering & Recommendations
+elif page == "KMeans Clustering & Recommendations":
+    st.subheader("KMeans Clustering & Recommendations")
+
+    # User input to choose the number of sample rows
+    sample_size = st.slider("Select the number of sample rows for clustering:", min_value=1000, max_value=100000, value=20000, step=1000)
+
+    # Sample the selected number of rows from the DataFrame
+    df_sample = df_kiva_loans_cleaned.sample(n=sample_size, random_state=42).copy()
+
+    # Keeping only the relevant columns and storing original indices
+    df_original = df_sample[['country','funded_amount', 'sector','repayment_interval']].copy()
+    df_original['original_index'] = df_sample.index  # Keep track of original indices
+
+    # Label Encoding for categorical variables and adding encoded columns with "_id" suffix
+    label_encoders = {}
+    for column in df_original.select_dtypes(include=['object']).columns:
+        le = LabelEncoder()
+        df_original[column + '_id'] = le.fit_transform(df_original[column])
+        label_encoders[column] = le
+
+    # Standardizing the data using the encoded columns
+    encoded_columns = [col + '_id' for col in df_original.select_dtypes(include=['object']).columns]
+    scaler = StandardScaler()
+    df_scaled = scaler.fit_transform(df_original[encoded_columns + ['funded_amount']])
+
+    # Applying PCA
+    pca = PCA(n_components=2)  # Reduce to 2 dimensions for visualization
+    df_pca = pca.fit_transform(df_scaled)
+
+    # Elbow Method to find the optimal number of clusters
+    inertia = []
+    for n in range(1, 11):
+        kmeans = KMeans(n_clusters=n, random_state=42)
+        kmeans.fit(df_pca)
+        inertia.append(kmeans.inertia_)
+
+    # Plotting the Elbow Method
+    plt.figure(figsize=(8, 6))
+    plt.plot(range(1, 11), inertia, marker='o', linestyle='--')
+    plt.title('Elbow Method for Optimal Number of Clusters')
+    plt.xlabel('Number of Clusters')
+    plt.ylabel('Inertia')
+    st.pyplot(plt.gcf())
+
+    # User input to choose the optimal number of clusters
+    optimal_clusters = st.slider("Select the number of optimal clusters:", min_value=1, max_value=10, value=4, step=1)
+   
+    # Apply KMeans with optimal clusters
+    kmeans = KMeans(n_clusters=optimal_clusters, random_state=42)
+    df_original['cluster'] = kmeans.fit_predict(df_pca)
+
+    # Visualize the clustering results at different iterations
+    max_iters = [1, 2, 5, 6, 8, 10]  # Different iterations you want to visualize
+
+    # Increase the figure size for better visibility
+    plt.figure(figsize=(15, 55))  # Adjusted the figsize to make plots larger
+    for i, max_iter in enumerate(max_iters, start=1):
+        kmeans = KMeans(n_clusters=optimal_clusters, random_state=42, max_iter=max_iter)
+        df_original['cluster'] = kmeans.fit_predict(df_pca)
+        
+        # Plotting the clusters
+        plt.subplot(6, 1, i)  # Changed the layout to 3 rows x 2 columns for larger plots
+        sns.scatterplot(x=df_pca[:, 0], y=df_pca[:, 1], hue=df_original['cluster'], palette='viridis', s=100)
+        
+        # Plotting the centroids
+        centroids = kmeans.cluster_centers_
+        plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=300, marker='X', label='Centroids')  # Increased centroid size
+        
+        plt.title(f'K-means Clustering - Iteration {max_iter}', fontsize=16)
+        plt.xlabel('Principal Component 1', fontsize=14)
+        plt.ylabel('Principal Component 2', fontsize=14)
+        plt.xticks(fontsize=12)
+        plt.yticks(fontsize=12)
+        if i == 1:
+            plt.legend()
+
+    plt.tight_layout()
+    st.pyplot(plt.gcf())
+
+    # Dynamic input for the new data point
+    st.subheader("Input New Data Point for Recommendations")
+
+    # Allow the user to select the country, sector, and repayment interval
+    country = st.selectbox("Select Country", options=df_kiva_loans_cleaned['country'].unique())
+    sector = st.selectbox("Select Sector", options=df_kiva_loans_cleaned['sector'].unique())
+    repayment_interval = st.selectbox("Select Repayment Interval", options=df_kiva_loans_cleaned['repayment_interval'].unique())
+
+    # Allow the user to select the funded amount using a slider
+    funded_amount = st.slider("Select Funded Amount", min_value=int(df_kiva_loans_cleaned['funded_amount'].min()), max_value=int(df_kiva_loans_cleaned['funded_amount'].max()), value=1500)
+
+    new_data = {
+        'country': country,
+        'funded_amount': funded_amount,
+        'sector': sector,
+        'repayment_interval': repayment_interval
+    }
+
+    # Convert new data to DataFrame
+    new_data_df = pd.DataFrame([new_data])
+
+    # Encode the new data point and add encoded columns with "_id" suffix
+    for column in new_data_df.select_dtypes(include=['object']).columns:
+        new_data_df[column + '_id'] = label_encoders[column].transform(new_data_df[column])
+
+    # Standardize the new data using the encoded columns
+    new_data_scaled = scaler.transform(new_data_df[[col + '_id' for col in new_data_df.select_dtypes(include=['object']).columns] + ['funded_amount']])
+
+    # Apply PCA to the new data
+    new_data_pca = pca.transform(new_data_scaled)
+
+    # Predict the cluster for the new data point
+    new_cluster = kmeans.predict(new_data_pca)[0]
+    
+    st.subheader("Top 5 Similar Items to the Input")
+    st.write(f"The new data point belongs to cluster: {new_cluster}")
+    # Get all data points in the same cluster
+    cluster_data = df_original[df_original['cluster'] == new_cluster]
+
+    # Apply the same PCA transformation to the scaled data of the entire cluster
+    cluster_data_pca = pca.transform(scaler.transform(cluster_data[encoded_columns + ['funded_amount']]))
+
+    # Calculate the Euclidean distance between the new data point and all points in the same cluster
+    distances = cdist(new_data_pca, cluster_data_pca, 'euclidean')[0]
+
+    # Add distances to the cluster data DataFrame
+    cluster_data = cluster_data.copy()
+    cluster_data['distance'] = distances
+
+    # Sort by distance and select the top 5 closest items
+    top_5_recommendations = cluster_data.sort_values('distance').head(5)
+
+    # Retrieve the original rows from the original DataFrame before encoding
+    recommended_indices = top_5_recommendations['original_index']
+    recommendations = df_kiva_loans_cleaned.loc[recommended_indices]
+
+    # Display the original rows as the top 5 recommendations
+
+    st.write(recommendations)
+
+
+# Page 8: Hierarchical Clustering & Dendrogram
+elif page == "Hierarchical Clustering & Dendrogram":
+    st.subheader("Hierarchical Clustering & Dendrogram")
+
+    # User input to choose the number of sample rows
+    sample_size = st.slider("Select the number of sample rows for clustering:", min_value=1000, max_value=5000, value=5000, step=50)
+
+    # User input to choose the number of clusters
+    n_clusters = st.slider("Select the number of clusters:", min_value=2, max_value=10, value=4, step=1)
+
+    # Sample the selected number of rows from the DataFrame
+    df_sample = df_kiva_loans_cleaned.sample(n=sample_size, random_state=42).copy()
+
+    # Keeping only the relevant columns and storing original indices
+    df_original = df_sample[['country','funded_amount', 'sector','repayment_interval']].copy()
+    df_original['original_index'] = df_sample.index  # Keep track of original indices
+
+    # Label Encoding for categorical variables and adding encoded columns with "_id" suffix
+    label_encoders = {}
+    for column in df_original.select_dtypes(include=['object']).columns:
+        le = LabelEncoder()
+        df_original[column + '_id'] = le.fit_transform(df_original[column])
+        label_encoders[column] = le
+
+    # Standardizing the data using the encoded columns
+    encoded_columns = [col + '_id' for col in df_original.select_dtypes(include=['object']).columns]
+    scaler = StandardScaler()
+    df_scaled = scaler.fit_transform(df_original[encoded_columns + ['funded_amount']])
+
+    # Applying PCA
+    pca = PCA(n_components=2)  # Reduce to 2 dimensions for visualization
+    df_pca = pca.fit_transform(df_scaled)
+
+    # Perform Agglomerative Clustering with dynamic n_clusters
+    agg_clustering = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward')
+    df_original['cluster'] = agg_clustering.fit_predict(df_pca)
+
+    # Plot the resulting clusters
+    plt.figure(figsize=(10, 7))
+    sns.scatterplot(x=df_pca[:, 0], y=df_pca[:, 1], hue=df_original['cluster'], palette='viridis', s=50)
+    plt.title(f'Agglomerative Clustering (Hierarchical) Results - {n_clusters} Clusters')
+    plt.xlabel('Principal Component 1')
+    plt.ylabel('Principal Component 2')
+    st.pyplot(plt.gcf())
+
+    # Dendrogram Visualization
+    linked = linkage(df_pca, method='ward')
+
+    plt.figure(figsize=(10, 7))
+    dendrogram(linked,
+               orientation='top',
+               distance_sort='descending',
+               show_leaf_counts=True)
+    plt.title('Hierarchical Clustering Dendrogram')
+    st.pyplot(plt.gcf())

requirements.txt

@@ -3,4 +3,5 @@ seaborn
 pandas
 matplotlib
 altair
-scipy
+scipy
+scikit-learn