# Import necessary libraries import streamlit as st import pandas as pd import altair as alt import matplotlib.pyplot as plt import seaborn as sns from scipy.stats import zscore from sklearn.preprocessing import LabelEncoder, StandardScaler from sklearn.decomposition import PCA from sklearn.cluster import KMeans, AgglomerativeClustering from scipy.spatial.distance import cdist from scipy.cluster.hierarchy import dendrogram, linkage # Function to load and clean data @st.cache_data def load_and_clean_data(file_path): # Load data df_kiva_loans = pd.read_csv(file_path) # Clean data df_kiva_loans = df_kiva_loans.drop(['use', 'disbursed_time', 'funded_time', 'posted_time', 'tags'], axis=1) df_kiva_loans.dropna(subset=['partner_id', 'borrower_genders'], inplace=True) # Calculate Z-scores z_scores = zscore(df_kiva_loans['funded_amount']) df_kiva_loans['outlier_funded_amount'] = (z_scores > 3) | (z_scores < -3) df_kiva_loans_cleaned = df_kiva_loans[~df_kiva_loans['outlier_funded_amount']] return df_kiva_loans_cleaned # Load the cleaned data file_path = 'kiva_loans.csv' df_kiva_loans_cleaned = load_and_clean_data(file_path) # Streamlit App Title st.title('BDS24_Weekly_Assignment_Week 2 | Tryfonas Karmiris') # Sidebar for navigation st.sidebar.title("Navigation") page = st.sidebar.radio("Select a page:", ["Introduction", "Data Overview", "Top Values by Selected Variable", "Repayment Interval by Selected Variable", "Country Comparison Deepdive", "Sector Comparison Deepdive", "KMeans Clustering & Recommendations","Hierarchical Clustering & Dendrogram"]) # Introduction Page if page == "Introduction": st.subheader("Introduction") st.write(""" This application provides insights into Kiva loans data. You can explore the distribution of funded amounts, analyze top values by selected variables, and visualize relationships between funded amounts and various factors such as Countries and Sectors that the loans were funded. """) # Data Overview Page elif page == "Data Overview": st.subheader("Data Overview") st.write("Here is a preview of the cleaned Kiva loans data:") # Display the cleaned data table st.table(df_kiva_loans_cleaned.head()) # Distribution of Funded Amounts st.subheader('Distribution of Funded Amounts') chart = alt.Chart(df_kiva_loans_cleaned).mark_bar().encode( alt.X('funded_amount', bin=alt.Bin(maxbins=50)), # Use funded_amount for distribution y='count()', ).properties( title='Distribution of Funded Amounts' ) st.altair_chart(chart, use_container_width=True) st.write("This chart shows the distribution of funded amounts for Kiva loans. The x-axis represents the funded amount, while the y-axis shows the count of loans that fall within each bin. As you can see most of the loans are low valued with most of them being in the range of 100 and 500") # Page 3: Top Values by Selected Variable elif page == "Top Values by Selected Variable": st.subheader('Top Values by Selected Variable') # Dropdown for plot type plot_type = st.selectbox("Select Variable to Display", ['country', 'repayment_interval', 'sector']) # Slider to select the number of top values to display num_columns = st.slider( "Select Number of Columns to Display on the Chart", min_value=5, max_value=50, value=10, # default value step=1 ) # Select the top values based on the selected variable and number of columns if plot_type == 'country': top_values = df_kiva_loans_cleaned.groupby('country')['funded_amount'].agg(['sum', 'count']).nlargest(num_columns, 'sum').reset_index() x_column = 'country' count_column = 'count' description = f"This chart displays the top {num_columns} countries by total funded amount. The blue bars represent the total funded amount, while the red line indicates the count of loans. In general Phillipines is the country with the most loans followed by Kenya and El Salvador." elif plot_type == 'repayment_interval': top_values = df_kiva_loans_cleaned.groupby('repayment_interval')['funded_amount'].agg(['sum', 'count']).nlargest(num_columns, 'sum').reset_index() x_column = 'repayment_interval' count_column = 'count' description = f"This chart shows the top {num_columns} repayment intervals by total funded amount. The blue bars represent the total funded amount, while the red line indicates the count of loans. Most of the loans are funded with a monthly repayment interval, where the bullet repayment is an unsusal choice" else: # sector top_values = df_kiva_loans_cleaned.groupby('sector')['funded_amount'].agg(['sum', 'count']).nlargest(num_columns, 'sum').reset_index() x_column = 'sector' count_column = 'count' description = f"This chart illustrates the top {num_columns} sectors by total funded amount. The blue bars represent the total funded amount, while the red line indicates the count of loans. Most loans are funded to the Agriculture Sector with Food and Retail completing the first three. Looks like that if the sector of the business is close to Primary production or its Basic Necessities(food) " # Display description st.write(description) # Create a dual-axis bar plot using Matplotlib fig, ax1 = plt.subplots(figsize=(12, 9)) plt.xticks(rotation=90) # Bar plot for funded_amount color = 'tab:blue' ax1.set_xlabel(x_column.replace("_", " ").title()) ax1.set_ylabel('Funded Amount', color=color) ax1.bar(top_values[x_column], top_values['sum'], color=color, alpha=0.6, label='Funded Amount') ax1.tick_params(axis='y', labelcolor=color) # Create a second y-axis for count ax2 = ax1.twinx() color = 'tab:red' ax2.set_ylabel('Count', color=color) ax2.plot(top_values[x_column], top_values[count_column], color=color, marker='o', linestyle='-', linewidth=2, label='Count') ax2.tick_params(axis='y', labelcolor=color) # Add titles and labels plt.title(f'Top {num_columns} by {plot_type.replace("_", " ").title()}') fig.tight_layout() st.pyplot(fig) # Boxplot after the dual-axis plot st.subheader('Funded Amount vs. Selected Variable') # Filter the data based on the selected variable and number of top values if plot_type == 'sector': top_values_boxplot = df_kiva_loans_cleaned.groupby('sector')['funded_amount'].agg('sum').nlargest(num_columns).index filtered_df_boxplot = df_kiva_loans_cleaned[df_kiva_loans_cleaned['sector'].isin(top_values_boxplot)] elif plot_type == 'country': top_values_boxplot = df_kiva_loans_cleaned.groupby('country')['funded_amount'].agg('sum').nlargest(num_columns).index filtered_df_boxplot = df_kiva_loans_cleaned[df_kiva_loans_cleaned['country'].isin(top_values_boxplot)] else: # repayment_interval filtered_df_boxplot = df_kiva_loans_cleaned # Create a boxplot fig, ax = plt.subplots(figsize=(12, 6)) if plot_type != 'repayment_interval': top_values_sorted = df_kiva_loans_cleaned.groupby(plot_type)['funded_amount'].agg('sum').nlargest(num_columns).index sns.boxplot(x=plot_type, y='funded_amount', data=filtered_df_boxplot, order=top_values_sorted, ax=ax) st.write(f"This boxplot shows the distribution of funded amounts for the top {num_columns} {plot_type.replace('_', ' ')}. It provides insights into the spread and outliers of funded amounts.") else: sns.boxplot(x=plot_type, y='funded_amount', data=filtered_df_boxplot, ax=ax) st.write(f"This boxplot shows the distribution of funded amounts for the top {num_columns} {plot_type.replace('_', ' ')}. It provides insights into the spread and outliers of funded amounts.") plt.title('Funded Amount by Selected Variable') plt.xlabel(plot_type) plt.ylabel('Funded Amount') plt.xticks(rotation=90) st.pyplot(fig) # Remaining pages (Repayment Interval by Selected Variable, Country Comparison Deepdive, Sector Comparison Deepdive) elif page == "Repayment Interval by Selected Variable": st.subheader('Repayment Interval by Selected Variable') # Dropdown for selecting variable for Seaborn countplot plot_var = st.selectbox("Select Variable for Countplot", ['sector', 'country']) # Slider to select the number of top values to display for Seaborn countplot num_top_values = st.slider( "Select Number of Top Values to Display", min_value=5, max_value=50, value=10, # default value step=1 ) # Filter the data based on the selected variable and number of top values if plot_var == 'sector': top_values_plot = df_kiva_loans_cleaned.groupby('sector')['funded_amount'].agg('count').nlargest(num_top_values).index filtered_df_plot = df_kiva_loans_cleaned[df_kiva_loans_cleaned['sector'].isin(top_values_plot)] description = f"This countplot shows the distribution of repayment intervals for the top {num_top_values} sectors based on the number of loans. In terms of sectors Agriculture got the most monthly repayment loans followed by food. Also a lot of irregulars were in the Food, Retail and Agriculture sectors, which again confirms that loans for first necessities are given more easily. " elif plot_var == 'country': top_values_plot = df_kiva_loans_cleaned.groupby('country')['funded_amount'].agg('count').nlargest(num_top_values).index filtered_df_plot = df_kiva_loans_cleaned[df_kiva_loans_cleaned['country'].isin(top_values_plot)] description = f"This countplot illustrates the distribution of repayment intervals for the top {num_top_values} countries based on the number of loans. In terms of countries the Philippines had a great number of Irregular loans." # Display description st.write(description) # Create a count plot fig, ax = plt.subplots(figsize=(10, 6)) # Count the occurrences of repayment intervals for the filtered data count_data = filtered_df_plot.groupby('repayment_interval')[plot_var].value_counts().unstack(fill_value=0) # Calculate total counts for sorting total_counts = count_data.sum(axis=1) # Sort the repayment intervals based on the total count of loans in descending order sorted_index = total_counts.sort_values(ascending=False).index count_data = count_data.loc[sorted_index] # Create a grouped bar plot count_data.plot(kind='bar', ax=ax, position=0, width=0.8) plt.title(f'Repayment Interval by {plot_var.replace("_", " ").title()}') plt.xlabel('Repayment Interval') plt.ylabel('Count of Loans') plt.xticks(rotation=45) plt.legend(title=plot_var.replace("_", " ").title(), bbox_to_anchor=(1.05, 1), loc='upper left') st.pyplot(fig) # Page 5: Country Comparison Deepdive elif page == "Country Comparison Deepdive": st.subheader("Country Comparison Deepdive") # Multi-select for countries selected_countries = st.multiselect("Select Countries to Compare (Please select one or more)", options=df_kiva_loans_cleaned['country'].unique()) # Option to choose between count or sum of funded amounts aggregation_option = st.radio("Select Aggregation Type:", ("Count of Loans", "Summary of Funded Amount")) if selected_countries: # Filter the data based on selected countries filtered_data = df_kiva_loans_cleaned[df_kiva_loans_cleaned['country'].isin(selected_countries)] # Create a combined bar plot for sector summary st.subheader("Total Funded Amounts by Sector for Selected Countries") if aggregation_option == "Sum": sector_summary = filtered_data.groupby(['country', 'sector']).agg( total_funded_amount=('funded_amount', 'sum') ).reset_index() st.write("This graph shows the total funded amount in each Sector for the selected Countries by the user.") else: # Count sector_summary = filtered_data.groupby(['country', 'sector']).agg( total_funded_amount=('funded_amount', 'count') ).reset_index() st.write("This graph shows the number of loans in each Sector for the selected Countries by the user.") fig, ax = plt.subplots(figsize=(12, 6)) sns.barplot(x='sector', y='total_funded_amount', hue='country', data=sector_summary, ax=ax) plt.title(f'Total Funded Amount by Sector for Selected Countries ({aggregation_option})') plt.xlabel('Sector') plt.ylabel('Total Funded Amount' if aggregation_option == "Sum" else 'Count of Loans') plt.xticks(rotation=45) st.pyplot(fig) # Create a combined bar plot for repayment summary st.subheader("Total Funded Amounts by Repayment Interval for Selected Countries") if aggregation_option == "Summary of Funded Amount": repayment_summary = filtered_data.groupby(['country', 'repayment_interval']).agg( total_funded_amount=('funded_amount', 'sum') ).reset_index() st.write("This graph shows the total funded amount in each Repayment interval for the selected Countries by the user.") else: # Count repayment_summary = filtered_data.groupby(['country', 'repayment_interval']).agg( total_funded_amount=('funded_amount', 'count') ).reset_index() st.write("This graph shows the number of loans in each Repayment interval for the selected Countries by the user.") fig, ax = plt.subplots(figsize=(12, 6)) sns.barplot(x='repayment_interval', y='total_funded_amount', hue='country', data=repayment_summary, ax=ax) plt.title(f'Total Funded Amount by Repayment Interval for Selected Countries ({aggregation_option})') plt.xlabel('Repayment Interval') plt.ylabel('Total Funded Amount' if aggregation_option == "Sum" else 'Count of Loans') plt.xticks(rotation=45) st.pyplot(fig) else: st.write("Please select one or more countries to compare from the dropdown above.") # Page 6: Sector Comparison Deepdive elif page == "Sector Comparison Deepdive": st.subheader("Sector Comparison Deepdive") # Multi-select for sectors selected_sectors = st.multiselect("Select Sectors to Compare (Please select one or more)", options=df_kiva_loans_cleaned['sector'].unique()) # Option to choose between count or sum of funded amounts aggregation_option = st.radio("Select Aggregation Type:", ("Count of Loans", "Summary of Funded Amount")) if selected_sectors: # Filter the data based on selected sectors filtered_data = df_kiva_loans_cleaned[df_kiva_loans_cleaned['sector'].isin(selected_sectors)] # Create a combined bar plot for sector summary by country st.subheader("Total Funded Amounts by Country for Selected Sectors") if aggregation_option == "Summary of Funded Amount": country_summary = filtered_data.groupby(['country', 'sector']).agg( total_funded_amount=('funded_amount', 'sum') ).reset_index() st.write("This graph shows the total funded amount in each Country, for the selected Sectors by the user.") else: # Count country_summary = filtered_data.groupby(['country', 'sector']).agg( total_funded_amount=('funded_amount', 'count') ).reset_index() st.write("This graph shows the number of loans in each Country, for the selected Sectors by the user.") fig, ax = plt.subplots(figsize=(12, 6)) sns.barplot(x='country', y='total_funded_amount', hue='sector', data=country_summary, ax=ax) plt.title(f'Total Funded Amount by Country for Selected Sectors ({aggregation_option})') plt.xlabel('Country') plt.ylabel('Total Funded Amount' if aggregation_option == "Sum" else 'Count of Loans') plt.legend(title='Sector', bbox_to_anchor=(1.05, 1), loc='upper left') plt.xticks(rotation=90) st.pyplot(fig) # Create a combined bar plot for repayment summary st.subheader("Total Funded Amounts by Repayment Interval for Selected Sectors") if aggregation_option == "Sum": repayment_summary = filtered_data.groupby(['repayment_interval', 'sector']).agg( total_funded_amount=('funded_amount', 'sum') ).reset_index() st.write("This graph shows the funded amount in each Repayment interval for the selected Sectors by the user.") else: # Count repayment_summary = filtered_data.groupby(['repayment_interval', 'sector']).agg( total_funded_amount=('funded_amount', 'count') ).reset_index() st.write("This graph shows the number of loans in each Repayment interval for the selected Sectors by the user.") fig, ax = plt.subplots(figsize=(12, 6)) sns.barplot(x='repayment_interval', y='total_funded_amount', hue='sector', data=repayment_summary, ax=ax) plt.title(f'Total Funded Amount by Repayment Interval for Selected Sectors ({aggregation_option})') plt.xlabel('Repayment Interval') plt.ylabel('Total Funded Amount' if aggregation_option == "Sum" else 'Count of Loans') plt.legend(title='Sector', bbox_to_anchor=(1.05, 1), loc='upper left') plt.xticks(rotation=90) st.pyplot(fig) else: st.write("Please select one or more countries to compare from the dropdown above.") # Page 7: KMeans Clustering & Recommendations elif page == "KMeans Clustering & Recommendations": st.subheader("KMeans Clustering & Recommendations") # User input to choose the number of sample rows sample_size = st.slider("Select the number of sample rows for clustering:", min_value=1000, max_value=100000, value=20000, step=1000) # Sample the selected number of rows from the DataFrame df_sample = df_kiva_loans_cleaned.sample(n=sample_size, random_state=42).copy() # Keeping only the relevant columns and storing original indices df_original = df_sample[['country','funded_amount', 'sector','repayment_interval']].copy() df_original['original_index'] = df_sample.index # Keep track of original indices # Label Encoding for categorical variables and adding encoded columns with "_id" suffix label_encoders = {} for column in df_original.select_dtypes(include=['object']).columns: le = LabelEncoder() df_original[column + '_id'] = le.fit_transform(df_original[column]) label_encoders[column] = le # Standardizing the data using the encoded columns encoded_columns = [col + '_id' for col in df_original.select_dtypes(include=['object']).columns] scaler = StandardScaler() df_scaled = scaler.fit_transform(df_original[encoded_columns + ['funded_amount']]) # Applying PCA pca = PCA(n_components=2) # Reduce to 2 dimensions for visualization df_pca = pca.fit_transform(df_scaled) # Elbow Method to find the optimal number of clusters inertia = [] for n in range(1, 11): kmeans = KMeans(n_clusters=n, random_state=42) kmeans.fit(df_pca) inertia.append(kmeans.inertia_) # Plotting the Elbow Method plt.figure(figsize=(8, 6)) plt.plot(range(1, 11), inertia, marker='o', linestyle='--') plt.title('Elbow Method for Optimal Number of Clusters') plt.xlabel('Number of Clusters') plt.ylabel('Inertia') st.pyplot(plt.gcf()) # User input to choose the optimal number of clusters optimal_clusters = st.slider("Select the number of optimal clusters:", min_value=1, max_value=10, value=4, step=1) # Apply KMeans with optimal clusters kmeans = KMeans(n_clusters=optimal_clusters, random_state=42) df_original['cluster'] = kmeans.fit_predict(df_pca) # Visualize the clustering results at different iterations max_iters = [1, 2, 5, 6, 8, 10] # Different iterations you want to visualize # Increase the figure size for better visibility plt.figure(figsize=(15, 55)) # Adjusted the figsize to make plots larger for i, max_iter in enumerate(max_iters, start=1): kmeans = KMeans(n_clusters=optimal_clusters, random_state=42, max_iter=max_iter) df_original['cluster'] = kmeans.fit_predict(df_pca) # Plotting the clusters plt.subplot(6, 1, i) # Changed the layout to 3 rows x 2 columns for larger plots sns.scatterplot(x=df_pca[:, 0], y=df_pca[:, 1], hue=df_original['cluster'], palette='viridis', s=100) # Plotting the centroids centroids = kmeans.cluster_centers_ plt.scatter(centroids[:, 0], centroids[:, 1], c='red', s=300, marker='X', label='Centroids') # Increased centroid size plt.title(f'K-means Clustering - Iteration {max_iter}', fontsize=16) plt.xlabel('Principal Component 1', fontsize=14) plt.ylabel('Principal Component 2', fontsize=14) plt.xticks(fontsize=12) plt.yticks(fontsize=12) if i == 1: plt.legend() plt.tight_layout() st.pyplot(plt.gcf()) # New Input: Select a cluster and display top 10 data points st.subheader("Explore a Cluster") selected_cluster = st.selectbox("Select a Cluster", options=sorted(df_original['cluster'].unique())) # Filter data based on selected cluster cluster_data = df_original[df_original['cluster'] == selected_cluster] st.write(f"Top 10 items in Cluster {selected_cluster}:") st.write(cluster_data.head(10)) # Dynamic input for the new data point st.subheader("Input New Data Point for Recommendations") # Allow the user to select the country, sector, and repayment interval country = st.selectbox("Select Country", options=df_kiva_loans_cleaned['country'].unique()) sector = st.selectbox("Select Sector", options=df_kiva_loans_cleaned['sector'].unique()) repayment_interval = st.selectbox("Select Repayment Interval", options=df_kiva_loans_cleaned['repayment_interval'].unique()) # Allow the user to select the funded amount using a slider funded_amount = st.slider("Select Funded Amount", min_value=int(df_kiva_loans_cleaned['funded_amount'].min()), max_value=int(df_kiva_loans_cleaned['funded_amount'].max()), value=1500) new_data = { 'country': country, 'funded_amount': funded_amount, 'sector': sector, 'repayment_interval': repayment_interval } # Convert new data to DataFrame new_data_df = pd.DataFrame([new_data]) # Encode the new data point and add encoded columns with "_id" suffix for column in new_data_df.select_dtypes(include=['object']).columns: new_data_df[column + '_id'] = label_encoders[column].transform(new_data_df[column]) # Standardize the new data using the encoded columns new_data_scaled = scaler.transform(new_data_df[[col + '_id' for col in new_data_df.select_dtypes(include=['object']).columns] + ['funded_amount']]) # Apply PCA to the new data new_data_pca = pca.transform(new_data_scaled) # Predict the cluster for the new data point new_cluster = kmeans.predict(new_data_pca)[0] st.subheader("Top 5 Similar Items to the Input") st.write(f"The new data point belongs to cluster: {new_cluster}") # Get all data points in the same cluster cluster_data = df_original[df_original['cluster'] == new_cluster] # Apply the same PCA transformation to the scaled data of the entire cluster cluster_data_pca = pca.transform(scaler.transform(cluster_data[encoded_columns + ['funded_amount']])) # Calculate the Euclidean distance between the new data point and all points in the same cluster distances = cdist(new_data_pca, cluster_data_pca, 'euclidean')[0] # Add distances to the cluster data DataFrame cluster_data = cluster_data.copy() cluster_data['distance'] = distances # Sort by distance and select the top 5 closest items top_5_recommendations = cluster_data.sort_values('distance').head(5) # Retrieve the original rows from the original DataFrame before encoding recommended_indices = top_5_recommendations['original_index'] recommendations = df_kiva_loans_cleaned.loc[recommended_indices] # Display the original rows as the top 5 recommendations st.write(recommendations) # Page 8: Hierarchical Clustering & Dendrogram elif page == "Hierarchical Clustering & Dendrogram": st.subheader("Hierarchical Clustering & Dendrogram") # User input to choose the number of sample rows sample_size = st.slider("Select the number of sample rows for clustering:", min_value=100, max_value=5000, value=150, step=50) # User input to choose the number of clusters n_clusters = st.slider("Select the number of clusters:", min_value=2, max_value=10, value=4, step=1) # Sample the selected number of rows from the DataFrame df_sample = df_kiva_loans_cleaned.sample(n=sample_size, random_state=42).copy() # Keeping only the relevant columns and storing original indices df_original = df_sample[['country','funded_amount', 'sector','repayment_interval']].copy() df_original['original_index'] = df_sample.index # Keep track of original indices # Label Encoding for categorical variables and adding encoded columns with "_id" suffix label_encoders = {} for column in df_original.select_dtypes(include=['object']).columns: le = LabelEncoder() df_original[column + '_id'] = le.fit_transform(df_original[column]) label_encoders[column] = le # Standardizing the data using the encoded columns encoded_columns = [col + '_id' for col in df_original.select_dtypes(include=['object']).columns] scaler = StandardScaler() df_scaled = scaler.fit_transform(df_original[encoded_columns + ['funded_amount']]) # Applying PCA pca = PCA(n_components=2) # Reduce to 2 dimensions for visualization df_pca = pca.fit_transform(df_scaled) # Perform Agglomerative Clustering with dynamic n_clusters agg_clustering = AgglomerativeClustering(n_clusters=n_clusters, linkage='ward') df_original['cluster'] = agg_clustering.fit_predict(df_pca) # Plot the resulting clusters plt.figure(figsize=(10, 7)) sns.scatterplot(x=df_pca[:, 0], y=df_pca[:, 1], hue=df_original['cluster'], palette='viridis', s=50) plt.title(f'Agglomerative Clustering (Hierarchical) Results - {n_clusters} Clusters') plt.xlabel('Principal Component 1') plt.ylabel('Principal Component 2') st.pyplot(plt.gcf()) # Dendrogram Visualization linked = linkage(df_pca, method='ward') plt.figure(figsize=(10, 7)) dendrogram(linked, orientation='top', distance_sort='descending', show_leaf_counts=True) plt.title('Hierarchical Clustering Dendrogram') st.pyplot(plt.gcf())