Adapt to latest Library

.gitignore

@@ -1,6 +1,7 @@
 Data/
 data/
 poster/
+env/
 
 *.csv
 *.xlsx

Main/data_cleanup.py

@@ -12,8 +12,8 @@
 # Importing the dataset
 filename = '../Data/listings.csv'
 reviews_filename = '../Data/reviews_cleaned.csv'
-data = pd.read_csv(filename)
-reviews = pd.read_csv(reviews_filename, names = ['listing_id', 'comments'])
+data = pd.read_csv(filename, low_memory=False)
+reviews = pd.read_csv(reviews_filename, names = ['listing_id', 'comments'], low_memory=False)
 # print(data.info)
 # print(list(data))
 # print(list(data)[43])
@@ -22,7 +22,7 @@
 
 # Taking out the unwanted columns
 print(len(data.columns))
-exit()
+# exit()
 data = pd.DataFrame.drop(data, columns=[
     'host_name',
     'notes', # Added PRK
@@ -86,11 +86,13 @@
 host_verification_set = set()
 
 def collect_host_verifications(entry):
-    entry_list = entry.replace("[", "").replace("]", "").replace("'", "").replace('"', "").replace(" ", "").split(',')
-    for verification in entry_list:
-        if (verification != "" and verification != 'None'):
-            host_verification_set.add(verification +"_verification")
+    if isinstance(entry, str):  # Periksa apakah entry adalah string
+        entry_list = entry.replace("[", "").replace("]", "").replace("'", "").replace('"', "").replace(" ", "").split(',')
+        for verification in entry_list:
+            if verification != "" and verification != 'None':
+                host_verification_set.add(verification + "_verification")
 
+data['host_verifications'] = data['host_verifications'].fillna('')
 data['host_verifications'].apply(collect_host_verifications)
 
 def generic_verification(entry, v):

Main/run_models.py

@@ -1,3 +1,8 @@
+import os
+os.environ['TF_ENABLE_ONEDNN_OPTS'] = '0'
+import warnings
+warnings.filterwarnings("ignore", category=FutureWarning)
+warnings.filterwarnings("ignore", category=UserWarning)
 import matplotlib.pyplot as plt
 import numpy as np
 import pandas as pd
@@ -39,6 +44,9 @@
 
 import copy
 
+
+
+
 # NN parameters
 NUM_ITERATIONS = 300
 BATCH_SIZE = 256
@@ -60,7 +68,7 @@ def get_mlp_regressor(num_hidden_units=51):
 
 
 def get_ensemble_models():
-    grad = GradientBoostingRegressor(n_estimators=17, random_state=42, loss='lad', learning_rate=0.12, max_depth=10)
+    grad = GradientBoostingRegressor(n_estimators=17, random_state=42, loss='absolute_error', learning_rate=0.12, max_depth=10)
     classifier_list = [grad]
     classifier_name_list = ['Gradient Boost']
     return classifier_list, classifier_name_list
@@ -130,14 +138,25 @@ def LinearModelLasso(X_train, y_train, X_val, y_val):
 
 
 def simple_neural_network(X_train, y_train, X_val, y_val):
+    # Pastikan tidak ada nilai NaN atau None
+    X_train = X_train.fillna(0)  # Mengganti NaN dengan 0, atau Anda bisa menggunakan strategi lainnya
+    y_train = y_train.fillna(0)
+
     model = Sequential()
-    model.add(Dense(units=20, activation='relu', input_dim=len(X_train.values[0])))
+    model.add(Dense(units=20, activation='relu', input_dim=X_train.shape[1]))  # Perbaikan input_dim
     model.add(Dense(units=5, activation='relu'))
     model.add(Dense(units=1, activation='linear'))
-    adam = optimizers.Adam(lr=LEARNING_RATE, beta_1=0.9, beta_2=0.999, epsilon=None, decay=DECAY_RATE, amsgrad=False)
+
+    # Pastikan LEARNING_RATE dan DECAY_RATE didefinisikan sebelumnya
+    adam = optimizers.Adam(learning_rate=LEARNING_RATE, beta_1=0.9, beta_2=0.999, decay=DECAY_RATE, amsgrad=False)
+
     model.compile(loss='mean_squared_error', optimizer=adam)
+
+    # Pastikan data tidak kosong dan dimensi sesuai
+    print(f"Training with data shape: X_train: {X_train.shape}, y_train: {y_train.shape}")
     model.fit(X_train, y_train, epochs=NUM_ITERATIONS, batch_size=BATCH_SIZE)
-    print("finished fitting")
+
+    print("Finished fitting")
     print_evaluation_metrics(model, "NN", X_val, y_val)
     print_evaluation_metrics2(model, "NN", X_train, y_train)
     return
@@ -161,7 +180,7 @@ def TreebasedModel(X_train, y_train, X_val, y_val):
 
 def kmeans(X_train, y_train, X_val, y_val):
     n_clusters = 8
-    kmeans = KMeans(n_clusters=n_clusters, random_state=0, verbose=0, n_jobs=int(0.8*n_cores)).fit(X_train)
+    kmeans = KMeans(n_clusters=n_clusters, random_state=0, verbose=0).fit(X_train)
     c_train = kmeans.predict(X_train)
     c_pred = kmeans.predict(X_val)
     centroids = kmeans.cluster_centers_
@@ -174,11 +193,11 @@ def kmeans(X_train, y_train, X_val, y_val):
         train_mask = c_train==i
         std_train = np.std(y_train[train_mask])
         mean_train = np.mean(y_train[train_mask])
-        print("# examples & price mean & std for training set within cluster %d is:(%d, %.2f, %.2f)" %(i, train_mask.sum(), np.float(mean_train), np.float(std_train)))
+        print("# examples & price mean & std for training set within cluster %d is:(%d, %.2f, %.2f)" %(i, train_mask.sum(), float(mean_train), float(std_train)))
         pred_mask = c_pred==i
         std_pred = np.std(y_val[pred_mask])
         mean_pred = np.mean(y_val[pred_mask])
-        print("# examples & price mean & std for validation set within cluster %d is:(%d, %.2f, %.2f)" %(i, pred_mask.sum(), np.float(mean_pred), np.float(std_pred)))
+        print("# examples & price mean & std for validation set within cluster %d is:(%d, %.2f, %.2f)" %(i, pred_mask.sum(), float(mean_pred), float(std_pred)))
         if pred_mask.sum() == 0:
             print('Zero membered test set! Skipping the test and training validation.')
             continue
@@ -194,8 +213,8 @@ def kmeans(X_train, y_train, X_val, y_val):
             labels_stats = copy.deepcopy(labels_pred)
 
         else:
-            y_val_stats = y_val_stats.append(y_val[pred_mask])
-            y_train_stats = y_train_stats.append(y_train[train_mask])
+            y_val_stats = pd.concat([y_val_stats, y_val[pred_mask]])
+            y_train_stats = pd.concat([y_train_stats, y_train[train_mask]])
             predicted_values = np.append(predicted_values, y_pred)
             labels_stats = np.append(labels_stats, labels_pred)
         print('--------Finished analyzing cluster %d--------' %i)
@@ -237,8 +256,8 @@ def linear_model_SGD(X_train, y_train, X_val, y_val):
     X_test = pd.read_csv('../Data/data_cleaned_test_comments_X.csv')
     y_test = pd.read_csv('../Data/data_cleaned_test_y.csv')
 
-    #coeffs = np.load('../Data/selected_coefs_pvals.npy')
-    coeffs = np.load('../Data/selected_coefs.npy')
+    coeffs = np.load('../Data/selected_coefs_pvals.npy', allow_pickle=True)
+    #coeffs = np.load('../Data/selected_coefs.npy')
     col_set = set()
     cherry_picked_list = [
     'host_identity_verified',
@@ -273,10 +292,10 @@ def linear_model_SGD(X_train, y_train, X_val, y_val):
     y_concat = pd.concat([y_train, y_val], ignore_index=True)
 
     #RUN WITHOUT FEATURE SELECTION FOR THE BASELINE
-    """
+
     print("--------------------Linear Regression--------------------")
     LinearModel(X_concat, y_concat, X_test, y_test)
-    """
+
 
     print("--------------------Tree-based Model--------------------")
     TreebasedModel(X_concat, y_concat, X_test, y_test)

requirements.txt

@@ -1,10 +1,11 @@
-numpy==1.16.2
-matplotlib==2.2.2
-Keras==2.2.4
-scipy==1.1.0
-pandas==0.24.2
-Shapely==1.6.4.post2
-geopandas==0.3.0
+numpy==1.23
+matplotlib==3.9.2
+Keras>=2.4.0
+scipy>=1.6.0
+pandas>=1.0.0
+Shapely==2.0.6
+geopandas>=0.8.0
 textblob==0.15.1
-statsmodels==0.9.0
-scikit_learn==0.21.3
+statsmodels>=0.12.0
+scikit-learn>=1.5.2
+tensorflow>=2.5.0