preprocess.py

from sklearn.model_selection import train_test_split
from diff_match_patch import diff_match_patch
from multiprocessing import Pool, cpu_count
from gensim.models import CoherenceModel
from jenkspy import JenksNaturalBreaks
from sklearn.impute import KNNImputer
import gensim.corpora as corpora
import pandas as pd
import numpy as np
import logging
import gensim
import zeyrek
import spacy
import nltk
import json
import re
WRITE_OUTPUT = True # işlenmiş veriyi kaydetmek için True yapın
preprocess_options = {
    "impute_missing_numeric_with": "KNN", # KNN, median, mean, 0, nan
    "impute_missing_categorical_with": "mode", # mode, nan, nan
    "output_path": "preprocessed.csv",
    "outlier_treatment": "nan", # Sınırlamak için cap, medyan için median, NaN için nan
    "outlier_method": "iqr", # iqr ya da zscore
    "outlier_iqr_multiplier": 5, # IQR için çarpan
    "outlier_zscore_threshold": 3.0, # Z-Score için eşik
    "add_topic_features": True, # LDA ile konuların eklenip eklenmeyeceği, k-fold için kapatın
}
DATA_PATH = "raw.csv"
logging.getLogger('nltk').setLevel(logging.ERROR)
logging.getLogger('zeyrek').setLevel(logging.ERROR)
df = pd.read_csv(DATA_PATH)
stop_words = open("stopwords.txt", "r").read().splitlines()
#Sütunları listingId'ye göre eşsiz olacak şekilde filtreliyoruz, 11 adet aynı id'li ilan var
def remove_duplicates():
    orig_amount= df.shape[0]
    df.drop_duplicates(subset='listingId', keep='first', inplace=True)
    print(orig_amount - df.shape[0], " adet satır eşsiz olmadığı için kaldırıldı.")

#TODO: Alakasız şehirlerdeki ilanları kaldırılacak
def remove_irrelevant_cities():
    global df
    target_city = "Sakarya"
    verbose = True
    
    # JSON string içinden şehir adı çekme
    def extract_city_name(val):
        try:
            return json.loads(val)["name"]
        except:
            return None

    # 'city' sütunundaki şehir isimleri
    df["city_name"] = df["city"].astype(str).apply(extract_city_name)

    # Silme işleminden önceki satır sayısı
    rows_before = df.shape[0]

    # Sakarya olmayan şehirler
    other_cities = df[df["city_name"] != target_city]["city_name"].value_counts()
    
    # Diğer illerden temizlenmiş df
    df = df[df["city_name"] == target_city].copy()
   
    # Silme işleminden sonraki satır sayısı
    rows_after = df.shape[0]

    if verbose:
        print(f"{rows_before - rows_after} satır farklı şehirden olduğu için kaldırıldı")
        if not other_cities.empty:
            print("Silinen şehirler:")
            for city, count in other_cities.items():
                print(f"   - {city}: {count} kayıt")
        else:
            print("Başka şehir verisi bulunamadı.")

    # Geçici city_name sütununu kaldırma
    df.drop(columns=["city_name"], inplace=True)

    return df

# 'netSqm':0 içeren satırları silen fonksiyon
def remove_land():
    global df
    # Silinecek satırları bulma
    matching_rows = df[df["sqm"].astype(str).str.contains(r'"netSqm"\s*:\s*0')]
    print(f"Silinecek satır sayısı: {len(matching_rows)}")
    print(matching_rows)  

    # Satırları silme işlemi
    df = df[~df["sqm"].astype(str).str.contains(r'"netSqm"\s*:\s*0')].reset_index(drop=True)

    # Kalanları kontrol etme işlemi
    remaining = df["sqm"].astype(str).str.contains(r'"netSqm"\s*:\s*0').sum()
    print(f"Kalan 'netSqm:0' içeren satır sayısı: {remaining}")


def fix_inconsistent_names():
    global train_df, test_df
    def get_most_common(df):
        not_json_dict = {}
        def process_column(col):
            id_name_counts = {}
            def process_val(val,is_recursive=False,key_prepend=""):
                if not is_recursive and (type(val)!=str or (not val.startswith("[") and not val.startswith("{"))):
                    if val is None or val == "" or val is np.nan:
                        return
                    not_json_dict[col.name] = val
                    return True
                val = json.loads(val) if not is_recursive else val
                items = val if isinstance(val, list) else [val]
                for item in items:
                    if not isinstance(item, dict) or 'id' not in item or 'name' not in item:
                        if is_recursive:
                            not_json_dict[col.name] = val
                        # attributes kısımları için array içeriyorsa her bir değer için de bu işlemi uyguluyoruz
                        has_list = False
                        if isinstance(item, dict):
                            for key in item:
                                if key is None:
                                    continue
                                if isinstance(item[key], list):
                                    process_val(item[key], True, key+"_")
                                    has_list = True
                        return not has_list
                    id_ = key_prepend + str(item['id'])
                    name = item['name']
                    if id_ not in id_name_counts:
                        id_name_counts[id_] = {}
                    if name not in id_name_counts[id_]:
                        id_name_counts[id_][name] = 0
                    id_name_counts[id_][name] += 1 + (3 if name and  "�" not in name else 0)
            for val in col:
                not_json=process_val(val)
                if not_json:
                    break

            id_to_most_common_name = {}
            for id_, name_counts in id_name_counts.items():
                most_common_name = max(name_counts.items(), key=lambda x: x[1])[0]
                id_to_most_common_name[id_] = most_common_name
            return id_to_most_common_name
        most_common_mapper = {}
        for col in df.columns:
            if col in df.columns and col not in most_common_mapper:
                most_common_mapper[col] = process_column(df[col])
        return most_common_mapper,not_json_dict
    def _process_column(id_to_most_common_name,not_json_dict):
        def col_mapper(col):
            def replace_names(val,key_prepend=""):
                is_recursive = key_prepend != ""
                if not is_recursive and (type(val)!=str or (not val.startswith("[") and not val.startswith("{"))):
                    return val
                items = json.loads(val) if not is_recursive else val
                is_arr = not is_recursive and val.startswith("[")
                if not is_arr:
                    items = [items]
                if len(items) == 0:
                    return val
                new_items=[]
                for item in items:
                    if item is None or item == "" or item is np.nan:
                        continue
                    is_dict = isinstance(item, dict)
                    dict_key = key_prepend + str(item["id"]) if is_dict and "id" in item else None
                    if is_dict and dict_key in id_to_most_common_name[col.name]:
                        item["name"] = id_to_most_common_name[col.name][dict_key]
                    if is_dict:
                        for key in item:
                            if isinstance(item[key], list):
                                item[key] = [replace_names(x, key+"_") for x in item[key]]
                    new_items.append(item)
                if not is_arr:
                    new_items = new_items[0]
                res=json.dumps(new_items,separators=(',', ':'),ensure_ascii=False) if not is_recursive else new_items
                return res
            if col.name in not_json_dict:
                return col
            return col.apply(replace_names)
        return col_mapper
    most_common,not_json_dict=get_most_common(train_df)
    process_column=_process_column(most_common,not_json_dict)
    train_df=train_df.apply(process_column)
    test_df=test_df.apply(process_column)

def fix_wrong_room_count():
    global df
    # "Stüdyo" "1+0" yap
    studio_mask = df['roomAndLivingRoom'] == '["Stüdyo"]'
    df.loc[studio_mask, 'roomAndLivingRoom'] = '["1+0"]'

    # "X+Y ve üzeri" "X+Y " şeklinde değiştir
    pattern = r'\["(\d+)\+(\d+)\sve üzeri"\]'
    df["roomAndLivingRoom"] = df["roomAndLivingRoom"].str.replace(
        pattern,
        lambda m: f'["{m.group(1)}+{m.group(2)}"]',
        regex=True
    )

    # "9", "9 ve üzeri" gibi olanları "9+""
    mask_9plus = df["roomAndLivingRoom"].str.startswith('["9')
    df.loc[mask_9plus, "roomAndLivingRoom"] = '["9+"]'

    # room ve livingRoom sayısal verilere dönüştürülüyor
    room = df['room'].str.extract(r'\[(\d+)\]').astype(float)[0]
    living = df['livingRoom'].str.extract(r'\[(\d+)\]').astype(float)[0]
    total = room.fillna(0) + living.fillna(0)

    structured = df['roomAndLivingRoom'].str.extract(r'\["(\d+)\+(\d+)"\]')
    structured_match = structured.notna().all(axis=1)

    structured_room = structured[0].astype(float)
    structured_living = structured[1].astype(float)

    # Uyuşmayanları bulma
    mismatch_structured = structured_match & (
        (room != structured_room) | (living != structured_living)
    )

    # "9+" olup toplamı 9'dan az olanları bul
    nine_plus_match = df['roomAndLivingRoom'].str.contains(r'\["9\+"\]', na=False)
    mismatch_nine_plus = nine_plus_match & (total < 9)

    df.loc[mismatch_structured, 'room'] = structured_room[mismatch_structured].apply(lambda x: f'[{int(x)}]')
    df.loc[mismatch_structured, 'livingRoom'] = structured_living[mismatch_structured].apply(lambda x: f'[{int(x)}]')

    df.loc[mismatch_nine_plus, 'room'] = '[9]'
    df.loc[mismatch_nine_plus, 'livingRoom'] = '[0]'

    return df
#Aynı değerleri taşıyan ancak yanlış isimlendirilmiş sütunların birleştirilmesi
def fix_wrong_column_names():
    global df

    # Hatalı sütun adlarını doğru sütun adlarıyla eşleştiren sözlük
    corrections = {
        'curr rency': 'currency',
        'ccurrency': 'currency',
        'curreency': 'currency',
        'ccurrencyId': 'currencyId',
        'enddDate': 'endDate',
        'endDaate': 'endDate',
        'are   eas': 'areas',
    }

    # Sütun adlarını sözlükteki düzeltmelere göre yeniden adlandır
    #df.rename(columns=corrections, inplace=True)

    # Birleştirilecek sütun kümeleri (aynı veriyi taşıyan ama adları farklı olanlar)
    merge_sets = {
        'currency': ['curr rency', 'ccurrency', 'curreency'],
        'currencyId': ['ccurrencyId'],
        'endDate': ['enddDate', 'endDaate'],
        'areas': ['are   eas'],
    }

    # Her doğru sütun için, hatalı olanlardan veriyi al ve birleştir
    for correct_col, duplicate_cols in merge_sets.items():
        for col in duplicate_cols:
            if col in df.columns:
                if correct_col in df.columns:
                    # Eğer doğru sütun varsa, eksik olan hücreleri hatalı sütundan doldur
                    df[correct_col] = df[correct_col].combine_first(df[col])
                else:
                    # Eğer doğru sütun yoksa, hatalı olanı doğrudan yeni sütun olarak ata
                    df[correct_col] = df[col]
                # Hatalı sütunu veri setinden kaldır
                df.drop(columns=[col], inplace=True)
                
def fix_wrong_sale_prices():
    price_column = 'price'
    print(df)
    print(f"Fixing wrong sale prices")
    # Fiyatı sayısal değere dönüştür
    df[price_column] = pd.to_numeric(df[price_column], errors='coerce')
    # 1-20 arasındaki fiyatları milyon ile 
    very_low_price_mask_1_20 = (df[price_column] > 0) & (df[price_column] <= 20)
    df.loc[very_low_price_mask_1_20, price_column] = df.loc[very_low_price_mask_1_20, price_column] * 1000000
    # 20-99 arasındaki fiyatları 100.000 ile
    very_low_price_mask_20_99 = (df[price_column] > 20) & (df[price_column] < 100)
    df.loc[very_low_price_mask_20_99, price_column] = df.loc[very_low_price_mask_20_99, price_column] * 100000
    # Şimdi düzeltilmiş verilerle istatistiksel aykırı değerleri tespit 
    q1 = df[price_column].quantile(0.25)
    q3 = df[price_column].quantile(0.75)
    iqr = q3 - q1
    lower_bound = q1 - 1.5 * iqr
    upper_bound = q3 + 1.5 * iqr
    # Aykırı değerleri tespit et
    outliers = (df[price_column] < lower_bound) | (df[price_column] > upper_bound)
    #print(f"Detected {outliers.sum()} outlier price values")
    extremely_high = df[price_column] > upper_bound
    if extremely_high.sum() > 0:
        pass
        #print(f"Adjusting {extremely_high.sum()} extremely high prices")
        #print("Extremely high prices:")
        #print(df.loc[extremely_high, price_column].head(10))
    
    # Çok düşük değerleri düzeltme
    extremely_low = df[price_column] < lower_bound
    if extremely_low.sum() > 0:
        pass
        #print(f"Adjusting {extremely_low.sum()} extremely low prices")
        #
        #print("Extremely low prices:")
        #print(df.loc[extremely_low, price_column].head(10))
# Kiralık fiyatına olmasına rağmen satılık ayarlanan ilanlar (varsa) düzeltilecek/silinecek
def remove_rental():
    global df
    price_column = 'price'
    
    # Fiyatı sayıya çevir
    df[price_column] = pd.to_numeric(df[price_column], errors='coerce')
    
    # 15.000 - 50.000 TL arası olan (kiralık gibi görünen) ilanları bul
    rental_like_mask = (df[price_column] >= 15000) & (df[price_column] <= 50000)
    
    # Silinecek ilanları ayrı bir DataFrame olarak tut
    removed_rows = df[rental_like_mask].copy()

    # Kaç ilan olduğunu yazdır
    print(f"{len(removed_rows)} adet ilan siliniyor (kiralık gibi).")

    # Silinecek ilanlardan örnek göster
    if not removed_rows.empty:
        print("\nSilinen ilanlardan örnek:")
        print(removed_rows[[price_column, 'listingId']].head())  # sadece 'price' ve 'listingId' gösteriliyor
       
    
    # Asıl verilerden bu ilanları çıkar
    df = df[~rental_like_mask].reset_index(drop=True)
    
def find_best_match_from_candidates(fuzzy_string, candidates, dmp_instance, threshold_ratio=0.6):
    if not fuzzy_string or not candidates:
        return None
    best_match_str = None
    highest_similarity_score = -1.0
    for candidate_str in candidates:
        if not isinstance(candidate_str, str) or candidate_str == fuzzy_string:
            continue
        diffs = dmp_instance.diff_main(fuzzy_string, candidate_str)
        lev_distance = dmp_instance.diff_levenshtein(diffs)
        max_len = max(len(fuzzy_string), len(candidate_str))
        if max_len == 0:
            similarity = 1.0 if lev_distance == 0 else 0.0
        else:
            similarity = 1.0 - (lev_distance / max_len)
        if similarity > highest_similarity_score:
            highest_similarity_score = similarity
            best_match_str = candidate_str
    if best_match_str and (highest_similarity_score >= threshold_ratio or len(fuzzy_string) < 5):
        return best_match_str
    return None

def correct_fuzzy_column_values(df, column_name, problem_char='�', similarity_threshold=0.75):
    dmp = diff_match_patch()
    unique_col_values = df[column_name].dropna().unique().tolist()
    clean_candidate_pool = [val for val in unique_col_values if isinstance(val, str) and problem_char not in val]
    no_space_candidates = {x.replace(" ", ""):x for x in clean_candidate_pool if " " in x}
    problematic_strings = [val for val in unique_col_values if isinstance(val, str) and (problem_char in val or val in no_space_candidates)]
    if not problematic_strings or not clean_candidate_pool:
        return df
    correction_map = {}
    for fuzzy_str in problematic_strings:
        best_replacement = find_best_match_from_candidates(fuzzy_str, clean_candidate_pool, dmp, threshold_ratio=similarity_threshold)
        if best_replacement and best_replacement != fuzzy_str:
            if problem_char not in best_replacement:
                correction_map[fuzzy_str] = best_replacement
    if correction_map:
        df[column_name] = df[column_name].replace(correction_map)
    return df

def apply_fuzzy_corrections_to_dataframe(input_df, problem_char='�', unique_values_threshold=100, similarity_score_threshold=0.75):
    df = input_df.copy()
    for col in df.columns:
        if df[col].dtype == 'object':
            num_unique = df[col].nunique()
            if num_unique < unique_values_threshold:
                df = correct_fuzzy_column_values(df, col, problem_char, similarity_score_threshold)
    return df
# Yanlış/alakasız sütunların silinmesi/düzeltilmesi, yazım yanlışlarının düzeltilmesi vb.
def fix_minor():
    global df
    build_state_mapper = {
        "Sifir": "Sıfır",
        'Ikinci El': "İkinci El",
        "0": "Sıfır",
        "Second Hand": "İkinci El",
    }

# Sütunlar içindeki ingilizce kelimeleri Türkçeye çevirme
    translation_map = {
        'Underfloor Heating': 'Yerden Isıtma',
        'Heating Stove': 'Odun Sobası',
        'Gas': 'Gaz',
        'Electric': 'Elektrik',
        'Reinforced Concrete': 'Betonarme',
        'Empty': 'Boş',
        'Not Available': 'Uygun değil',
        'Not Done': 'Yapılmaz',
        'Makes': 'Yapılır',
        'Owner': 'Mülk Sahibi',
        'Condominium': 'Kat Mülkiyeti',
        'Land': 'Arsa',
        'Residence': 'Rezidans',
        "Ground Floor": "Zemin",
        "Middle Floor": "Ara Kat",
        "2. Floor": "2. Kat",
        "3. Floor": "3. Kat",
        "Top Floor": "Çatı Katı",
    }
    low_unique_cols = [col for col in df.columns if df[col].nunique() < 30 and df[col].dtype == object]
    def translate_column_values(df):
        for col in low_unique_cols:
            unique_vals = df[col].dropna().unique()
            value_map = {val: translation_map[val] for val in unique_vals if val in translation_map}
            if value_map:
                df[col] = df[col].map(lambda x: value_map.get(x, x))
        return df

    df = translate_column_values(df)
    df["buildState"] = df["buildState"].fillna("İkinci El")
    df["buildState"]=df["buildState"].map(lambda x: build_state_mapper.get(x, x))
    df["heating"] = df["heating"].fillna("Belirtilmemiş")
    df=apply_fuzzy_corrections_to_dataframe(df)
def split_dataset():
    # 0.7 0.3
    global df, train_df, test_df
    df = df.sample(frac=1, random_state=42).reset_index(drop=True)
    jnb = JenksNaturalBreaks(5)
    class_labels = ["Çok Ucuz","Ucuz", "Orta", "Pahalı", "Çok Pahalı"]
    class_amount = len(class_labels)
    df["price_log"] = np.log(df["price"])
    jnb.fit(df['price_log'])
    df.drop(columns=["price_log"], inplace=True)
    price_ranges = sorted(list(map(lambda x: np.round(np.exp(x)), jnb.breaks_)))
    with open("price_classes.txt", "w") as f:
        for i in range(class_amount):
            f.write(f"{class_labels[i]} - {price_ranges[i]} - {price_ranges[i+1]}\n")
    df["price_class"] = pd.cut(df["price"], bins=price_ranges, labels=class_labels, include_lowest=True)
    train_df, test_df = train_test_split(df, test_size=0.2, random_state=42, stratify=df["price_class"])
    test_df.drop(columns=["price_class"], inplace=True)
    train_df.drop(columns=["price_class"], inplace=True)
    df.drop(columns=["price_class"], inplace=True)
    
def sanitize_text(text):
    text = re.sub(r'<[^>]+?>', '', text).replace("&nbsp;", " ")
    text = re.sub(r'\\n|[^a-zA-ZĞÜÜŞİÖÇığüşöç]|\n+|"|\'', ' ', text).strip()
    text = re.sub(r'\s+', ' ', text)
    text = text.replace("İ","i").lower().replace("ı", "i").replace("ğ", "g").replace("ü", "u").replace("ş", "s").replace("ö", "o").replace("ç", "c")
    text = re.sub(r'\s.\s', ' ', text)
    return text
def compute_topic_feature(args):
    df, n, topic_words, dmp, is_test = args
    col = f"topic_{n}"
    df[col] = df['description'].apply(
        lambda text: len([
            word for word in topic_words 
            if not pd.isna(text) and dmp.match_main(text, word[1], 0) != -1
        ])
    )
    if not is_test: print("Proccessed topic", n)
    return df[[col]]
def LDA(train_df,test_df,write_to_file=False):
    #nltk.download('punkt_tab')
    # Kontaminasyon olmaması adına konular yalnızca train'den alınacak, işlem ise verinin tamamına ayrı ayrı uygulanacak
    max_occurance = 2
    max_err_ratio = 0.1
    analyzer = zeyrek.MorphAnalyzer()
    nlp = spacy.blank("tr")
    nlp.max_length = 40000000
    classes = [x.split(" - ") for x in open("price_classes.txt", "r").read().splitlines()]
    sorted_by_price = train_df.sort_values(by='price')
    all_topics=[]
    for i in range(len(classes)):
        filtered = sorted_by_price[(sorted_by_price['price'] >= float(classes[i][1])) & (sorted_by_price['price'] <= float(classes[i][2]))]
        print(f"Filtered {len(filtered)} rows for class {classes[i][0]}")
        whole_text=sanitize_text(filtered["description"].astype(str).str.cat(sep=' '))
        tokens = whole_text.split()
        tokens = [token for token in tokens if token not in stop_words]
        doc = nlp(" ".join(tokens))
        lemmas = []
        known_lemmas={}
        counter =0
        for token in doc:
            counter += 1
            if counter % 1000 == 0:
                print(f"Processed {counter}/{len(doc)} tokens", end="\r")
            if str(token) in known_lemmas:
                lemmas.append(known_lemmas[str(token)])
                continue
            lemma=str(token)
            #analyzed = analyzer.lemmatize(str(token))
            #lemma=analyzed[0][1][0] if analyzed[0][1] else str(token)
            known_lemmas[str(token)] = lemma
            lemmas.append(lemma)
        id2word = corpora.Dictionary([lemmas])
        del known_lemmas
        corpus = [id2word.doc2bow(lemmas)]
        lda_model = gensim.models.ldamodel.LdaModel(corpus=corpus,
                                                id2word=id2word,
                                                num_topics=3, 
                                                random_state=123,
                                                update_every=0,
                                                chunksize=1000,
                                                passes=8,
                                                alpha='auto',
                                                per_word_topics=False)
        topics = lda_model.print_topics(num_words=50)
        all_topics.append(topics)
        for topic in topics:
            pass
            #print(classes[i][0]," Konu:",topic,"\n")
            #coherence_model_lda = CoherenceModel(model=lda_model, texts=lemmas, dictionary=id2word, coherence='c_v')
            #coherence_lda = coherence_model_lda.get_coherence()
            #print('\nCoherence Score: ', coherence_lda)
    print()
    general_seen_count={}
    parsed_topics=[]
    dmp = diff_match_patch()
    dmp.match_distance = 1000
    dmp.match_threshold = max_err_ratio
    i=0
    for topics in all_topics:
        seen={}
        curr_topics=[]
        for topic in topics:
            curr=topic[1].split(" + ")
            curr = [x.split("*") for x in curr]
            curr = [(float(x[0]), x[1].replace('"', "").strip()) for x in curr]
            curr = [x for x in curr if x[1] not in seen and x[0] >= 0.001]
            curr_topics.append(curr)
            for x in curr:
                if x[1] not in general_seen_count:
                    general_seen_count[x[1]] = 0
                if not x[1] in seen:
                    seen[x[1]] = 0
                    general_seen_count[x[1]] += 1
                seen[x[1]] += 1
        index=i//3
        parsed_topics.append(curr_topics)
        i+=1
    for i in range(len(parsed_topics)):
        for j in range(len(parsed_topics[i])):
            parsed_topics[i][j] = [(x[0], x[1]) for x in parsed_topics[i][j] if general_seen_count[x[1]] <= max_occurance]
            print("Topic", i*3+j, ":", parsed_topics[i][j])
    tasks = []
    topic_count=0
    for i in parsed_topics:
        for j in i:
            tasks.append((train_df.copy(), topic_count,  j,dmp, False))
            tasks.append((test_df.copy(), topic_count, j,dmp,True))
            topic_count += 1
    with Pool(cpu_count()) as pool:
        results = pool.map(compute_topic_feature, tasks)
        for i in range(len(results)):
            if i % 2 == 0:
                train_df = pd.concat([train_df, results[i]], axis=1)
            else:
                test_df = pd.concat([test_df, results[i]], axis=1)
    if write_to_file:
        train_df.to_csv("train.csv", index=False)
        test_df.to_csv("test.csv", index=False)

def safe_json_extract(val, key, default=np.nan, is_list_first_element=False, ignore_other=False):
    if pd.isna(val):
        return default
    try:
        data = json.loads(val)
        if isinstance(data, list):
            if ignore_other:
                data= [item for item in data if item["name"]!="Diğer"]
            if is_list_first_element and len(data) > 0 and isinstance(data[0], dict):
                return data[0].get(key, default)
            elif not is_list_first_element:
                 return len(data)
            return default 
        extracted = data.get(key, default)
        if is_list_first_element and isinstance(extracted, list) and len(extracted) > 0:
            return extracted[0]
        return extracted
    except (json.JSONDecodeError, TypeError, AttributeError):
        return default

def safe_list_extract_first(val, default=np.nan):
    if pd.isna(val):
        return default
    try:
        if isinstance(val, str) and val.startswith('[') and val.endswith(']'):
            val_cleaned = val[1:-1].strip('"')
            if not val_cleaned:
                return default
            return pd.to_numeric(val_cleaned, errors='coerce')
        if isinstance(val, list) and len(val) > 0:
            return pd.to_numeric(val[0], errors='coerce')
        return pd.to_numeric(val, errors='coerce')
    except: 
        return default
def fix_empty(opts):
    curr_opts={"impute_missing_numeric_with":"KNN", "impute_missing_categorical_with":"mode"}
    for col in opts:
        if "impute_missing" in col:
            curr_opts[col]= opts[col]
    #TODO: use options
    global df
    original_numeric_cols = ['price', 'currencyId', 'age', 'numberOfBuilding', 'entranceHeight', 'bedCount', 'floorAreaRatio', 'additionalGuestFee', 'cleaningFee', 'guestCount', 'maximumStay', 'minimumStay', 'doubleBedCount', 'singleBedCount', 'searchScore']
    for col in original_numeric_cols:
        if col in df.columns:
            df[col] = pd.to_numeric(df[col], errors='coerce')
        else:
            df[col] = np.nan
    numeric_cols_for_imputation = df.select_dtypes(include=np.number).columns.tolist()
    numeric_cols_for_imputation = [
        col for col in numeric_cols_for_imputation if df[col].notna().any()
    ]
    cols_to_exclude_from_imputation_if_numeric = ['no', 'realtyId']
    numeric_cols_for_imputation = [
        col for col in numeric_cols_for_imputation if col not in cols_to_exclude_from_imputation_if_numeric
    ]
    if numeric_cols_for_imputation and not train_df[numeric_cols_for_imputation].empty:
        train_df_numeric_subset = train_df[numeric_cols_for_imputation].copy()
        cols_with_nans_to_impute = train_df_numeric_subset.columns[train_df_numeric_subset.isnull().any()].tolist()
        if cols_with_nans_to_impute:
            imputer = KNNImputer(n_neighbors=5) 
            if len(train_df_numeric_subset) > 0:
                #yalnızca train verisinde fit
                imputer.fit(train_df_numeric_subset)
                df_imputed_values = imputer.transform(train_df_numeric_subset)
                df_imputed_subset = pd.DataFrame(df_imputed_values, columns=numeric_cols_for_imputation, index=train_df_numeric_subset.index)
                test_transformed_values = imputer.transform(test_df[numeric_cols_for_imputation])
                test_transformed = pd.DataFrame(test_transformed_values, columns=numeric_cols_for_imputation, index=test_df.index)
                for col in numeric_cols_for_imputation:
                    train_df[col] = df_imputed_subset[col]
                    test_df[col] = test_transformed[col]
    df["residence"].fillna(df["residence"].mode()[0], inplace=True)
    cols_to_drop=['timeShareTerm', 'tagProducts', 'mainCategory', 'additionalGuestFee', 'fee', 'rental', 'deposit', 'star', 'listingId.1', 'numberOfBuilding', 'streetView', 'cleaningFee', 'address', 'superRealty', 'entranceHeight', 'videoUrl', 'bedCount', 'bid', 'cleanLandRegisterStateTypeId', 'petAllowed', 'singleBedCount', 'doubleBedCount', 'hasBranded', 'fixedDailyPrice', 'minimumStay', 'balcony', 'parking', 'currencyId', 'activateId', 'groundStudies', 'dormitory', 'invoiceIncluded', 'byTransfer', 'populated', 'listingPropertyOrder', 'parentCategoryId', 'cleaningFeeCurrency', 'realtyOwnerUrl', 'category', 'holdDate', 'fixedDailyPriceCurrencyId', 'flatReceived', 'isMapHidden', 'checkInTime', 'floorAreaRatio', 'attributeCapacity', 'areaIds', 'realtyIdentificationNo', 'projectId', 'city', 'gabarite', 'imageCapacity', 'housingComplex', 'timeShareName', 'cleanLandRegisterFilePath', 'realtyPermitDocumentNo', 'guestCount', 'nearLocation', 'advertiseOwner', 'additionalGuestFeeCurrency', 'mapZoomLevel', 'subCategory', 'period', 'useContactInfo', 'checkOutTime', 'virtualTour', 'publishId', 'maximumStay', 'productCount', 'showPrice', 'housingEstate', 'locationId', 'registerId', 'showAddress', 'entertainmentAllowed', "redirectLink"]
    df.drop(columns=cols_to_drop, inplace=True)
    train_df.drop(columns=cols_to_drop, inplace=True)
    test_df.drop(columns=cols_to_drop, inplace=True)
def fix_outliers(train_df, test_df, opts,exclude_cols=None, columns=None):
    curr_opts={"method":"iqr", "treatment":"nan", "iqr_multiplier":5, "zscore_threshold":3.0}
    for col in opts:
        if "outlier" in col:
            curr_opts[col.split("_")[1]] = opts[col]
    method = curr_opts["method"]
    treatment = curr_opts["treatment"]
    iqr_multiplier = curr_opts["iqr_multiplier"]
    zscore_threshold = curr_opts["zscore_threshold"]
    for index, row in train_df.iterrows():
        if row['sqm_net'] > 2 * row['sqm_gross']:
            train_df.at[index, 'sqm_net'] = np.nan
    for index, row in test_df.iterrows():
        if row['sqm_net'] > 2 * row['sqm_gross']:
            test_df.at[index, 'sqm_net'] = np.nan
    if exclude_cols is None:
        default_excludes = ['realtyId', 'no', 'map_lat', 'map_lon',"price"]
        topic_pattern = re.compile(r'^topic_\d+$')
        date_pattern = re.compile(r'^(startDate|endDate|createdDate|updatedDate|listingUpdatedDate)')
        topic_and_date_cols = [col for col in train_df.columns if topic_pattern.match(col) or date_pattern.match(col)]
        exclude_cols = list(set(default_excludes + topic_and_date_cols))
    else:
        exclude_cols = list(set(exclude_cols))
    if columns is None:
        potential_cols = train_df.select_dtypes(include=np.number).columns.tolist()
        columns_to_process = [col for col in potential_cols if col not in exclude_cols]
    else:
        columns_to_process = [
            col for col in columns if col in train_df.columns and \
            pd.api.types.is_numeric_dtype(train_df[col])
        ]
        if not columns_to_process:
            return train_df, test_df

    for col in columns_to_process:
        if train_df[col].isnull().all():
            continue
        lower_bound = np.nan
        upper_bound = np.nan
        train_col_median_for_treatment = np.nan
        if method == 'iqr':
            Q1 = train_df[col].quantile(0.25)
            Q3 = train_df[col].quantile(0.75)
            IQR = Q3 - Q1
            if IQR == 0:
                continue
            lower_bound = Q1 - iqr_multiplier * IQR
            upper_bound = Q3 + iqr_multiplier * IQR
        elif method == 'zscore':
            col_mean_train = train_df[col].mean()
            col_std_train = train_df[col].std()
            if col_std_train == 0 or pd.isna(col_std_train):
                continue
            lower_bound = col_mean_train - zscore_threshold * col_std_train
            upper_bound = col_mean_train + zscore_threshold * col_std_train
        else:
            continue
        if treatment == 'median':
            train_col_median_for_treatment = train_df[col].median()
            if pd.isna(train_col_median_for_treatment):
                continue
        outlier_indices_lower_train = train_df[train_df[col] < lower_bound].index
        outlier_indices_upper_train = train_df[train_df[col] > upper_bound].index
        all_outlier_indices_train = outlier_indices_lower_train.union(outlier_indices_upper_train)
        num_outliers_train = len(all_outlier_indices_train)
        if num_outliers_train > 0:
            print(num_outliers_train, "outliers in column", col)
            if treatment == 'cap':
                train_df.loc[outlier_indices_lower_train, col] = lower_bound
                train_df.loc[outlier_indices_upper_train, col] = upper_bound
            elif treatment == 'nan':
                train_df.loc[all_outlier_indices_train, col] = np.nan
            elif treatment == 'median':
                train_df.loc[all_outlier_indices_train, col] = train_col_median_for_treatment
        if col in test_df.columns:
            if test_df[col].isnull().all():
                continue
            else:
                outlier_indices_lower_test = test_df[test_df[col] < lower_bound].index
                outlier_indices_upper_test = test_df[test_df[col] > upper_bound].index
                all_outlier_indices_test = outlier_indices_lower_test.union(outlier_indices_upper_test)
                num_outliers_test = len(all_outlier_indices_test)
                if num_outliers_test > 0:
                    if treatment == 'cap':
                        test_df.loc[outlier_indices_lower_test, col] = lower_bound
                        test_df.loc[outlier_indices_upper_test, col] = upper_bound
                    elif treatment == 'nan':
                        test_df.loc[all_outlier_indices_test, col] = np.nan
                    elif treatment == 'median':
                        test_df.loc[all_outlier_indices_test, col] = train_col_median_for_treatment
def fix_outliers_and_empty(train_df, test_df):
    fix_outliers(train_df,test_df,preprocess_options)
    fix_empty(preprocess_options)
def parse_json():
    global df
    date_cols = ['startDate', 'endDate', 'createdDate', 'updatedDate', 'listingUpdatedDate', 'holdDate']
    for col in date_cols:
        if col in df.columns:
            df[col] = pd.to_datetime(df[col], errors='coerce').astype(np.int64) // 10**9
    bool_cols_direct = [
        'showAddress', 'furnished', 'balcony', 'parking', 
        'byTransfer', 'populated', 'rental', 'authorizedRealtor', 
        'stale', 'hasBranded', 'hasUpdateBooster', 
        'entertainmentAllowed', 'invoiceIncluded', 'petAllowed'
    ]
    for col in bool_cols_direct:
        if col in df.columns:
            new_col_name = col
            map_dict = {
                True: 1, False: 0,
                'True': 1, 'False': 0,
                'true': 1, 'false': 0,
                '1': 1, '0': 0,
                1: 1, 0: 0,           
                1.0: 1, 0.0: 0        
            }
            df[new_col_name] = df[col].map(map_dict)
            df[new_col_name] = pd.to_numeric(df[new_col_name], errors='coerce')
    df['desc_phone_count'] = df['descriptionPhoneNumbers'].apply(lambda x: safe_json_extract(x,key=None, is_list_first_element=False))
    df['image_count'] = df['images'].apply(lambda x: safe_json_extract(x, key=None, is_list_first_element=False))
    df['areas'] = df['areas'].apply(lambda x: safe_json_extract(x, 'name', is_list_first_element=True,ignore_other=True)).fillna("Diğer")
    df['district_tier'] = df['district'].apply(lambda x: safe_json_extract(x, 'tier'))
    df['district'] = df['district'].apply(lambda x: safe_json_extract(x, 'name'))
    df['county'] = df['county'].apply(lambda x: safe_json_extract(x, 'name'))
    df['city'] = df['city'].apply(lambda x: safe_json_extract(x, 'name'))
    df['residence'] = df['residence'].apply(lambda x: safe_json_extract(x, 'name'))
    df['sqm_net'] = df['sqm'].apply(lambda x: safe_json_extract(x, 'netSqm'))
    df['sqm_gross'] = df['sqm'].apply(lambda x: safe_json_extract(x, 'grossSqm', is_list_first_element=True))
    df['floor_count'] = df['floor'].apply(lambda x: safe_json_extract(x, 'count'))
    df["floor_name"] = df['floor'].apply(lambda x: safe_json_extract(x, 'name'))
    json_id_cols = ['heating', 'fuel', 'build', 'buildState', 'usage', 'credit', 'barter', 'flatReceived',
                    'parentCategoryId', 'category', 'subCategory', 'mainCategory']
    for col in json_id_cols:
        if col in df.columns:
            df[col]= df[col].apply(lambda x: safe_json_extract(x, 'name'))
    df["usage"] = df["usage"].fillna("Belirtilmemiş")
    df['fee_amount'] = df['fee'].apply(lambda x: safe_json_extract(x, 'amount'))
    list_extract_cols = ['room', 'livingRoom', 'bathRoom']
    for col in list_extract_cols:
        if col in df.columns:
            df[col] = df[col].apply(safe_list_extract_first)
    df['map_lat'] = df['mapLocation'].apply(lambda x: safe_json_extract(x, 'lat'))
    df['map_lon'] = df['mapLocation'].apply(lambda x: safe_json_extract(x, 'lon'))
    register_state_normalization_map = {
        'katmülkiyeti': 'Kat Mülkiyeti',
        'condominium': 'Kat Mülkiyeti',
        'katm�lkiyeti': 'Kat Mülkiyeti',
        'katmülkiyeti': 'Kat Mülkiyeti',    
        'flooreasement': 'Kat İrtifakı',
        'katirtifaki': 'Kat İrtifakı',    
        'kat�rtifak�': 'Kat İrtifakı',  
        'land': 'Arsa',
        'tapuyok': 'Tapu Yok',
        'hisselitapu': 'Hisseli Tapu',
        'yabancidan': 'Yabancıdan Satılık',
    }
    df['registerState'] = df['registerState'].fillna("Bilinmiyor").map(lambda x: register_state_normalization_map.get(x.replace(" ", "").replace("ı","i").replace("İ","i").lower(), x))

    category_mapper = {
        "mustakil":"Müstakil",
        "ciftlik":"Çiftlik",
        "yazlik":"Yazlık",
        "kosk":"Köşk",
        "koy":"Köy", # Köy evi için
    }
    df["subcategory"] = df["redirectLink"].apply(lambda x: " ".join(map(lambda e:category_mapper.get(e,e).title(),safe_json_extract(x, "linkedSubcategoryUrl").split("-"))))

def preprocess():
    remove_duplicates()
    remove_irrelevant_cities()
    remove_land()
    # diğer verilerden bağımsız gerçekleşen ön işlemele aşamaları
    fix_wrong_room_count()
    fix_wrong_column_names()
    #fix_wrong_sale_prices()
    remove_rental()
    parse_json()
    fix_minor()
    split_dataset()
    # Konu analizi, tutarsız (farklı dilde, türkçe harfleri vs. içermeyen ama id'si aynı olan) adlandırmaların bulunması, 
    # aykırıların bulunması, boş değerlerin doldurulmasını yalnızca train verisine göre gerçekleştirdik
    fix_inconsistent_names()
    fix_outliers_and_empty(train_df, test_df)
    if preprocess_options["add_topic_features"]:
        LDA(train_df,test_df,True)

if __name__ == "__main__":
    preprocess()
    print(df)
    if WRITE_OUTPUT:
        df.to_csv(preprocess_options["output_path"], index=False)