# import pandas as pd # import re # import csv # import os # def detect_header_line(path, max_rows=10): # with open(path, 'r', encoding='utf-8', errors='ignore') as f: # lines = [next(f) for _ in range(max_rows)] # header_line_idx = 0 # best_score = -1 # for i, line in enumerate(lines): # cells = re.split(r'[;,|\t]', line.strip()) # alpha_ratio = sum(bool(re.search(r'[A-Za-z]', c)) for c in cells) / max(len(cells), 1) # digit_ratio = sum(bool(re.search(r'\d', c)) for c in cells) / max(len(cells), 1) # score = alpha_ratio - digit_ratio # if score > best_score: # best_score = score # header_line_idx = i # return header_line_idx # def detect_delimiter(path, sample_size=2048): # with open(path, 'r', encoding='utf-8', errors='ignore') as f: # sample = f.read(sample_size) # sniffer = csv.Sniffer() # try: # dialect = sniffer.sniff(sample) # return dialect.delimiter # except Exception: # for delim in [',', ';', '\t', '|']: # if delim in sample: # return delim # return ',' # def read_csv(path: str): # ext = os.path.splitext(path)[1].lower() # ambil ekstensi file # try: # if ext in ['.csv', '.txt']: # # === Baca file CSV === # header_line = detect_header_line(path) # delimiter = detect_delimiter(path) # print(f"[INFO] Detected header line: {header_line + 1}, delimiter: '{delimiter}'") # df = pd.read_csv(path, header=header_line, sep=delimiter, encoding='utf-8', low_memory=False, thousands=',') # elif ext in ['.xlsx', '.xls']: # # === Baca file Excel === # print(f"[INFO] Membaca file Excel: {os.path.basename(path)}") # pre_df = pd.read_excel(path, header=0, dtype=str) # baca semua sebagai string # df = pre_df.copy() # for col in df.columns: # if df[col].str.replace(',', '', regex=False).str.match(r'^-?\d+(\.\d+)?$').any(): # df[col] = df[col].str.replace(',', '', regex=False) # df[col] = pd.to_numeric(df[col], errors='ignore') # else: # raise ValueError("Format file tidak dikenali (hanya .csv, .txt, .xlsx, .xls)") # except Exception as e: # print(f"[WARN] Gagal membaca file ({e}), fallback ke default") # df = pd.read_csv(path, encoding='utf-8', low_memory=False, thousands=',') # # Bersihkan kolom dan baris kosong # df = df.loc[:, ~df.columns.astype(str).str.contains('^Unnamed')] # df.columns = [str(c).strip() for c in df.columns] # df = df.dropna(how='all') # return df import pandas as pd import re import csv import os def detect_header_line(path, max_rows=10): with open(path, 'r', encoding='utf-8', errors='ignore') as f: lines = [next(f) for _ in range(max_rows)] header_line_idx = 0 best_score = -1 for i, line in enumerate(lines): cells = re.split(r'[;,|\t]', line.strip()) alpha_ratio = sum(bool(re.search(r'[A-Za-z]', c)) for c in cells) / max(len(cells), 1) digit_ratio = sum(bool(re.search(r'\d', c)) for c in cells) / max(len(cells), 1) score = alpha_ratio - digit_ratio if score > best_score: best_score = score header_line_idx = i return header_line_idx def detect_delimiter(path, sample_size=2048): with open(path, 'r', encoding='utf-8', errors='ignore') as f: sample = f.read(sample_size) sniffer = csv.Sniffer() try: dialect = sniffer.sniff(sample) return dialect.delimiter except Exception: for delim in [',', ';', '\t', '|']: if delim in sample: return delim return ',' # def read_csv(path: str): # ext = os.path.splitext(path)[1].lower() # try: # if ext in ['.csv']: # # === Baca file CSV === # header_line = detect_header_line(path) # delimiter = detect_delimiter(path) # print(f"[INFO] Detected header line: {header_line + 1}, delimiter: '{delimiter}'") # df = pd.read_csv(path, header=header_line, sep=delimiter, encoding='utf-8', low_memory=False, thousands=',') # elif ext in ['.xlsx', '.xls']: # # === Baca file Excel === # print(f"[INFO] Membaca file Excel: {os.path.basename(path)}") # xls = pd.ExcelFile(path) # print(f"[INFO] Ditemukan {len(xls.sheet_names)} sheet: {xls.sheet_names}") # # Evaluasi tiap sheet untuk mencari yang paling relevan # best_sheet = None # best_score = -1 # best_df = None # for sheet_name in xls.sheet_names: # try: # df = pd.read_excel(xls, sheet_name=sheet_name, header=0, dtype=str) # df = df.dropna(how='all').dropna(axis=1, how='all') # if len(df) == 0 or len(df.columns) < 2: # continue # # hitung "skor relevansi" # text_ratio = df.applymap(lambda x: isinstance(x, str)).sum().sum() / (df.size or 1) # row_score = len(df) # score = (row_score * 0.7) + (text_ratio * 100) # if score > best_score: # best_score = score # best_sheet = sheet_name # best_df = df # except Exception as e: # print(f"[WARN] Gagal membaca sheet {sheet_name}: {e}") # continue # if best_df is not None: # print(f"[INFO] Sheet terpilih: '{best_sheet}' dengan skor {best_score:.2f}") # df = best_df # else: # raise ValueError("Tidak ada sheet valid yang dapat dibaca.") # # Konversi tipe numerik jika ada # for col in df.columns: # if df[col].astype(str).str.replace(',', '', regex=False).str.match(r'^-?\d+(\.\d+)?$').any(): # df[col] = df[col].astype(str).str.replace(',', '', regex=False) # df[col] = pd.to_numeric(df[col], errors='ignore') # else: # raise ValueError("Format file tidak dikenali (hanya .csv, .xlsx, .xls)") # except Exception as e: # print(f"[WARN] Gagal membaca file ({e}), fallback ke default reader.") # df = pd.read_csv(path, encoding='utf-8', low_memory=False, thousands=',') # # Bersihkan kolom dan baris kosong # df = df.loc[:, ~df.columns.astype(str).str.contains('^Unnamed')] # df.columns = [str(c).strip() for c in df.columns] # df = df.dropna(how='all') # return df def read_csv(path: str, sheet: str = None): ext = os.path.splitext(path)[1].lower() try: if ext in ['.csv']: # === Baca file CSV === header_line = detect_header_line(path) delimiter = detect_delimiter(path) print(f"[INFO] Detected header line: {header_line + 1}, delimiter: '{delimiter}'") df = pd.read_csv( path, header=header_line, sep=delimiter, encoding='utf-8', low_memory=False, thousands=',' ) elif ext in ['.xlsx', '.xls']: # === Baca file Excel === print(f"[INFO] Membaca file Excel: {os.path.basename(path)}") xls = pd.ExcelFile(path) print(f"[INFO] Ditemukan {len(xls.sheet_names)} sheet: {xls.sheet_names}") # === Jika user memberikan nama sheet === if sheet: if sheet not in xls.sheet_names: raise ValueError(f"Sheet '{sheet}' tidak ditemukan dalam file {os.path.basename(path)}") print(f"[INFO] Membaca sheet yang ditentukan: '{sheet}'") df = pd.read_excel(xls, sheet_name=sheet, header=0, dtype=str) df = df.dropna(how='all').dropna(axis=1, how='all') else: # === Auto-detect sheet terbaik === print("[INFO] Tidak ada sheet yang ditentukan, mencari sheet paling relevan...") best_sheet = None best_score = -1 best_df = None for sheet_name in xls.sheet_names: try: temp_df = pd.read_excel(xls, sheet_name=sheet_name, header=0, dtype=str) temp_df = temp_df.dropna(how='all').dropna(axis=1, how='all') if len(temp_df) == 0 or len(temp_df.columns) < 2: continue # hitung skor relevansi text_ratio = temp_df.applymap(lambda x: isinstance(x, str)).sum().sum() / (temp_df.size or 1) row_score = len(temp_df) score = (row_score * 0.7) + (text_ratio * 100) if score > best_score: best_score = score best_sheet = sheet_name best_df = temp_df except Exception as e: print(f"[WARN] Gagal membaca sheet {sheet_name}: {e}") continue if best_df is not None: print(f"[INFO] Sheet terpilih: '{best_sheet}' dengan skor {best_score:.2f}") df = best_df else: raise ValueError("Tidak ada sheet valid yang dapat dibaca.") # Konversi tipe numerik jika ada for col in df.columns: if df[col].astype(str).str.replace(',', '', regex=False).str.match(r'^-?\d+(\.\d+)?$').any(): df[col] = df[col].astype(str).str.replace(',', '', regex=False) df[col] = pd.to_numeric(df[col], errors='ignore') else: raise ValueError("Format file tidak dikenali (hanya .csv, .xlsx, .xls)") except Exception as e: print(f"[WARN] Gagal membaca file ({e}), fallback ke default reader.") df = pd.read_csv(path, encoding='utf-8', low_memory=False, thousands=',') # Bersihkan kolom dan baris kosong df = df.loc[:, ~df.columns.astype(str).str.contains('^Unnamed')] df.columns = [str(c).strip() for c in df.columns] df = df.dropna(how='all') return df