finetune_class #1
|
@ -13,7 +13,7 @@ class UpscaleDataset(Dataset):
|
||||||
combined_df = pd.DataFrame()
|
combined_df = pd.DataFrame()
|
||||||
for parquet_file in parquet_files:
|
for parquet_file in parquet_files:
|
||||||
# Load data with chunking for memory efficiency
|
# Load data with chunking for memory efficiency
|
||||||
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(5000)
|
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(4000)
|
||||||
combined_df = pd.concat([combined_df, df], ignore_index=True)
|
combined_df = pd.concat([combined_df, df], ignore_index=True)
|
||||||
|
|
||||||
# Validate data format
|
# Validate data format
|
||||||
|
|
Loading…
Reference in New Issue