Machine-Learning
/
aiuNN
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

downsized samples to max of 10k

This commit is contained in:
Falko Victor Habel 2025-02-21 21:47:06 +01:00
parent 6a31e8fa7f
commit b06dad23af
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/aiunn/finetune.py
View File

@ -13,7 +13,7 @@ class UpscaleDataset(Dataset):
combined_df = pd.DataFrame()
for parquet_file in parquet_files:
# Load data with chunking for memory efficiency
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(10000)
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(5000)
combined_df = pd.concat([combined_df, df], ignore_index=True)
# Validate data format