Machine-Learning
/
aiuNN
1
0
Fork 0
Code Issues 1 Pull Requests Actions Packages Projects Releases Wiki Activity

downsized batchsize increased images

This commit is contained in:
Falko Victor Habel 2025-02-24 16:39:25 +01:00
parent 153ead15b7
commit 79e0aeb269
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/aiunn/finetune.py
View File

@ -41,7 +41,7 @@ class UpscaleDataset(Dataset):
combined_df = pd.DataFrame()
for parquet_file in parquet_files:
# Load a subset (head(2500)) from each parquet file
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(1250)
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(2500)
combined_df = pd.concat([combined_df, df], ignore_index=True)
# Validate rows (ensuring each value is bytes or str)
@ -128,7 +128,7 @@ dataset = UpscaleDataset([
"/root/training_data/vision-dataset/image_upscaler.parquet",
"/root/training_data/vision-dataset/image_vec_upscaler.parquet"
], transform=transform)
data_loader = DataLoader(dataset, batch_size=2, shuffle=True) # Consider adjusting num_workers if needed.
data_loader = DataLoader(dataset, batch_size=1, shuffle=True) # Consider adjusting num_workers if needed.
# Define loss function and optimizer.
criterion = nn.MSELoss()