Machine-Learning
/
aiuNN
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

cpu training

This commit is contained in:
Falko Victor Habel 2025-02-21 21:54:33 +01:00
parent 992bc0eb7f
commit 4af12873f2
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/aiunn/finetune.py
View File

@ -13,7 +13,7 @@ class UpscaleDataset(Dataset):
combined_df = pd.DataFrame() combined_df = pd.DataFrame()
for parquet_file in parquet_files: for parquet_file in parquet_files:
# Load data with chunking for memory efficiency # Load data with chunking for memory efficiency
df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(4000) df = pd.read_parquet(parquet_file, columns=['image_512', 'image_1024']).head(10000)
combined_df = pd.concat([combined_df, df], ignore_index=True) combined_df = pd.concat([combined_df, df], ignore_index=True)
# Validate data format # Validate data format
@ -87,14 +87,14 @@ pretrained_model_path = "/root/vision/AIIA/AIIA-base-512"
# Load the model using the AIIA.load class method (the implementation copied in your query) # Load the model using the AIIA.load class method (the implementation copied in your query)
model = AIIABase.load(pretrained_model_path) model = AIIABase.load(pretrained_model_path)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") device = 'cpu' #torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = model.to(device) model = model.to(device)
from torch import nn, optim from torch import nn, optim
from torch.utils.data import DataLoader from torch.utils.data import DataLoader
# Create your dataset and dataloader # Create your dataset and dataloader
dataset = UpscaleDataset(["/root/training_data/vision-dataset/image_upscaler.parquet", "/root/training_data/vision-dataset/image_vec_upscaler.parquet"], transform=transform) dataset = UpscaleDataset(["/root/training_data/vision-dataset/image_upscaler.parquet", "/root/training_data/vision-dataset/image_vec_upscaler.parquet"], transform=transform)
data_loader = DataLoader(dataset, batch_size=1, shuffle=True) data_loader = DataLoader(dataset, batch_size=4, shuffle=True)
# Define a loss function and optimizer # Define a loss function and optimizer
criterion = nn.MSELoss() criterion = nn.MSELoss()