Training loss significantly higher than validation loss, consistently

[dempsey-ryan]

I'm trying to figure out if I'm using validation_step and training_step properly. It seems I am, but maybe it's not accumulating properly? I've checked for data leakage, there is none. From my LightningModule, here are the training and validation loops, as well as a small portion of init to give context for certain variables:

import torch.nn as nn
import torch
from torchmetrics.functional import mean_absolute_error, mean_squared_error

   def __init__(self, loss_fn, ...):
   ...
         if loss_fn.lower() == "mse":
          self.loss = nn.MSELoss(reduction="mean")
          # self.metric = nn.L1Loss()
          self.metric = mean_absolute_error
          self._loss = "mse"
          self._metric = "mae"
      elif loss_fn.lower() == "mae":
          self.loss = nn.L1Loss()
          # self.metric = nn.MSELoss()
          self.metric = mean_squared_error
          self._loss = "mae"
          self._metric = "mse"
    ...
    def training_step(self, batch, batch_idx):
        loss, _ = self.calculate_metrics_and_loss(batch, batch_idx, metric=False)
        # self.log("train_loss", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True, sync_dist=True)
        # self.log("train_loss", loss, on_step=False, on_epoch=True, prog_bar=True, logger=True, sync_dist=True)
        # self.log("train_loss", loss, prog_bar=True)
        # next try this
        self.log("train_loss", loss, on_epoch=True, prog_bar=True)
        return loss

    def validation_step(self, batch, batch_idx):
        val_l, val_metric = self.calculate_metrics_and_loss(batch, batch_idx, metric=True)
        # self.val_losses.append(val_l)
        self.log_dict({"val_loss": val_l, f"val_{self._metric}": val_metric}, prog_bar=True)
        # self.log_dict({"val_loss": val_l.detach(), f"val_{self._metric}": val_metric}, on_step=False, on_epoch=True, prog_bar=True)
        # self.log("val_loss", val_l.detach(), on_step=False, on_epoch=True, prog_bar=True)
        return val_l

and here is the data split:

import pytorch_lightning as pl
from sklearn.model_selection import train_test_split
from torch.utils.data import DataLoader, SubsetRandomSampler, Subset

train_split = 0.8
train_idx, val_idx = train_test_split(np.arange(len(train_dataset)), train_size=train_split)
pinmem = True
train_dl = DataLoader(
    train_dataset, batch_size=batch_size, pin_memory=pinmem, 
    sampler=SubsetRandomSampler(indices=train_idx),
    collate_fn=dataset.collate_zero_pad, num_workers=num_workers
)
val_dl = DataLoader(
    Subset(train_dataset, indices=val_idx),
    batch_size=batch_size, pin_memory=pinmem,
    shuffle=False, collate_fn=dataset.collate_zero_pad, num_workers=num_workers
)

csv_logger = pl.loggers.CSVLogger(...)

mod_ckpt = pl.callbacks.ModelCheckpoint(monitor='val_loss', save_top_k=3, save_last=1)

trainer = pl.Trainer(
    accelerator="cuda", log_every_n_steps=1, max_epochs=epochs,
    callbacks=[mod_ckpt], logger=csv_logger
)
# finally
trainer.fit(model=model, train_dataloaders=train_dl, val_dataloaders=val_dl)

Is it my data split? Perhaps SubsetRandomSampler and/or Subset don't work as I'm expecting, or maybe the loops aren't written correctly