diff --git a/sagemaker-mlops/tests/integ/data/v3_trainer/preference_train.jsonl b/sagemaker-mlops/tests/integ/data/v3_trainer/preference_train.jsonl
index 3802958661..c999b15d75 100644
--- a/sagemaker-mlops/tests/integ/data/v3_trainer/preference_train.jsonl
+++ b/sagemaker-mlops/tests/integ/data/v3_trainer/preference_train.jsonl
@@ -1,20 +1,20 @@
-{"chosen": [{"role": "user", "content": "What is ML?"}, {"role": "assistant", "content": "ML is a subset of AI that enables systems to learn from data."}], "rejected": [{"role": "user", "content": "What is ML?"}, {"role": "assistant", "content": "I don't know."}]}
-{"chosen": [{"role": "user", "content": "What is SageMaker?"}, {"role": "assistant", "content": "SageMaker is a managed machine learning service by AWS."}], "rejected": [{"role": "user", "content": "What is SageMaker?"}, {"role": "assistant", "content": "It's a thing."}]}
-{"chosen": [{"role": "user", "content": "What is fine-tuning?"}, {"role": "assistant", "content": "Fine-tuning adapts a pre-trained model to a specific task."}], "rejected": [{"role": "user", "content": "What is fine-tuning?"}, {"role": "assistant", "content": "Making things fine."}]}
-{"chosen": [{"role": "user", "content": "What is LoRA?"}, {"role": "assistant", "content": "LoRA is Low-Rank Adaptation for efficient model fine-tuning."}], "rejected": [{"role": "user", "content": "What is LoRA?"}, {"role": "assistant", "content": "A name."}]}
-{"chosen": [{"role": "user", "content": "What is RLHF?"}, {"role": "assistant", "content": "RLHF is Reinforcement Learning from Human Feedback."}], "rejected": [{"role": "user", "content": "What is RLHF?"}, {"role": "assistant", "content": "An acronym."}]}
-{"chosen": [{"role": "user", "content": "What is DPO?"}, {"role": "assistant", "content": "DPO is Direct Preference Optimization for alignment."}], "rejected": [{"role": "user", "content": "What is DPO?"}, {"role": "assistant", "content": "Something."}]}
-{"chosen": [{"role": "user", "content": "What is a transformer?"}, {"role": "assistant", "content": "A neural network architecture using self-attention mechanisms."}], "rejected": [{"role": "user", "content": "What is a transformer?"}, {"role": "assistant", "content": "A robot."}]}
-{"chosen": [{"role": "user", "content": "What is tokenization?"}, {"role": "assistant", "content": "Splitting text into smaller units called tokens."}], "rejected": [{"role": "user", "content": "What is tokenization?"}, {"role": "assistant", "content": "Tokens."}]}
-{"chosen": [{"role": "user", "content": "What is inference?"}, {"role": "assistant", "content": "Using a trained model to generate predictions on new data."}], "rejected": [{"role": "user", "content": "What is inference?"}, {"role": "assistant", "content": "Guessing."}]}
-{"chosen": [{"role": "user", "content": "What is a GPU?"}, {"role": "assistant", "content": "A processor for parallel computation used in ML training."}], "rejected": [{"role": "user", "content": "What is a GPU?"}, {"role": "assistant", "content": "A card."}]}
-{"chosen": [{"role": "user", "content": "What is backpropagation?"}, {"role": "assistant", "content": "An algorithm for computing gradients in neural networks."}], "rejected": [{"role": "user", "content": "What is backpropagation?"}, {"role": "assistant", "content": "Going back."}]}
-{"chosen": [{"role": "user", "content": "What is an epoch?"}, {"role": "assistant", "content": "One complete pass through the training dataset."}], "rejected": [{"role": "user", "content": "What is an epoch?"}, {"role": "assistant", "content": "A time period."}]}
-{"chosen": [{"role": "user", "content": "What is overfitting?"}, {"role": "assistant", "content": "When a model memorizes training data instead of learning."}], "rejected": [{"role": "user", "content": "What is overfitting?"}, {"role": "assistant", "content": "Too much fitting."}]}
-{"chosen": [{"role": "user", "content": "What is a loss function?"}, {"role": "assistant", "content": "Measures how far predictions are from actual values."}], "rejected": [{"role": "user", "content": "What is a loss function?"}, {"role": "assistant", "content": "Loss."}]}
-{"chosen": [{"role": "user", "content": "What is batch size?"}, {"role": "assistant", "content": "Number of samples processed before updating weights."}], "rejected": [{"role": "user", "content": "What is batch size?"}, {"role": "assistant", "content": "A size."}]}
-{"chosen": [{"role": "user", "content": "What is learning rate?"}, {"role": "assistant", "content": "Controls how much weights change during training."}], "rejected": [{"role": "user", "content": "What is learning rate?"}, {"role": "assistant", "content": "A rate."}]}
-{"chosen": [{"role": "user", "content": "What is regularization?"}, {"role": "assistant", "content": "Techniques to prevent overfitting in models."}], "rejected": [{"role": "user", "content": "What is regularization?"}, {"role": "assistant", "content": "Making regular."}]}
-{"chosen": [{"role": "user", "content": "What is an embedding?"}, {"role": "assistant", "content": "A dense vector representation of discrete data."}], "rejected": [{"role": "user", "content": "What is an embedding?"}, {"role": "assistant", "content": "A vector."}]}
-{"chosen": [{"role": "user", "content": "What is attention?"}, {"role": "assistant", "content": "A mechanism that weighs importance of different inputs."}], "rejected": [{"role": "user", "content": "What is attention?"}, {"role": "assistant", "content": "Focus."}]}
-{"chosen": [{"role": "user", "content": "What is a pipeline?"}, {"role": "assistant", "content": "An automated workflow for ML model development."}], "rejected": [{"role": "user", "content": "What is a pipeline?"}, {"role": "assistant", "content": "A pipe."}]}
+{"prompt": [{"role": "user", "content": "What is ML?"}], "chosen": [{"role": "assistant", "content": "ML is a subset of AI that enables systems to learn from data."}], "rejected": [{"role": "assistant", "content": "I don't know."}]}
+{"prompt": [{"role": "user", "content": "What is SageMaker?"}], "chosen": [{"role": "assistant", "content": "SageMaker is a managed machine learning service by AWS."}], "rejected": [{"role": "assistant", "content": "It's a thing."}]}
+{"prompt": [{"role": "user", "content": "What is fine-tuning?"}], "chosen": [{"role": "assistant", "content": "Fine-tuning adapts a pre-trained model to a specific task."}], "rejected": [{"role": "assistant", "content": "Making things fine."}]}
+{"prompt": [{"role": "user", "content": "What is LoRA?"}], "chosen": [{"role": "assistant", "content": "LoRA is Low-Rank Adaptation for efficient model fine-tuning."}], "rejected": [{"role": "assistant", "content": "A name."}]}
+{"prompt": [{"role": "user", "content": "What is RLHF?"}], "chosen": [{"role": "assistant", "content": "RLHF is Reinforcement Learning from Human Feedback."}], "rejected": [{"role": "assistant", "content": "An acronym."}]}
+{"prompt": [{"role": "user", "content": "What is DPO?"}], "chosen": [{"role": "assistant", "content": "DPO is Direct Preference Optimization for alignment."}], "rejected": [{"role": "assistant", "content": "Something."}]}
+{"prompt": [{"role": "user", "content": "What is a transformer?"}], "chosen": [{"role": "assistant", "content": "A neural network architecture using self-attention mechanisms."}], "rejected": [{"role": "assistant", "content": "A robot."}]}
+{"prompt": [{"role": "user", "content": "What is tokenization?"}], "chosen": [{"role": "assistant", "content": "Splitting text into smaller units called tokens."}], "rejected": [{"role": "assistant", "content": "Tokens."}]}
+{"prompt": [{"role": "user", "content": "What is inference?"}], "chosen": [{"role": "assistant", "content": "Using a trained model to generate predictions on new data."}], "rejected": [{"role": "assistant", "content": "Guessing."}]}
+{"prompt": [{"role": "user", "content": "What is a GPU?"}], "chosen": [{"role": "assistant", "content": "A processor for parallel computation used in ML training."}], "rejected": [{"role": "assistant", "content": "A card."}]}
+{"prompt": [{"role": "user", "content": "What is backpropagation?"}], "chosen": [{"role": "assistant", "content": "An algorithm for computing gradients in neural networks."}], "rejected": [{"role": "assistant", "content": "Going back."}]}
+{"prompt": [{"role": "user", "content": "What is an epoch?"}], "chosen": [{"role": "assistant", "content": "One complete pass through the training dataset."}], "rejected": [{"role": "assistant", "content": "A time period."}]}
+{"prompt": [{"role": "user", "content": "What is overfitting?"}], "chosen": [{"role": "assistant", "content": "When a model memorizes training data instead of learning."}], "rejected": [{"role": "assistant", "content": "Too much fitting."}]}
+{"prompt": [{"role": "user", "content": "What is a loss function?"}], "chosen": [{"role": "assistant", "content": "Measures how far predictions are from actual values."}], "rejected": [{"role": "assistant", "content": "Loss."}]}
+{"prompt": [{"role": "user", "content": "What is batch size?"}], "chosen": [{"role": "assistant", "content": "Number of samples processed before updating weights."}], "rejected": [{"role": "assistant", "content": "A size."}]}
+{"prompt": [{"role": "user", "content": "What is learning rate?"}], "chosen": [{"role": "assistant", "content": "Controls how much weights change during training."}], "rejected": [{"role": "assistant", "content": "A rate."}]}
+{"prompt": [{"role": "user", "content": "What is regularization?"}], "chosen": [{"role": "assistant", "content": "Techniques to prevent overfitting in models."}], "rejected": [{"role": "assistant", "content": "Making regular."}]}
+{"prompt": [{"role": "user", "content": "What is an embedding?"}], "chosen": [{"role": "assistant", "content": "A dense vector representation of discrete data."}], "rejected": [{"role": "assistant", "content": "A vector."}]}
+{"prompt": [{"role": "user", "content": "What is attention?"}], "chosen": [{"role": "assistant", "content": "A mechanism that weighs importance of different inputs."}], "rejected": [{"role": "assistant", "content": "Focus."}]}
+{"prompt": [{"role": "user", "content": "What is a pipeline?"}], "chosen": [{"role": "assistant", "content": "An automated workflow for ML model development."}], "rejected": [{"role": "assistant", "content": "A pipe."}]}