diff --git a/Common/ImageSamplers/itkImageRandomSampler.h b/Common/ImageSamplers/itkImageRandomSampler.h
index 9693344f1..090c7220c 100644
--- a/Common/ImageSamplers/itkImageRandomSampler.h
+++ b/Common/ImageSamplers/itkImageRandomSampler.h
@@ -19,6 +19,7 @@
 #define itkImageRandomSampler_h
 
 #include "itkImageRandomSamplerBase.h"
+#include <optional>
 
 namespace itk
 {
@@ -91,14 +92,25 @@ class ITK_TEMPLATE_EXPORT ImageRandomSampler : public ImageRandomSamplerBase<TIn
   {
     ITK_DISALLOW_COPY_AND_ASSIGN(UserData);
 
-    const std::vector<double> *    m_RandomNumberList{};
-    std::vector<ImageSampleType> * m_Samples{};
-    const InputImageType *         m_InputImage{};
+    UserData(const std::vector<double> &    randomNumberList,
+             std::vector<ImageSampleType> & samples,
+             const InputImageType &         inputImage,
+             const InputImageRegionType &   region)
+      : m_RandomNumberList(randomNumberList)
+      , m_Samples(samples)
+      , m_InputImage(inputImage)
+      , m_RegionIndex(region.GetIndex())
+      , m_RegionSize(region.GetSize())
+    {}
+
+    const std::vector<double> &    m_RandomNumberList;
+    std::vector<ImageSampleType> & m_Samples;
+    const InputImageType &         m_InputImage;
     InputImageIndexType            m_RegionIndex{};
     InputImageSizeType             m_RegionSize{};
   };
 
-  UserData m_UserData{};
+  std::optional<UserData> m_OptionalUserData{};
 
   static ITK_THREAD_RETURN_FUNCTION_CALL_CONVENTION
   ThreaderCallback(void * arg);
diff --git a/Common/ImageSamplers/itkImageRandomSampler.hxx b/Common/ImageSamplers/itkImageRandomSampler.hxx
index f12841153..ad67d2826 100644
--- a/Common/ImageSamplers/itkImageRandomSampler.hxx
+++ b/Common/ImageSamplers/itkImageRandomSampler.hxx
@@ -49,20 +49,14 @@ ImageRandomSampler<TInputImage>::GenerateData()
     auto &       samples = elastix::Deref(sampleContainer).CastToSTLContainer();
     samples.resize(randomNumberList.size());
 
-    m_UserData.m_RandomNumberList = &randomNumberList;
-    m_UserData.m_Samples = &samples;
-    m_UserData.m_InputImage = inputImage;
-
-    const InputImageRegionType region = this->GetCroppedInputImageRegion();
-
-    m_UserData.m_RegionIndex = region.GetIndex();
-    m_UserData.m_RegionSize = region.GetSize();
+    m_OptionalUserData.emplace(
+      randomNumberList, samples, elastix::Deref(inputImage), this->GetCroppedInputImageRegion());
 
     auto & randomVariateGenerator = elastix::Deref(Statistics::MersenneTwisterRandomVariateGenerator::GetInstance());
     randomVariateGenerator.SetSeed(randomVariateGenerator.GetNextSeed());
 
     MultiThreaderBase & multiThreader = elastix::Deref(this->ProcessObject::GetMultiThreader());
-    multiThreader.SetSingleMethod(&Self::ThreaderCallback, &m_UserData);
+    multiThreader.SetSingleMethod(&Self::ThreaderCallback, &*m_OptionalUserData);
     multiThreader.SingleMethodExecute();
     return;
   }
@@ -163,11 +157,8 @@ ImageRandomSampler<TInputImage>::ThreaderCallback(void * const arg)
   assert(info.UserData);
   auto & userData = *static_cast<UserData *>(info.UserData);
 
-  assert(userData.m_RandomNumberList);
-  const auto & randomNumberList = *(userData.m_RandomNumberList);
-
-  assert(userData.m_Samples);
-  auto & samples = *(userData.m_Samples);
+  const auto & randomNumberList = userData.m_RandomNumberList;
+  auto &       samples = userData.m_Samples;
 
   const auto totalNumberOfSamples = samples.size();
   assert(totalNumberOfSamples == randomNumberList.size());
@@ -179,7 +170,7 @@ ImageRandomSampler<TInputImage>::ThreaderCallback(void * const arg)
     info.WorkUnitID * numberOfSamplesPerWorkUnit + std::min<size_t>(info.WorkUnitID, remainderNumberOfSamples);
   const auto   beginOfRandomNumbers = randomNumberList.data() + offset;
   const auto   beginOfSamples = samples.data() + offset;
-  const auto & inputImage = *(userData.m_InputImage);
+  const auto & inputImage = userData.m_InputImage;
 
   const InputImageSizeType  regionSize = userData.m_RegionSize;
   const InputImageIndexType regionIndex = userData.m_RegionIndex;