From a6b5a3e8a33dadf6e48075d76674fd5cb89ab5ed Mon Sep 17 00:00:00 2001
From: Liwei Ji <jiliwei.phys@gmail.com>
Date: Thu, 11 Apr 2024 18:22:18 +0000
Subject: [PATCH] TestSubcycling: fill wstate using Yfs

---
 TestSubcyclingMC/schedule.ccl             | 20 ++++++++++++--------
 TestSubcyclingMC/src/testsubcyclingmc.cxx | 19 +++++++++++++++++++
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/TestSubcyclingMC/schedule.ccl b/TestSubcyclingMC/schedule.ccl
index 7c14ce91a..144b9d792 100644
--- a/TestSubcyclingMC/schedule.ccl
+++ b/TestSubcyclingMC/schedule.ccl
@@ -22,8 +22,9 @@ SCHEDULE TestSubcyclingMC_CalcY1 IN TestSubcyclingMC_RK4Group
   READS: ustate(interior)
   WRITES: pstate(interior)
   # calculate Y1
-  WRITES: wstate(interior)
-  SYNC: wstate
+  READS: Y1(everywhere)
+  WRITES: wstate(everywhere)
+  #SYNC: wstate
 } "calc Y1"
 
 SCHEDULE TestSubcyclingMC_CalcY2 IN TestSubcyclingMC_RK4Group AFTER TestSubcyclingMC_CalcY1
@@ -37,8 +38,9 @@ SCHEDULE TestSubcyclingMC_CalcY2 IN TestSubcyclingMC_RK4Group AFTER TestSubcycli
   WRITES: ustate(interior)
   # calculate Y2
   READS: pstate(interior)
-  WRITES: wstate(interior)
-  SYNC: wstate
+  READS: Y2(everywhere)
+  WRITES: wstate(everywhere)
+  #SYNC: wstate
 } "calc Y2"
 
 SCHEDULE TestSubcyclingMC_CalcY3 IN TestSubcyclingMC_RK4Group AFTER TestSubcyclingMC_CalcY2
@@ -52,8 +54,9 @@ SCHEDULE TestSubcyclingMC_CalcY3 IN TestSubcyclingMC_RK4Group AFTER TestSubcycli
   WRITES: ustate(interior)
   # calculate Y3
   READS: pstate(interior)
-  WRITES: wstate(interior)
-  SYNC: wstate
+  READS: Y3(everywhere)
+  WRITES: wstate(everywhere)
+  #SYNC: wstate
 } "calc Y3"
 
 SCHEDULE TestSubcyclingMC_CalcY4 IN TestSubcyclingMC_RK4Group AFTER TestSubcyclingMC_CalcY3
@@ -67,8 +70,9 @@ SCHEDULE TestSubcyclingMC_CalcY4 IN TestSubcyclingMC_RK4Group AFTER TestSubcycli
   WRITES: ustate(interior)
   # calculate Y4
   READS: pstate(interior)
-  WRITES: wstate(interior)
-  SYNC: wstate
+  READS: Y4(everywhere)
+  WRITES: wstate(everywhere)
+  #SYNC: wstate
 } "calc Y4"
 
 SCHEDULE TestSubcyclingMC_UpdateU IN TestSubcyclingMC_RK4Group AFTER TestSubcyclingMC_CalcY4
diff --git a/TestSubcyclingMC/src/testsubcyclingmc.cxx b/TestSubcyclingMC/src/testsubcyclingmc.cxx
index 8e7b0d4b9..794695a0c 100644
--- a/TestSubcyclingMC/src/testsubcyclingmc.cxx
+++ b/TestSubcyclingMC/src/testsubcyclingmc.cxx
@@ -234,6 +234,21 @@ void CalcYfFromKcs(const Loop::GridDescBaseDevice &grid,
   }
 }
 
+void CopyWsFromYs(const Loop::GridDescBaseDevice &grid,
+                  // output
+                  const Loop::GF3D2<CCTK_REAL> &u_w,
+                  const Loop::GF3D2<CCTK_REAL> &rho_w,
+                  // input
+                  const Loop::GF3D2<const CCTK_REAL> &u_Y,
+                  const Loop::GF3D2<const CCTK_REAL> &rho_Y) {
+  grid.loop_ghosts_device<0, 0, 0>(grid.nghostzones,
+                                   [=] CCTK_DEVICE(const Loop::PointDesc &p)
+                                       CCTK_ATTRIBUTE_ALWAYS_INLINE {
+                                         u_w(p.I) = u_Y(p.I);
+                                         rho_w(p.I) = rho_Y(p.I);
+                                       });
+}
+
 extern "C" void TestSubcyclingMC_CalcY1(CCTK_ARGUMENTS) {
   DECLARE_CCTK_ARGUMENTSX_TestSubcyclingMC_CalcY1;
   grid.loop_int_device<0, 0, 0>(grid.nghostzones,
@@ -244,6 +259,7 @@ extern "C" void TestSubcyclingMC_CalcY1(CCTK_ARGUMENTS) {
                                       u_w(p.I) = u(p.I);
                                       rho_w(p.I) = rho(p.I);
                                     });
+  CopyWsFromYs(grid, u_w, rho_w, u_Y1, rho_Y1);
 }
 
 extern "C" void TestSubcyclingMC_CalcY2(CCTK_ARGUMENTS) {
@@ -252,6 +268,7 @@ extern "C" void TestSubcyclingMC_CalcY2(CCTK_ARGUMENTS) {
                     CCTK_DELTA_TIME / CCTK_REAL(6.)); // k1
   CalcYs(grid, u_w, rho_w, u_p, rho_p, u_k1, rho_k1,
          CCTK_DELTA_TIME * CCTK_REAL(0.5)); // Y2
+  CopyWsFromYs(grid, u_w, rho_w, u_Y2, rho_Y2);
 }
 
 extern "C" void TestSubcyclingMC_CalcY3(CCTK_ARGUMENTS) {
@@ -260,6 +277,7 @@ extern "C" void TestSubcyclingMC_CalcY3(CCTK_ARGUMENTS) {
                     CCTK_DELTA_TIME / CCTK_REAL(3.)); // k2
   CalcYs(grid, u_w, rho_w, u_p, rho_p, u_k2, rho_k2,
          CCTK_DELTA_TIME * CCTK_REAL(0.5)); // Y3
+  CopyWsFromYs(grid, u_w, rho_w, u_Y3, rho_Y3);
 }
 
 extern "C" void TestSubcyclingMC_CalcY4(CCTK_ARGUMENTS) {
@@ -268,6 +286,7 @@ extern "C" void TestSubcyclingMC_CalcY4(CCTK_ARGUMENTS) {
                     CCTK_DELTA_TIME / CCTK_REAL(3.)); // k3
   CalcYs(grid, u_w, rho_w, u_p, rho_p, u_k3, rho_k3,
          CCTK_DELTA_TIME); // Y4
+  CopyWsFromYs(grid, u_w, rho_w, u_Y4, rho_Y4);
 }
 
 extern "C" void TestSubcyclingMC_UpdateU(CCTK_ARGUMENTS) {