From 6222f1fee79193581a588f7eb8966a8d65ab7bb0 Mon Sep 17 00:00:00 2001
From: Mingyu Gao <gaomy@tsinghua.edu.cn>
Date: Mon, 23 Dec 2019 10:24:10 +0800
Subject: [PATCH] [scheduler] Fix a race during transiting fake leave to leave.

---
 src/scheduler.cpp |  7 +++++++
 src/scheduler.h   | 10 ++++++++++
 2 files changed, 17 insertions(+)

diff --git a/src/scheduler.cpp b/src/scheduler.cpp
index 23ea76dd..4191d495 100644
--- a/src/scheduler.cpp
+++ b/src/scheduler.cpp
@@ -153,12 +153,19 @@ void Scheduler::watchdogThreadFunc() {
 
                     uint64_t pc = fl->pc;
                     do {
+                        // NOTE(gaomy): to avoid race with join() if a thread is not actually blocked but just waiting too long (e.g., heavily loaded host).
+                        // Because we finish fake leave and release lock before doing actual leave, the join could happen in between,
+                        // when we haven't done leave.
+                        th->flWord = 1;
                         finishFakeLeave(th);
 
                         futex_unlock(&schedLock);
                         leave(pid, tid, cid);
                         futex_lock(&schedLock);
 
+                        th->flWord = 0;
+                        syscall(SYS_futex, &th->flWord, FUTEX_WAKE, 1, nullptr, nullptr, 0);
+
                         // also do real leave for other threads blocked at the same pc ...
                         fl = fakeLeaves.front();
                         if (fl == nullptr || getPid(th->gid) != pid || fl->pc != pc)
diff --git a/src/scheduler.h b/src/scheduler.h
index ac50a1fc..d833b66d 100644
--- a/src/scheduler.h
+++ b/src/scheduler.h
@@ -107,6 +107,7 @@ class Scheduler : public GlobAlloc, public Callee {
             g_vector<bool> mask;
 
             FakeLeaveInfo* fakeLeave; // for accurate join-leaves, see below
+            volatile uint32_t flWord; // if non-zero, currently transiting fake leave to true leave
 
             FutexJoinInfo futexJoin;
 
@@ -124,6 +125,7 @@ class Scheduler : public GlobAlloc, public Callee {
                 for (auto b : mask) if (b) count++;
                 if (count == 0) panic("Empty mask on gid %d!", gid);
                 fakeLeave = nullptr;
+                flWord = 0;
                 futexJoin.action = FJA_NONE;
             }
         };
@@ -294,6 +296,14 @@ class Scheduler : public GlobAlloc, public Callee {
                 uint32_t cid = th->cid;
                 futex_unlock(&schedLock);
                 return cid;
+            } else if (th->flWord) {
+                // We are just finishing fake leave and transiting into true leave. Wait until done.
+                futex_unlock(&schedLock);
+                while (true) {
+                    int futex_res = syscall(SYS_futex, &th->flWord, FUTEX_WAIT, 1, nullptr, nullptr, 0);
+                    if (futex_res == 0 || th->futexWord != 1) break;
+                }
+                futex_lock(&schedLock);
             }
 
             assert(!th->markedForSleep);