Skip to content

Commit

Permalink
feat(all): add exit manager
Browse files Browse the repository at this point in the history
  • Loading branch information
halibobo1205 committed Dec 4, 2024
1 parent f16c3f8 commit 94f1cd4
Show file tree
Hide file tree
Showing 22 changed files with 276 additions and 141 deletions.
4 changes: 4 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,10 @@ subprojects {
reproducibleFileOrder = true
duplicatesStrategy = DuplicatesStrategy.INCLUDE // allow duplicates
}
tasks.withType(Test).configureEach {
// https://docs.gradle.org/current/dsl/org.gradle.api.tasks.testing.Test.html#org.gradle.api.tasks.testing.Test:environment
environment 'CI', 'true'
}
}

task copyToParent(type: Copy) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
import static org.fusesource.leveldbjni.JniDBFactory.factory;

import com.google.common.collect.Sets;
import java.io.File;
import com.google.common.primitives.Bytes;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
Expand All @@ -30,18 +30,14 @@
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;

import com.google.common.primitives.Bytes;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.iq80.leveldb.CompressionType;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBIterator;
import org.iq80.leveldb.Logger;
Expand All @@ -50,6 +46,7 @@
import org.iq80.leveldb.WriteBatch;
import org.iq80.leveldb.WriteOptions;
import org.slf4j.LoggerFactory;
import org.tron.common.exit.ExitManager;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.storage.WriteOptionsWrapper;
import org.tron.common.storage.metric.DbStat;
Expand All @@ -59,6 +56,7 @@
import org.tron.core.db.common.iterator.StoreIterator;
import org.tron.core.db2.common.Instance;
import org.tron.core.db2.common.WrappedByteArray;
import org.tron.core.exception.DatabaseExitException;

@Slf4j(topic = "DB")
@NoArgsConstructor
Expand Down Expand Up @@ -151,13 +149,14 @@ private void openDatabase(Options dbOptions) throws IOException {
dbOptions.cacheSize() / 1024 / 1024, dbOptions.maxOpenFiles());
}
} catch (IOException e) {
String msg;
if (e.getMessage().contains("Corruption:")) {
logger.error("Database {} corrupted, please delete database directory({}) and restart.",
dataBaseName, parentPath, e);
msg = String.format("Database %s corrupted, please delete database directory(%s) "
+ "and restart.", dataBaseName, parentPath);
} else {
logger.error("Open Database {} failed", dataBaseName, e);
msg = String.format("Open Database %s failed", dataBaseName);
}
System.exit(1);
ExitManager.exit(msg, new DatabaseExitException(e));
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.rocksdb.WriteBatch;
import org.rocksdb.WriteOptions;
import org.slf4j.LoggerFactory;
import org.tron.common.exit.ExitManager;
import org.tron.common.setting.RocksDbSettings;
import org.tron.common.storage.WriteOptionsWrapper;
import org.tron.common.storage.metric.DbStat;
Expand All @@ -45,6 +46,7 @@
import org.tron.core.db.common.iterator.RockStoreIterator;
import org.tron.core.db2.common.Instance;
import org.tron.core.db2.common.WrappedByteArray;
import org.tron.core.exception.DatabaseExitException;


@Slf4j(topic = "DB")
Expand Down Expand Up @@ -266,13 +268,14 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) {
try {
database = RocksDB.open(options, dbPath.toString());
} catch (RocksDBException e) {
String msg;
if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) {
logger.error("Database {} corrupted, please delete database directory({}) " +
"and restart.", dataBaseName, parentPath, e);
msg = String.format("Database %s corrupted, please delete database directory(%s) "
+ "and restart.", dataBaseName, parentPath);
} else {
logger.error("Open Database {} failed", dataBaseName, e);
msg = String.format("Open Database %s failed", dataBaseName);
}
System.exit(1);
ExitManager.exit(msg, new DatabaseExitException(e));
}

alive = true;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.LockSupport;
import java.util.stream.Collectors;
import javax.annotation.PostConstruct;
import lombok.Getter;
Expand All @@ -29,6 +28,7 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.tron.common.error.TronDBException;
import org.tron.common.es.ExecutorServiceManager;
import org.tron.common.exit.ExitManager;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.storage.WriteOptionsWrapper;
import org.tron.common.utils.FileUtil;
Expand All @@ -41,6 +41,8 @@
import org.tron.core.db2.common.Key;
import org.tron.core.db2.common.Value;
import org.tron.core.db2.common.WrappedByteArray;
import org.tron.core.exception.ConfigExitException;
import org.tron.core.exception.DatabaseExitException;
import org.tron.core.exception.RevokingStoreIllegalStateException;
import org.tron.core.store.CheckPointV2Store;
import org.tron.core.store.CheckTmpStore;
Expand Down Expand Up @@ -68,7 +70,6 @@ public class SnapshotManager implements RevokingDatabase {

private volatile int flushCount = 0;

private Thread exitThread;
private volatile boolean hitDown;

private Map<String, ListeningExecutorService> flushServices = new HashMap<>();
Expand Down Expand Up @@ -105,15 +106,6 @@ public void init() {
}
}, 10000, 3600, TimeUnit.MILLISECONDS);
}
exitThread = new Thread(() -> {
LockSupport.park();
// to Guarantee Some other thread invokes unpark with the current thread as the target
if (hitDown) {
System.exit(1);
}
});
exitThread.setName("exit-thread");
exitThread.start();
}

public static String simpleDecode(byte[] bytes) {
Expand Down Expand Up @@ -281,13 +273,6 @@ public void shutdown() {
ExecutorServiceManager.shutdownAndAwaitTermination(pruneCheckpointThread, pruneName);
flushServices.forEach((key, value) -> ExecutorServiceManager.shutdownAndAwaitTermination(value,
"flush-service-" + key));
try {
exitThread.interrupt();
// help GC
exitThread = null;
} catch (Exception e) {
logger.warn("exitThread interrupt error", e);
}
}

public void updateSolidity(int hops) {
Expand Down Expand Up @@ -365,9 +350,9 @@ public void flush() {
System.currentTimeMillis() - checkPointEnd
);
} catch (TronDBException e) {
logger.error(" Find fatal error, program will be exited soon.", e);
String msg = " Find fatal error, program will be exited soon.";
hitDown = true;
LockSupport.unpark(exitThread);
ExitManager.exit(msg, new DatabaseExitException(e));
}
}
}
Expand Down Expand Up @@ -490,10 +475,10 @@ public void check() {
if (!isV2Open()) {
List<String> cpList = getCheckpointList();
if (cpList != null && cpList.size() != 0) {
logger.error("checkpoint check failed, the checkpoint version of database not match your " +
"config file, please set storage.checkpoint.version = 2 in your config file " +
"and restart the node.");
System.exit(-1);
String msg = "checkpoint check failed, the checkpoint version of database not match your "
+ "config file, please set storage.checkpoint.version = 2 in your config file "
+ "and restart the node.";
ExitManager.exit(new ConfigExitException(msg));
}
checkV1();
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@
import org.springframework.stereotype.Component;
import org.tron.common.error.TronDBException;
import org.tron.common.es.ExecutorServiceManager;
import org.tron.common.exit.ExitManager;
import org.tron.common.parameter.CommonParameter;
import org.tron.common.utils.ByteArray;
import org.tron.common.utils.MerkleRoot;
import org.tron.common.utils.Pair;
import org.tron.common.utils.Sha256Hash;
import org.tron.core.db.common.iterator.DBIterator;
import org.tron.core.db2.common.DB;
import org.tron.core.exception.DatabaseExitException;
import org.tron.core.store.DelegationStore;
import org.tron.core.store.DynamicPropertiesStore;
import org.tron.core.store.RewardViStore;
Expand Down Expand Up @@ -120,8 +122,8 @@ private void maybeRun() {
}
}
} catch (Exception e) {
logger.error(" Find fatal error, program will be exited soon.", e);
System.exit(1);
String msg = " Find fatal error, program will be exited soon.";
ExitManager.exit(msg, new DatabaseExitException(e));
}
}

Expand Down
79 changes: 79 additions & 0 deletions common/src/main/java/org/tron/common/exit/ExitManager.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
package org.tron.common.exit;

import java.util.Arrays;
import java.util.Objects;
import java.util.concurrent.ThreadFactory;
import lombok.extern.slf4j.Slf4j;
import org.tron.core.exception.TronExitException;

@Slf4j(topic = "Exit")
public class ExitManager {

private static final String[] CI_ENVIRONMENT_VARIABLES = {
"CI",
"JENKINS_URL",
"TRAVIS",
"CIRCLECI",
"GITHUB_ACTIONS",
"GITLAB_CI"
};

private static final int EXIT_CODE_NORMAL = 0;

private static final ThreadFactory exitThreadFactory = r -> {
Thread thread = new Thread(r, "System-Exit-Thread");
thread.setDaemon(true);
return thread;
};

private ExitManager() {
}

public static void exit() {
exit((String) null);
}

public static void exit(String msg) {
exit(msg, null);
}

public static void exit(TronExitException cause) {
exit(cause.getMessage(), cause);
}

public static void exit(String msg, TronExitException cause) {
TronExit exit = new TronExit(msg, cause);
if (isRunningInCI()) {
if (Objects.nonNull(cause)) {
throw cause;
} else if (Objects.nonNull(msg)) {
logger.info("{}", msg);
}
} else {
logAndExit(exit);
}
}

private static boolean isRunningInCI() {
return Arrays.stream(CI_ENVIRONMENT_VARIABLES).anyMatch(System.getenv()::containsKey);
}

private static void logAndExit(TronExit exit) {
String msg = exit.getMsg();
TronExitException cause = exit.getException();
final int code = Objects.isNull(cause) ? EXIT_CODE_NORMAL : cause.getExitCode();
if (code == EXIT_CODE_NORMAL) {
if (Objects.nonNull(msg)) {
logger.info("Exiting, {}.", msg);
}
} else {
if (Objects.isNull(msg)) {
logger.error("Exiting with code: {}.", code, cause);
} else {
logger.error("Exiting with code: {}, {}.", code, msg, cause);
}
}
Thread exitThread = exitThreadFactory.newThread(() -> System.exit(code));
exitThread.start();
}
}
13 changes: 13 additions & 0 deletions common/src/main/java/org/tron/common/exit/TronExit.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.tron.common.exit;

import lombok.AllArgsConstructor;
import lombok.Getter;
import org.tron.core.exception.TronExitException;

@Getter
@AllArgsConstructor
public class TronExit {

private String msg;
private TronExitException exception;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.tron.core.exception;

public class ConfigExitException extends TronExitException {

public ConfigExitException(String message) {
super(message);
setExitCode(1);
}

public ConfigExitException(String message, Throwable cause) {
super(message, cause);
setExitCode(1);
}

public ConfigExitException(Throwable cause) {
super(cause);
setExitCode(1);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.tron.core.exception;

public class DatabaseExitException extends TronExitException {

public DatabaseExitException(String message) {
super(message);
setExitCode(2);
}

public DatabaseExitException(String message, Throwable cause) {
super(message, cause);
setExitCode(2);
}

public DatabaseExitException(Throwable cause) {
super(cause);
setExitCode(2);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package org.tron.core.exception;

public class EventExitException extends TronExitException {

public EventExitException(String message) {
super(message);
setExitCode(3);
}

public EventExitException(String message, Throwable cause) {
super(message, cause);
setExitCode(3);
}

public EventExitException(Throwable cause) {
super(cause);
setExitCode(3);
}
}
Loading

0 comments on commit 94f1cd4

Please sign in to comment.