Using BoltDB (#430)

* use BoltDB for storages which uses the ObjectStorage * Ignore local hive.go * - Adapting interfaces to be able to switch between badger and bolt * Skip garbage collection with bolt * - Use DeletePrefix instead of iterating and then deleting * - Properly handling ErrKeyNotFound in database pkg * - Remove the custom prefixes from the ledger db when streaming the keys * - Remove badger and all its options - Replaced database package with new store package that utilized the hive.go kvstore * - Load maximum of 100 bundles in the dashboard explorer to avoid killing the node with bundle replay attacks * - Removed store pkg - Creating different bolt db files to separate tangle, snapshot and spent addresses * - Properly close the autopeering db when shutting down the plugin * - Fixed the db size metrics in the dashboard * - Tuning bolt db options to have better performance for our usecase * - Renamed lots of functions that had badger in the name * Add YRTSI and ORTSI to transaction metadata * - Updated to latest hive.go * - Added back GOMAXPROCS support * - Removed NoFreelistSync from the bolt options, since this means opening the db will take a lot of time, since the whole db needs to be scanned to build the freelist * - Properly handle the abortion of the ledger iteration * - Review changes * - Prepare for rc9 * - Fix for Max * - Run packr with latest changes * - Woof woof * Fix spammer settings * Fixed changelog Co-authored-by: Luca Moser <[email protected]> Co-authored-by: muXxer <[email protected]>
iotaledger · May 18, 2020 · 69da96d · 69da96d
1 parent 219781c
commit 69da96d
Show file tree

Hide file tree

Showing 45 changed files with 838 additions and 1,104 deletions.
diff --git a/.gitignore b/.gitignore
@@ -44,3 +44,4 @@ hornet.log
 layer*.csv
 coordinator.state
 coordinator.tree
+hive.go/
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -2,6 +2,134 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.4.0-rc9] - 19.05.2020
+
+**Breaking change:**<br>
+Database implementation changed (moved from Badger to Bolt)<br><br>
+_Update note:_ Please remove your database and restart HORNET.
+
+### Added
+
+    - Coordinator plugin
+    - Dashboard:
+      - Responsive design
+      - Visualizer (ported from GoShimmer)
+      - Spam transactions graph
+      - Show IOTA units
+      - Value-tx only filter
+    - API:
+      - `pruneDatabase` call
+      - `getLedgerState` call
+      - `getFundsOnSpentAddresses` call
+    - Flag to overwrite coo address at startup
+
+### Removed
+
+    - `getSnapshot` API call
+
+### Changed
+
+    - Moved from Badger to Bolt (reduced RAM usage)
+    - Milestone logic
+    - Pruning logic
+    - Database pressure reduced
+    - Renamed `ZeroMQ` plugin to `ZMQ`
+    - Dashboard graph colors
+    - Graph explorer link is now configurable
+    - Improved spammer plugin
+    - Local snapshot doesn't write to database if triggered externally
+    - API:
+      - Handle `minWeightMagnitude` as an optional parameter
+      - Renamed `createSnapshot` to `createSnapshotFile`
+      - Improved error handling in `createSnapshotFile`
+
+### Fixed
+
+    - Database revalidation
+    - Websocket messages
+    - ZMQ `address` topic
+
+### Config file changes
+
+Added option:
+
+`config.json`
+
+```diff
++"coordinator": {
++  "address": "EQSAUZXULTTYZCLNJNTXQTQHOMOFZERHTCGTXOLTVAHKSA9OGAZDEKECURBRIXIJWNPFCQIOVFVVXJVD9",
++  "securityLevel": 2,
++  "merkleTreeDepth": 23,
++  "mwm": 14,
++  "stateFilePath": "coordinator.state",
++  "merkleTreeFilePath": "coordinator.tree",
++  "intervalSeconds": 60,
++  "checkpointTransactions": 5
++},
+"spammer": {
++  "cpuMaxUsage": 0.5,
+},
+"graph": {
++  "explorerTxLink": "http://localhost:8081/explorer/tx/",
++  "explorerBundleLink": "http://localhost:8081/explorer/bundle/"
+},
+```
+
+`config_comnet.json`
+
+```diff
++"coordinator": {
++  "address": "ZNCCPOTBCDZXCBQYBWUYYFO9PLRHNAROWOS9KGMYWNVIXWGYGUSJBZUTUQBNQRADHPUEONZZTYGVMSRZD",
++  "securityLevel": 2,
++  "merkleTreeDepth": 23,
++  "mwm": 10,
++  "stateFilePath": "coordinator.state",
++  "merkleTreeFilePath": "coordinator.tree",
++  "intervalSeconds": 60,
++  "checkpointTransactions": 5
++},
+"spammer": {
++  "cpuMaxUsage": 0.5,
+},
+"graph": {
++  "explorerTxLink": "http://localhost:8081/explorer/tx/",
++  "explorerBundleLink": "http://localhost:8081/explorer/bundle/"
+},
+```
+
+Removed option:
+
+`config.json` + `config_comnet.json`
+
+```diff
+-"milestones": {
+-  "coordinator": "ZNCCPOTBCDZXCBQYBWUYYFO9PLRHNAROWOS9KGMYWNVIXWGYGUSJBZUTUQBNQRADHPUEONZZTYGVMSRZD",
+-  "coordinatorSecurityLevel": 2,
+-  "numberOfKeysInAMilestone": 23
+-}
+-"compass": {
+-  "loadLSMIAsLMI": false
+-},
+-"protocol": {
+-  "mwm": 14
+-},
+```
+
+`config.json` + `config_comnet.json`
+
+```diff
+"spammer": {
+-  "tpsRateLimit": 0.1,
++  "tpsRateLimit": 0.0,
+-  "workers": 1
++  "workers": 0
+}
+"monitor": {
+-  "initialTransactionsCount": 15000,
++  "initialTransactions": 15000,
+}
+```
+
 ## [0.4.0-rc8] - 06.04.2020
 
 ### Fixed

diff --git a/config.json b/config.json
@@ -42,8 +42,7 @@
     }
   },
   "db": {
-    "path": "mainnetdb",
-    "debugLog": false
+    "path": "mainnetdb"
   },
   "snapshots": {
     "loadType": "local",

diff --git a/config_comnet.json b/config_comnet.json
@@ -118,8 +118,8 @@
     "message": "Spamming with HORNET tipselect",
     "tag": "HORNET99INTEGRATED99SPAMMER",
     "cpuMaxUsage": 0.5,
-    "tpsRateLimit": 0.1,
-    "workers": 1
+    "tpsRateLimit": 0.0,
+    "workers": 0
   },
   "graph": {
     "webRootPath": "IOTAtangle/webroot",

diff --git a/go.mod b/go.mod
@@ -2,11 +2,8 @@ module github.com/gohornet/hornet
 
 go 1.14
 
-replace github.com/dgraph-io/badger/v2 v2.0.1 => github.com/muXxer/badger/v2 v2.0.3-hotfix
-
 require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
-	github.com/dgraph-io/badger/v2 v2.0.1
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eclipse/paho.mqtt.golang v1.2.1-0.20200506085104-5ee50844ed64
 	github.com/fhmq/hmq v0.0.0-20200508032626-3f60d23e85bb
@@ -21,7 +18,7 @@ require (
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/gorilla/websocket v1.4.2
 	github.com/hashicorp/go-version v1.2.0 // indirect
-	github.com/iotaledger/hive.go v0.0.0-20200508125657-76ee9eb66cf8
+	github.com/iotaledger/hive.go v0.0.0-20200518225146-500916854cb2
 	github.com/iotaledger/iota.go v1.0.0-beta.14.0.20200505162045-3dfaebe62d8b
 	github.com/labstack/echo/v4 v4.1.16
 	github.com/labstack/gommon v0.3.0
@@ -33,6 +30,7 @@ require (
 	github.com/spf13/viper v1.6.3
 	github.com/stretchr/testify v1.5.1
 	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
+	go.etcd.io/bbolt v1.3.4
 	go.uber.org/atomic v1.6.0
 	golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
 	golang.org/x/net v0.0.0-20200506145744-7e3656a0809f