Fix SupportedFeatureSets (#594)

* Fix SupportedFeatureSets

* Fix loop counter type

* Release 0.5.0-rc8

* Fix syncedNeighbors calculation

* Delete bundles during revalidation of which their tail tx doesn't exist anymore

* Flush storages during revalidation

CHANGELOG.md

@@ -2,6 +2,31 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.5.0-rc8] - 07.08.2020
+
+**Breaking change:**
+** DO NOT USE IT ON MAINNET, IT WILL CRASH IMMEDIATELY AND IT WILL DESTROY YOUR DATABASE !!! **
+
+### Added
+
+    - Latest solid milestone and neighbor counts of every neighbor to dashboard
+    - powsrv.io support
+
+### Changed
+
+    - Breaking protocol version change due to whiteflag
+    - Heartbeat message was extended by neighbors count and latest solid milestone
+    - Requests are now sent if an unsynced neighbor may also have the data based on latest solid milestone
+
+### Fixed
+
+    - SupportedFeatureSets logic in handshake
+    - Milestone missing / Milestone updated panics
+    - Database revalidation
+    - Autopeered neighbors can now be added as static neighbors
+    - Hash conflicts in the visualizer
+    - Wrong CTPS calculation on bundle tail reattachments
+
 ## [0.5.0-rc7] - 04.08.2020
 
 **Breaking change:**

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/stretchr/testify v1.6.1
 	github.com/tcnksm/go-latest v0.0.0-20170313132115-e3007ae9052e
 	github.com/willf/bitset v1.1.10
-	gitlab.com/powsrv.io/go/client v0.0.0-20200726093809-495292e91ff6
+	gitlab.com/powsrv.io/go/client v0.0.0-20200807151725-8bc5209c1820
 	go.etcd.io/bbolt v1.3.5
 	go.uber.org/atomic v1.6.0
 	golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de

go.sum

@@ -437,8 +437,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
 gitlab.com/powsrv.io/go/api v0.0.0-20180825222338-acd4e4dcc0bf h1:Q1kPNjEw7kD+UmemYT+D1SP0A0Nv0fZnhvt9ye8NyIw=
 gitlab.com/powsrv.io/go/api v0.0.0-20180825222338-acd4e4dcc0bf/go.mod h1:09ZRgU0bsyuSW8fFmSLTWIPTddU0SxGYk5Kxzl0ShKI=
-gitlab.com/powsrv.io/go/client v0.0.0-20200726093809-495292e91ff6 h1:3bo427+i8otkYqtLSlP+1STNlMTUeM9JVIja4Pycjfk=
-gitlab.com/powsrv.io/go/client v0.0.0-20200726093809-495292e91ff6/go.mod h1:f+BJhVoikWwMAKtKCYFYaWReSZM3T5WVcwdv8h+ePw0=
+gitlab.com/powsrv.io/go/client v0.0.0-20200807151725-8bc5209c1820 h1:P0KkIVcwC3sz64TdjXO/MqgkGbbYjF+UJRJZUsezPSM=
+gitlab.com/powsrv.io/go/client v0.0.0-20200807151725-8bc5209c1820/go.mod h1:f+BJhVoikWwMAKtKCYFYaWReSZM3T5WVcwdv8h+ePw0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=

pkg/model/tangle/bundle_storage.go

@@ -187,6 +187,13 @@ func GetCachedBundleOrNil(tailTxHash hornet.Hash) *CachedBundle {
 	return &CachedBundle{CachedObject: cachedBundle}
 }
 
+// ForEachBundleHash loops over all bundle hashes.
+func ForEachBundleHash(consumer TransactionHashBytesConsumer) {
+	bundleStorage.ForEachKeyOnly(func(tailTxHash []byte) bool {
+		return consumer(tailTxHash)
+	}, false)
+}
+
 // bundle +-0
 func ContainsBundle(tailTxHash hornet.Hash) bool {
 	return bundleStorage.Contains(databaseKeyForBundle(tailTxHash))

pkg/peering/peering.go

@@ -393,7 +393,7 @@ func (m *Manager) ConnectedAndSyncedPeerCount() (uint8, uint8) {
 			latestIndex = lsi
 		}
 
-		if p.LatestHeartbeat.SolidMilestoneIndex >= (latestIndex - isNeighborSyncedThreshold) {
+		if p.LatestHeartbeat.SolidMilestoneIndex < (latestIndex - isNeighborSyncedThreshold) {
 			// node not synced
 			continue
 		}

pkg/protocol/handshake/handshake.go

@@ -6,6 +6,8 @@ import (
 	"errors"
 	"time"
 
+	"github.com/willf/bitset"
+
 	"github.com/gohornet/hornet/pkg/protocol/message"
 	"github.com/gohornet/hornet/pkg/protocol/tlv"
 )
@@ -57,65 +59,42 @@ type Handshake struct {
 }
 
 // SupportedVersion returns the highest supported protocol version.
-func (hs Handshake) SupportedVersion(supportedMessageTypes []byte) (version int, err error) {
-	highestSupportedVersion := 0
-	highestSupportedVersionByPeer := 0
-
-	for i, ownSupportedVersion := range supportedMessageTypes {
-		// max check up to advertised versions by the peer
-		if i >= len(hs.SupportedVersions) {
-			break
-		}
-
-		// get versions matched by both
-		supported := hs.SupportedVersions[i] & ownSupportedVersion
-
-		// none supported
-		if supported == 0 {
-			continue
-		}
-
-		// iterate through all bits and find highest (more to the left is higher)
-		highest := 0
-		var j uint
-		for j = 0; j < 8; j++ {
-			if ((supported >> j) & 1) == 1 {
-				highest = int(j + 1)
-			}
-		}
-		highestSupportedVersion = highest + (i * 8)
-	}
-
-	// if the highest version is still 0, it means that we don't support
-	// any protocol version the peer supports
-	if highestSupportedVersion == 0 {
-		// grab last byte denoting the highest versions.
-		// a node will only hold version bytes if at least one version in that
-		// byte is supported, therefore it's safe to assume, that the last byte contains
-		// the highest supported version of a given node.
-		lastVersionsByte := hs.SupportedVersions[len(hs.SupportedVersions)-1]
-
-		// iterate through all bits and find highest (more to the left is higher)
-		highest := 0
-		var j uint
-		for j = 0; j < 8; j++ {
-			if ((lastVersionsByte >> j) & 0x01) == 0x01 {
-				highest = int(j + 1)
+func (hs Handshake) SupportedVersion(ownSupportedMessagesBitset *bitset.BitSet) (version int, err error) {
+	hsSupportedMessagesBitset := bitset.New(uint(len(hs.SupportedVersions) * 8))
+	hsSupportedMessagesBitset.UnmarshalBinary(hs.SupportedVersions)
+
+	bothSupportedMessagesBitset := hsSupportedMessagesBitset.Union(ownSupportedMessagesBitset)
+
+	if !bothSupportedMessagesBitset.Any() {
+		// we don't support any protocol version the peer supports
+		// return the highest supported version of a given node
+		for i := int(hsSupportedMessagesBitset.Len()) - 1; i >= 0; i-- {
+			if hsSupportedMessagesBitset.Test(uint(i)) {
+				return 1 << i, ErrVersionNotSupported
 			}
 		}
 
-		highestSupportedVersionByPeer = highest + ((len(hs.SupportedVersions) - 1) * 8)
+	}
 
-		return highestSupportedVersionByPeer, ErrVersionNotSupported
+	for i := int(bothSupportedMessagesBitset.Len()) - 1; i >= 0; i-- {
+		if bothSupportedMessagesBitset.Test(uint(i)) {
+			return 1 << i, nil
+		}
 	}
 
-	return highestSupportedVersion, nil
+	return 0, ErrVersionNotSupported
 }
 
 // NewHandshakeMessage creates a new handshake message.
-func NewHandshakeMessage(supportedMessageTypes []byte, ownSourcePort uint16, ownByteEncodedCooAddress []byte, ownUsedMWM byte) ([]byte, error) {
+func NewHandshakeMessage(ownSupportedMessagesBitset *bitset.BitSet, ownSourcePort uint16, ownByteEncodedCooAddress []byte, ownUsedMWM byte) ([]byte, error) {
 
 	maxLength := HandshakeMessageDefinition.MaxBytesLength
+
+	supportedMessageTypes, err := ownSupportedMessagesBitset.MarshalBinary()
+	if err != nil {
+		return nil, err
+	}
+
 	payloadLengthBytes := maxLength - (maxLength - 60) + uint16(len(supportedMessageTypes))
 	buf := bytes.NewBuffer(make([]byte, 0, tlv.HeaderMessageDefinition.MaxBytesLength+payloadLengthBytes))
 

pkg/protocol/protocol.go

@@ -7,6 +7,8 @@ import (
 	"strconv"
 	"sync/atomic"
 
+	"github.com/willf/bitset"
+
 	"github.com/iotaledger/hive.go/byteutils"
 	"github.com/iotaledger/hive.go/events"
 	"github.com/iotaledger/hive.go/syncutils"
@@ -32,7 +34,7 @@ var (
 	*/
 
 	// supported protocol messages/feature sets
-	SupportedFeatureSets = []byte{sting.FeatureSet}
+	SupportedFeatureSets = bitset.From([]uint64{sting.FeatureSet})
 )
 
 var (

plugins/cli/plugin.go

@@ -21,7 +21,7 @@ import (
 
 var (
 	// AppVersion version number
-	AppVersion          = "0.5.0-rc7"
+	AppVersion          = "0.5.0-rc8"
 	LatestGithubVersion = AppVersion
 
 	// AppName app code name

plugins/peering/plugin.go

@@ -64,7 +64,6 @@ func Manager() *peering.Manager {
 			ValidHandshake: handshake.Handshake{
 				ByteEncodedCooAddress: cooAddrBytes,
 				MWM:                   byte(mwm),
-				SupportedVersions:     protocol.SupportedFeatureSets,
 			},
 			MaxConnected:  config.PeeringConfig.GetInt(config.CfgPeeringMaxPeers),
 			AcceptAnyPeer: config.PeeringConfig.GetBool(config.CfgPeeringAcceptAnyConnection),

plugins/tangle/revalidation.go

@@ -74,6 +74,20 @@ func revalidateDatabase() error {
 		return err
 	}
 
+	// clean up bundles of which their tail tx doesn't exist in the database anymore
+	if err := cleanupBundles(); err != nil {
+		return err
+	}
+
+	tangle.FlushTransactionStorage()
+	tangle.FlushBundleStorage()
+	tangle.FlushMilestoneStorage()
+	tangle.FlushAddressStorage()
+	tangle.FlushApproversStorage()
+	tangle.FlushBundleTransactionsStorage()
+	tangle.FlushTagsStorage()
+	tangle.FlushUnconfirmedTxsStorage()
+
 	// Get the ledger state of the last snapshot
 	snapshotBalances, snapshotIndex, err := tangle.GetAllSnapshotBalances(nil)
 	if err != nil {
@@ -216,3 +230,45 @@ func cleanupTransactions(info *tangle.SnapshotInfo) error {
 
 	return nil
 }
+
+// deletes all bundles of which their tail tx doesn't exist in the database anymore.
+func cleanupBundles() error {
+
+	start := time.Now()
+
+	var bundleCounter int64
+	var deletionCounter int64
+	tangle.ForEachBundleHash(func(tailTxHash hornet.Hash) bool {
+		bundleCounter++
+
+		if daemon.IsStopped() {
+			return false
+		}
+
+		if (bundleCounter % 50000) == 0 {
+			if daemon.IsStopped() {
+				return false
+			}
+			log.Infof("analyzed %d bundles", bundleCounter)
+		}
+
+		storedTx := tangle.GetStoredTransactionOrNil(tailTxHash)
+
+		// delete bundle if transaction doesn't exist
+		if storedTx == nil {
+			tangle.DeleteBundle(tailTxHash)
+			deletionCounter++
+			return true
+		}
+
+		return true
+	})
+
+	if daemon.IsStopped() {
+		return tangle.ErrOperationAborted
+	}
+
+	log.Infof("%d bundles deleted, took %v", deletionCounter, time.Since(start))
+
+	return nil
+}