From 2aff7cd2cd78318e87c0821a10467026b1487bf0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 25 Dec 2024 19:18:55 +0300 Subject: [PATCH] node/object/put: push first and previous parts to chain A first object has its own OID to simplify detection of such objects by indexed meta-information. Closes #2935. Signed-off-by: Pavel Karpy --- pkg/core/object/replicate.go | 16 +++++++++++++--- pkg/core/object/replicate_test.go | 18 +++++++++++++----- .../transport/object/grpc/replication.go | 13 ++++++++++--- .../transport/object/grpc/replication_test.go | 2 +- pkg/services/object/put/distributed.go | 9 ++++++++- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/pkg/core/object/replicate.go b/pkg/core/object/replicate.go index 5df0fc79f6..08f3976a76 100644 --- a/pkg/core/object/replicate.go +++ b/pkg/core/object/replicate.go @@ -17,8 +17,10 @@ const ( networkMagicKey = "network" // optional - deletedKey = "deleted" - lockedKey = "locked" + firstPartKey = "firstPart" + previousPartKey = "previousPart" + deletedKey = "deleted" + lockedKey = "locked" ) // EncodeReplicationMetaInfo uses NEO's map (strict order) serialized format as a raw @@ -31,11 +33,13 @@ const ( // "size": payload size // "validUntil": last valid block number for meta information // "network": network magic +// "firstPart": [OPTIONAL] _raw_ object ID (32 bytes) +// "previousPart": [OPTIONAL] _raw_ object ID (32 bytes) // "deleted": [OPTIONAL] array of _raw_ object IDs // "locked": [OPTIONAL] array of _raw_ object IDs // // Last valid epoch is object's creation epoch + 10. -func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64, +func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, deleted, locked []oid.ID, vub uint64, magicNumber uint32) []byte { kvs := []stackitem.MapElement{ kv(cidKey, cID[:]), @@ -45,6 +49,12 @@ func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64, kv(networkMagicKey, magicNumber), } + if !firstPart.IsZero() { + kvs = append(kvs, kv(firstPartKey, firstPart[:])) + } + if !previousPart.IsZero() { + kvs = append(kvs, kv(previousPartKey, previousPart[:])) + } if len(deleted) > 0 { kvs = append(kvs, oidsKV(deletedKey, deleted)) } diff --git a/pkg/core/object/replicate_test.go b/pkg/core/object/replicate_test.go index 39c7521694..e92bfaf71a 100644 --- a/pkg/core/object/replicate_test.go +++ b/pkg/core/object/replicate_test.go @@ -17,11 +17,13 @@ func TestMetaInfo(t *testing.T) { oID := oidtest.ID() cID := cidtest.ID() size := rand.Uint64() + first := oidtest.ID() + prev := oidtest.ID() deleted := oidtest.IDs(10) locked := oidtest.IDs(10) validUntil := rand.Uint64() - raw := EncodeReplicationMetaInfo(cID, oID, size, deleted, locked, validUntil, network) + raw := EncodeReplicationMetaInfo(cID, oID, first, prev, size, deleted, locked, validUntil, network) item, err := stackitem.Deserialize(raw) require.NoError(t, err) @@ -44,11 +46,17 @@ func TestMetaInfo(t *testing.T) { require.Equal(t, networkMagicKey, string(mm[4].Key.Value().([]byte))) require.Equal(t, network, uint32(mm[4].Value.Value().(*big.Int).Uint64())) - require.Equal(t, deletedKey, string(mm[5].Key.Value().([]byte))) - require.Equal(t, deleted, stackItemToOIDs(t, mm[5].Value)) + require.Equal(t, firstPartKey, string(mm[5].Key.Value().([]byte))) + require.Equal(t, first[:], mm[5].Value.Value().([]byte)) - require.Equal(t, lockedKey, string(mm[6].Key.Value().([]byte))) - require.Equal(t, locked, stackItemToOIDs(t, mm[6].Value)) + require.Equal(t, previousPartKey, string(mm[6].Key.Value().([]byte))) + require.Equal(t, prev[:], mm[6].Value.Value().([]byte)) + + require.Equal(t, deletedKey, string(mm[7].Key.Value().([]byte))) + require.Equal(t, deleted, stackItemToOIDs(t, mm[7].Value)) + + require.Equal(t, lockedKey, string(mm[8].Key.Value().([]byte))) + require.Equal(t, locked, stackItemToOIDs(t, mm[8].Value)) } func stackItemToOIDs(t *testing.T, value stackitem.Item) []oid.ID { diff --git a/pkg/network/transport/object/grpc/replication.go b/pkg/network/transport/object/grpc/replication.go index 4c99e5cf89..0a50463a33 100644 --- a/pkg/network/transport/object/grpc/replication.go +++ b/pkg/network/transport/object/grpc/replication.go @@ -206,6 +206,13 @@ func objectFromMessage(gMsg *objectGRPC.Object) (*object.Object, error) { } func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) { + firstObj := o.GetFirstID() + if o.HasParent() && firstObj.IsZero() && o.SplitID() == nil { + // object itself is the first one + firstObj = o.GetID() + } + prevObj := o.GetPreviousID() + var deleted []oid.ID var locked []oid.ID switch o.Type() { @@ -235,9 +242,9 @@ func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) { secondBlock := firstBlock + currentEpochDuration thirdBlock := secondBlock + currentEpochDuration - firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, firstBlock, s.mNumber) - secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, secondBlock, s.mNumber) - thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber) + firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, firstBlock, s.mNumber) + secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, secondBlock, s.mNumber) + thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber) var firstSig neofscrypto.Signature var secondSig neofscrypto.Signature diff --git a/pkg/network/transport/object/grpc/replication_test.go b/pkg/network/transport/object/grpc/replication_test.go index 6dbd1dbae7..35beec0694 100644 --- a/pkg/network/transport/object/grpc/replication_test.go +++ b/pkg/network/transport/object/grpc/replication_test.go @@ -408,7 +408,7 @@ func TestServer_Replicate(t *testing.T) { require.Equal(t, signer.PublicKeyBytes, sig.PublicKeyBytes()) require.True(t, sig.Verify(objectcore.EncodeReplicationMetaInfo( - o.GetContainerID(), o.GetID(), o.PayloadSize(), nil, nil, + o.GetContainerID(), o.GetID(), o.GetFirstID(), o.GetPreviousID(), o.PayloadSize(), nil, nil, uint64((123+1+i)*240), mNumber))) sigsRaw = sigsRaw[:4+l] diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index ce5d26f3ae..b83604a417 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -121,6 +121,13 @@ func (t *distributedTarget) Close() (oid.ID, error) { t.encodedObject.b = nil }() + firstObj := t.obj.GetFirstID() + if t.obj.HasParent() && firstObj.IsZero() && t.obj.SplitID() == nil { + // object itself is the first one + firstObj = t.obj.GetID() + } + prevObj := t.obj.GetPreviousID() + t.obj.SetPayload(t.encodedObject.b[t.encodedObject.pldOff:]) tombOrLink := t.obj.Type() == objectSDK.TypeLink || t.obj.Type() == objectSDK.TypeTombstone @@ -152,7 +159,7 @@ func (t *distributedTarget) Close() (oid.ID, error) { } expectedVUB := (uint64(t.currentBlock)/t.currentEpochDuration + 2) * t.currentEpochDuration - t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), t.obj.PayloadSize(), deletedObjs, + t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), firstObj, prevObj, t.obj.PayloadSize(), deletedObjs, lockedObjs, expectedVUB, t.networkMagicNumber) id := t.obj.GetID() err := t.placementIterator.iterateNodesForObject(id, t.sendObject)