meta: support big objects

Support chained objects in meta-service. Refs #3189, #3138.

Signed-off-by: Pavel Karpy <[email protected]>

Author: carpawell

pkg/services/meta/containers.go

@@ -299,6 +299,8 @@ func deleteObjectsOps(dbKV map[string][]byte, s storage.Store, objects []byte, c
 	rng := storage.SeekRange{}
 	mptKV := make(map[string][]byte)
 
+	objects = expandChildren(s, objects)
+
 	// nil value means "delete" operation
 
 	for len(objects) > 0 {
@@ -591,3 +593,121 @@ func (s *containerStorage) handleNewEpoch(e uint64) error {
 
 	return errors.Join(mptErr, dbErr)
 }
+
+func expandChildren(st storage.Store, rootOIDs []byte) []byte {
+	if len(rootOIDs) == 0 {
+		return nil
+	}
+
+	// sorting before every SEEK is used for exact single operation for every
+	// children searching subroutine; search is done in 3 steps:
+	//  1. root -> last/link object search
+	//  2. last/link -> first part ID search
+	//  3. first part ID -> all children search
+
+	var childrenWithParentID [][]byte
+	var rng storage.SeekRange
+	rng.Prefix = slices.Concat([]byte{attrPlainToOIDIndex}, []byte(objectsdk.FilterParentID), object.AttributeDelimiter)
+	rng.Start = make([]byte, oid.Size)
+
+	rootsSorted := slices.SortedFunc(slices.Chunk(rootOIDs, oid.Size), bytes.Compare)
+	lastRoot := rootsSorted[len(rootsSorted)-1]
+	copy(rng.Start, rootsSorted[0])
+
+	st.Seek(rng, func(k, _ []byte) bool {
+		currRoot := k[len(rng.Prefix) : len(rng.Prefix)+oid.Size]
+		if len(rootsSorted) == 0 || bytes.Compare(currRoot, lastRoot) > 0 {
+			return false
+		}
+		for i := 0; i < len(rootsSorted); i++ {
+			switch bytes.Compare(currRoot, rootsSorted[i]) {
+			case -1:
+				return true
+			case +1:
+				rootsSorted = rootsSorted[i+1:]
+				i = -1
+				continue
+			case 0:
+				childrenWithParentID = append(childrenWithParentID, slices.Clone(k[len(rng.Prefix)+oid.Size+1:]))
+				rootsSorted = rootsSorted[i+1:]
+				return true
+			}
+		}
+
+		return true
+	})
+
+	if len(childrenWithParentID) == 0 {
+		// all objects are roots, no additional children
+		return rootOIDs
+	}
+
+	firstPartOIDs := make([][]byte, 0, len(childrenWithParentID))
+	rng.Prefix = []byte{oidToAttrIndex}
+	copy(rng.Start, childrenWithParentID[0])
+
+	slices.SortFunc(childrenWithParentID, bytes.Compare)
+	st.Seek(rng, func(k, _ []byte) bool {
+		if len(childrenWithParentID) == 0 {
+			return false
+		}
+		if !bytes.HasPrefix(k[1+oid.Size:], []byte(objectsdk.FilterFirstSplitObject)) {
+			return true
+		}
+		currChild := k[1 : 1+oid.Size]
+		for i := 0; i < len(childrenWithParentID); i++ {
+			switch bytes.Compare(currChild, childrenWithParentID[i]) {
+			case -1:
+				return true
+			case +1:
+				// should never happen, it means we have info
+				// about child object with parent ID but child
+				// object does not have first part ID, nothing
+				// can be done, just skip
+				childrenWithParentID = childrenWithParentID[i+1:]
+				i = -1
+				continue
+			case 0:
+				firstPartOIDs = append(firstPartOIDs, k[1+oid.Size+len(objectsdk.FilterFirstSplitObject)+1:])
+				childrenWithParentID = childrenWithParentID[i+1:]
+				return true
+			}
+		}
+
+		return true
+	})
+
+	if len(firstPartOIDs) == 0 {
+		// unexpected but nothing to do
+		return rootOIDs
+	}
+
+	children := slices.Concat(firstPartOIDs...) // first objects are children too
+	rng.Prefix = slices.Concat([]byte{attrPlainToOIDIndex}, []byte(objectsdk.FilterFirstSplitObject), object.AttributeDelimiter)
+	copy(rng.Start, firstPartOIDs[0])
+
+	slices.SortFunc(firstPartOIDs, bytes.Compare)
+	st.Seek(rng, func(k, _ []byte) bool {
+		currChild := k[len(rng.Prefix) : len(rng.Prefix)+oid.Size]
+		if len(firstPartOIDs) == 0 {
+			return false
+		}
+		for i := 0; i < len(firstPartOIDs); i++ {
+			switch bytes.Compare(currChild, firstPartOIDs[i]) {
+			case -1:
+				return true
+			case +1:
+				firstPartOIDs = firstPartOIDs[i+1:]
+				i = -1
+				continue
+			case 0:
+				children = append(children, slices.Clone(k[len(rng.Prefix)+oid.Size+1:])...)
+				return true
+			}
+		}
+
+		return true
+	})
+
+	return slices.Concat(rootOIDs, children)
+}

pkg/services/meta/containers_test.go

@@ -1,6 +1,7 @@
 package meta
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"math/big"
@@ -41,6 +42,7 @@ func TestObjectExpiration(t *testing.T) {
 		var oo []object.Object
 		for i := range objNum {
 			o := objecttest.Object()
+			o.ResetRelations()
 			o.SetContainerID(cID)
 			setExpiration(&o, uint64(i))
 
@@ -99,9 +101,11 @@ func TestObjectExpiration(t *testing.T) {
 
 		o := objecttest.Object()
 		oID := o.GetID()
+		o.ResetRelations()
 		o.SetContainerID(cID)
 		setExpiration(&o, objExp)
 		lock := objecttest.Object()
+		lock.ResetRelations()
 		lock.SetContainerID(cID)
 		setExpiration(&lock, lockExp)
 
@@ -153,3 +157,141 @@ func TestObjectExpiration(t *testing.T) {
 		})
 	})
 }
+
+func objectChain(cID cid.ID, length int) (object.Object, []object.Object) {
+	reset := func(o *object.Object) {
+		o.SetContainerID(cID)
+		o.ResetRelations()
+		o.SetType(object.TypeRegular)
+	}
+
+	root := objecttest.Object()
+	reset(&root)
+	first := objecttest.Object()
+	reset(&first)
+
+	children := make([]object.Object, length-1)
+	children[0] = first
+	for i := range children {
+		if i != 0 {
+			children[i] = objecttest.Object()
+			reset(&children[i])
+			children[i].SetFirstID(first.GetID())
+			children[i].SetPreviousID(children[i-1].GetID())
+		}
+		if i == len(children)-1 {
+			children[i].SetParentID(root.GetID())
+			children[i].SetParent(&root)
+		}
+	}
+
+	link := objecttest.Object()
+	reset(&link)
+	link.SetFirstID(first.GetID())
+	link.SetParentID(root.GetID())
+	link.SetParent(&root)
+	link.SetType(object.TypeLink)
+
+	return root, append(children, link)
+}
+
+type bigObj struct {
+	root     object.Object
+	children []object.Object
+}
+
+func TestBigObjects(t *testing.T) {
+	cID := cidtest.ID()
+	l := zaptest.NewLogger(t)
+	ctx := context.Background()
+
+	var bigObjs []bigObj
+	for range 10 {
+		root, children := objectChain(cID, 10)
+		bigObjs = append(bigObjs, bigObj{root, children})
+	}
+
+	var rawObjSlice []object.Object
+	for _, obj := range bigObjs {
+		rawObjSlice = append(rawObjSlice, obj.children...)
+	}
+
+	net := testNetwork{}
+	net.setContainers([]cid.ID{cID})
+	net.setObjects(objsToAddrMap(rawObjSlice))
+	ee := make([]objEvent, 0, len(rawObjSlice))
+	for _, o := range rawObjSlice {
+		ev := objEvent{
+			cID:     cID,
+			oID:     o.GetID(),
+			size:    big.NewInt(testObjectSize),
+			network: big.NewInt(testNetworkMagic),
+		}
+		ev.typ = o.Type()
+		if id := o.GetPreviousID(); !id.IsZero() {
+			ev.prevObject = id[:]
+		}
+		if id := o.GetFirstID(); !id.IsZero() {
+			ev.firstObject = id[:]
+		}
+
+		ee = append(ee, ev)
+	}
+
+	st, err := storageForContainer(t.TempDir(), cID)
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		_ = st.drop()
+	})
+
+	st.putObjects(ctx, l, 0, ee, &net)
+
+	tsObj := objecttest.Object()
+	tsObj.ResetRelations()
+	tsObj.SetContainerID(cID)
+	tsObj.SetType(object.TypeTombstone)
+	tsEv := objEvent{
+		cID:     cID,
+		oID:     tsObj.GetID(),
+		size:    big.NewInt(testObjectSize),
+		network: big.NewInt(testNetworkMagic),
+	}
+	net.setObjects(objsToAddrMap([]object.Object{tsObj}))
+
+	for _, bigO := range bigObjs {
+		rID := bigO.root.GetID()
+		tsEv.deletedObjects = rID[:]
+
+		st.putObjects(ctx, l, 0, []objEvent{tsEv}, &net)
+
+		k := make([]byte, 1+oid.Size)
+		k[0] = oidIndex
+
+		for _, child := range bigO.children {
+			chID := child.GetID()
+			copy(k[1:], chID[:])
+
+			_, err = st.db.Get(k)
+			require.ErrorIs(t, err, storage.ErrKeyNotFound)
+		}
+	}
+
+	// only last operation from TS should be kept
+	tsID := tsObj.GetID()
+	st.db.Seek(storage.SeekRange{}, func(k, _ []byte) bool {
+		switch k[0] {
+		case oidIndex, oidToAttrIndex, deletedIndex:
+			if bytes.Equal(k[1:1+oid.Size], tsID[:]) {
+				return true
+			}
+		case attrIntToOIDIndex, attrPlainToOIDIndex:
+			if bytes.Equal(k[len(k)-oid.Size:], tsID[:]) {
+				return true
+			}
+		default:
+		}
+
+		t.Fatalf("empty db is expected after full clean, found key: %d", k[0])
+		return true
+	})
+}