GT-176 Add support for Inverted index (#425)

* Inverted index

* Add support for Inverted index

fix linter warnings

* Adjust inverted index test not to use 'nested' on CE

* Rename field primarySortCompression -> primarySortCompression for inverted index field primary sort

* Add more docs and update Features field type

Co-authored-by: Nikita Vaniasin <[email protected]>

Author: jwierzbo

CHANGELOG.md

@@ -11,6 +11,7 @@
 - New `LegacyPolygons` parameter for Geo Indexes
 - New parameters (`cacheEnabled` and `storedValues`) for Persistent Indexes
 - New analyzers: `classification`, `nearest neighbors`, `minhash`
+- Add support for Inverted index
 
 ## [1.3.3](https://github.com/arangodb/go-driver/tree/v1.3.3) (2022-07-27)
 - Fix `lastValue` field type

collection_indexes.go

@@ -74,6 +74,10 @@ type CollectionIndexes interface {
 	// EnsureZKDIndex creates a ZKD multi-dimensional index for the collection, if it does not already exist.
 	// Note that zkd indexes are an experimental feature in ArangoDB 3.9.
 	EnsureZKDIndex(ctx context.Context, fields []string, options *EnsureZKDIndexOptions) (Index, bool, error)
+
+	// EnsureInvertedIndex creates an inverted index in the collection, if it does not already exist.
+	// Available in ArangoDB 3.10 and later.
+	EnsureInvertedIndex(ctx context.Context, options *InvertedIndexOptions) (Index, bool, error)
 }
 
 // EnsureFullTextIndexOptions contains specific options for creating a full text index.
@@ -196,3 +200,54 @@ type EnsureZKDIndexOptions struct {
 	// TODO: The sparse property is not supported yet
 	// Sparse bool
 }
+
+// InvertedIndexOptions provides specific options for creating an inverted index
+// Available since ArangoDB 3.10
+type InvertedIndexOptions struct {
+	// Name optional user defined name used for hints in AQL queries
+	Name string `json:"name"`
+	// InBackground if true will not hold an exclusive collection lock for the entire index creation period (rocksdb only).
+	InBackground   bool `json:"inBackground,omitempty"`
+	IsNewlyCreated bool `json:"isNewlyCreated,omitempty"`
+
+	Parallelism int `json:"parallelism,omitempty"`
+	// PrimarySort describes how individual fields are sorted
+	PrimarySort InvertedIndexPrimarySort `json:"primarySort,omitempty"`
+	// StoredValues these values specifies how the index should track values.
+	StoredValues []StoredValue `json:"storedValues,omitempty"`
+	// Analyzer to be used for indexing
+	Analyzer ArangoSearchAnalyzerType `json:"analyzer,omitempty"`
+	// Features list of analyzer features, default []
+	Features []ArangoSearchAnalyzerFeature `json:"features,omitempty"`
+	// IncludeAllFields If set to true, all fields of this element will be indexed. Defaults to false.
+	IncludeAllFields bool `json:"includeAllFields,omitempty"`
+	// TrackListPositions If set to true, values in a listed are treated as separate values. Defaults to false.
+	TrackListPositions bool `json:"trackListPositions,omitempty"`
+	// Fields contains the properties for individual fields of the element.
+	// The key of the map are field names.
+	Fields []InvertedIndexField `json:"fields,omitempty"`
+}
+
+// InvertedIndexPrimarySort defines compression and list of fields to be sorted.
+type InvertedIndexPrimarySort struct {
+	Fields []ArangoSearchPrimarySortEntry `json:"fields,omitempty"`
+	// Compression optional
+	Compression PrimarySortCompression `json:"compression,omitempty"`
+}
+
+// InvertedIndexField contains configuration for indexing of the field
+type InvertedIndexField struct {
+	// Name of the field
+	Name string `json:"name"`
+	// Analyzer optional
+	Analyzer ArangoSearchAnalyzerType `json:"analyzer,omitempty"`
+	// IncludeAllFields If set to true, all fields of this element will be indexed. Defaults to false.
+	IncludeAllFields bool `json:"includeAllFields,omitempty"`
+	// TrackListPositions If set to true, values in a listed are treated as separate values. Defaults to false.
+	TrackListPositions bool `json:"trackListPositions,omitempty"`
+	// Features list of analyzer features, default [].
+	Features []ArangoSearchAnalyzerFeature `json:"features,omitempty"`
+	// Nested
+	// Enterprise-only feature
+	Nested []InvertedIndexField `json:"nested,omitempty"`
+}

collection_indexes_impl.go

@@ -24,6 +24,7 @@ package driver
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 )
 
@@ -54,8 +55,7 @@ type indexData struct {
 }
 
 type indexListResponse struct {
-	Indexes     []indexData          `json:"indexes,omitempty"`
-	Identifiers map[string]indexData `json:"identifiers,omitempty"`
+	Indexes []json.RawMessage `json:"indexes,omitempty"`
 	ArangoError
 }
 
@@ -123,7 +123,7 @@ func (c *collection) Indexes(ctx context.Context) ([]Index, error) {
 	}
 	result := make([]Index, 0, len(data.Indexes))
 	for _, x := range data.Indexes {
-		idx, err := newIndex(x, c)
+		idx, err := newIndexFromMap(x, c)
 		if err != nil {
 			return nil, WithStack(err)
 		}
@@ -313,6 +313,48 @@ func (c *collection) EnsureZKDIndex(ctx context.Context, fields []string, option
 	return idx, created, nil
 }
 
+type invertedIndexData struct {
+	InvertedIndexOptions
+	Type string `json:"type"`
+	ID   string `json:"id,omitempty"`
+
+	ArangoError `json:",inline"`
+}
+
+// EnsureInvertedIndex creates an inverted index in the collection, if it does not already exist.
+// Available in ArangoDB 3.10 and later.
+func (c *collection) EnsureInvertedIndex(ctx context.Context, options *InvertedIndexOptions) (Index, bool, error) {
+	req, err := c.conn.NewRequest("POST", path.Join(c.db.relPath(), "_api/index"))
+	if err != nil {
+		return nil, false, WithStack(err)
+	}
+	if options == nil {
+		options = &InvertedIndexOptions{}
+	}
+	req.SetQuery("collection", c.name)
+	if _, err := req.SetBody(invertedIndexData{InvertedIndexOptions: *options, Type: string(InvertedIndex)}); err != nil {
+		return nil, false, WithStack(err)
+	}
+	resp, err := c.conn.Do(ctx, req)
+	if err != nil {
+		return nil, false, WithStack(err)
+	}
+	if err := resp.CheckStatus(200, 201); err != nil {
+		return nil, false, WithStack(err)
+	}
+	created := resp.StatusCode() == 201
+
+	var data invertedIndexData
+	if err := resp.ParseBody("", &data); err != nil {
+		return nil, false, WithStack(err)
+	}
+	idx, err := newInvertedIndex(data, c)
+	if err != nil {
+		return nil, false, WithStack(err)
+	}
+	return idx, created, nil
+}
+
 // ensureIndex creates a persistent index in the collection, if it does not already exist.
 // Fields is a slice of attribute paths.
 // The index is returned, together with a boolean indicating if the index was newly created (true) or pre-existing (false).

edge_collection_indexes_impl.go

@@ -135,3 +135,13 @@ func (c *edgeCollection) EnsureZKDIndex(ctx context.Context, fields []string, op
 	}
 	return result, created, nil
 }
+
+// EnsureInvertedIndex creates an inverted index in the collection, if it does not already exist.
+// Available in ArangoDB 3.10 and later.
+func (c *edgeCollection) EnsureInvertedIndex(ctx context.Context, options *InvertedIndexOptions) (Index, bool, error) {
+	result, created, err := c.rawCollection().EnsureInvertedIndex(ctx, options)
+	if err != nil {
+		return nil, false, WithStack(err)
+	}
+	return result, created, nil
+}

index.go

@@ -38,6 +38,7 @@ const (
 	EdgeIndex       = IndexType("edge")
 	TTLIndex        = IndexType("ttl")
 	ZKDIndex        = IndexType("zkd")
+	InvertedIndex   = IndexType("inverted")
 )
 
 // Index provides access to a single index in a single collection.
@@ -95,4 +96,7 @@ type Index interface {
 
 	// StoredValues returns a list of stored values for this index - PersistentIndex only
 	StoredValues() []string
+
+	// InvertedIndexOptions returns the inverted index options for this index - InvertedIndex only
+	InvertedIndexOptions() InvertedIndexOptions
 }

index_impl.go

@@ -24,6 +24,7 @@ package driver
 
 import (
 	"context"
+	"encoding/json"
 	"path"
 	"strings"
 )
@@ -49,6 +50,8 @@ func indexStringToType(indexTypeString string) (IndexType, error) {
 		return TTLIndex, nil
 	case string(ZKDIndex):
 		return ZKDIndex, nil
+	case string(InvertedIndex):
+		return InvertedIndex, nil
 	default:
 		return "", WithStack(InvalidArgumentError{Message: "unknown index type"})
 	}
@@ -79,12 +82,76 @@ func newIndex(data indexData, col *collection) (Index, error) {
 	}, nil
 }
 
+// newIndex creates a new Index implementation.
+func newInvertedIndex(data invertedIndexData, col *collection) (Index, error) {
+	if data.ID == "" {
+		return nil, WithStack(InvalidArgumentError{Message: "id is empty"})
+	}
+	parts := strings.Split(data.ID, "/")
+	if len(parts) != 2 {
+		return nil, WithStack(InvalidArgumentError{Message: "id must be `collection/name`"})
+	}
+	if col == nil {
+		return nil, WithStack(InvalidArgumentError{Message: "col is nil"})
+	}
+	indexType, err := indexStringToType(data.Type)
+	if err != nil {
+		return nil, WithStack(err)
+	}
+
+	dataIndex := indexData{
+		ID:             data.ID,
+		Type:           data.Type,
+		InBackground:   &data.InvertedIndexOptions.InBackground,
+		IsNewlyCreated: &data.InvertedIndexOptions.IsNewlyCreated,
+		Name:           data.InvertedIndexOptions.Name,
+		ArangoError:    data.ArangoError,
+	}
+	return &index{
+		indexData:         dataIndex,
+		invertedDataIndex: data,
+		indexType:         indexType,
+		col:               col,
+		db:                col.db,
+		conn:              col.conn,
+	}, nil
+}
+
+// newIndexFrom map returns Index implementation based on index type extracted from rawData
+func newIndexFromMap(rawData json.RawMessage, col *collection) (Index, error) {
+	type generalIndexData struct {
+		Type string `json:"type"`
+	}
+	var gen generalIndexData
+	err := json.Unmarshal(rawData, &gen)
+	if err != nil {
+		return nil, WithStack(err)
+	}
+
+	if IndexType(gen.Type) == InvertedIndex {
+		var idxData invertedIndexData
+		err = json.Unmarshal(rawData, &idxData)
+		if err != nil {
+			return nil, WithStack(err)
+		}
+		return newInvertedIndex(idxData, col)
+	}
+
+	var idxData indexData
+	err = json.Unmarshal(rawData, &idxData)
+	if err != nil {
+		return nil, WithStack(err)
+	}
+	return newIndex(idxData, col)
+}
+
 type index struct {
 	indexData
-	indexType IndexType
-	db        *database
-	col       *collection
-	conn      Connection
+	invertedDataIndex invertedIndexData
+	indexType         IndexType
+	db                *database
+	col               *collection
+	conn              Connection
 }
 
 // relPath creates the relative path to this index (`_db/<db-name>/_api/index`)
@@ -197,6 +264,11 @@ func (i *index) StoredValues() []string {
 	return i.indexData.StoredValues
 }
 
+// InvertedIndexOptions returns the inverted index options for this index - InvertedIndex only
+func (i *index) InvertedIndexOptions() InvertedIndexOptions {
+	return i.invertedDataIndex.InvertedIndexOptions
+}
+
 // Remove removes the entire index.
 // If the index does not exist, a NotFoundError is returned.
 func (i *index) Remove(ctx context.Context) error {

test/index_ensure_test.go

@@ -501,3 +501,106 @@ func TestEnsureZKDIndexWithOptions(t *testing.T) {
 	err = idx.Remove(nil)
 	require.NoError(t, err)
 }
+
+// TestEnsureInvertedIndex creates a collection with an inverted index
+func TestEnsureInvertedIndex(t *testing.T) {
+	ctx := context.Background()
+
+	c := createClientFromEnv(t, true)
+	EnsureVersion(t, ctx, c).CheckVersion(MinimumVersion("3.10.0"))
+
+	db := ensureDatabase(ctx, c, "index_test", nil, t)
+	col := ensureCollection(ctx, db, fmt.Sprintf("inverted_index_opt_test"), nil, t)
+
+	type testCase struct {
+		IsEE    bool
+		Options driver.InvertedIndexOptions
+	}
+	testCases := []testCase{
+		{
+			IsEE: false,
+			Options: driver.InvertedIndexOptions{
+				Name: "inverted-opt",
+				PrimarySort: driver.InvertedIndexPrimarySort{
+					Fields: []driver.ArangoSearchPrimarySortEntry{
+						{Field: "test1", Ascending: newBool(true)},
+						{Field: "test2", Ascending: newBool(false)},
+					},
+					Compression: driver.PrimarySortCompressionLz4,
+				},
+				Features:     []driver.ArangoSearchAnalyzerFeature{},
+				StoredValues: []driver.StoredValue{},
+				Fields: []driver.InvertedIndexField{
+					{Name: "field1", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeatureFrequency}, Nested: nil},
+					{Name: "field2", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeaturePosition}, TrackListPositions: false, Nested: nil},
+				},
+			},
+		},
+		{
+			IsEE: true,
+			Options: driver.InvertedIndexOptions{
+				Name: "inverted-opt-nested",
+				PrimarySort: driver.InvertedIndexPrimarySort{
+					Fields: []driver.ArangoSearchPrimarySortEntry{
+						{Field: "test1", Ascending: newBool(true)},
+						{Field: "test2", Ascending: newBool(false)},
+					},
+					Compression: driver.PrimarySortCompressionLz4,
+				},
+				Features:     []driver.ArangoSearchAnalyzerFeature{},
+				StoredValues: []driver.StoredValue{},
+				Fields: []driver.InvertedIndexField{
+					{Name: "field1", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeatureFrequency}, Nested: nil},
+					{Name: "field2", Features: []driver.ArangoSearchAnalyzerFeature{driver.ArangoSearchAnalyzerFeaturePosition}, TrackListPositions: false,
+						Nested: []driver.InvertedIndexField{
+							{
+								Name: "some-nested-field",
+								Nested: []driver.InvertedIndexField{
+									{Name: "test"},
+									{Name: "bas", Nested: []driver.InvertedIndexField{
+										{Name: "a", Features: nil},
+									}},
+									{Name: "kas", Nested: []driver.InvertedIndexField{
+										{Name: "b", TrackListPositions: true},
+										{Name: "c"},
+									}},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.Options.Name, func(t *testing.T) {
+			if tc.IsEE {
+				skipNoEnterprise(t)
+			}
+
+			idx, created, err := col.EnsureInvertedIndex(ctx, &tc.Options)
+			require.NoError(t, err)
+			require.True(t, created)
+
+			tc.Options.IsNewlyCreated = true
+			tc.Options.Analyzer = driver.ArangoSearchAnalyzerTypeIdentity // default value for analyzer
+
+			requireIdxEquality := func(invertedIdx driver.Index) {
+				require.Equal(t, driver.InvertedIndex, idx.Type())
+				require.Equal(t, tc.Options.Name, idx.UserName())
+				require.Equal(t, tc.Options, idx.InvertedIndexOptions())
+			}
+			requireIdxEquality(idx)
+
+			indexes, err := col.Indexes(ctx)
+			require.NoError(t, err)
+			require.NotEmpty(t, indexes)
+
+			requireIdxEquality(indexes[0])
+
+			err = idx.Remove(nil)
+			require.NoError(t, err)
+		})
+	}
+}