set to on_disk mode in default

Author: JHZhang2736

deploy/magda-csv-semantic-indexer/templates/deployment.yaml

@@ -44,7 +44,7 @@ spec:
 {{ toYaml .Values.resources | indent 10 }}
         env:
         - name: NODE_PORT
-          value: "80"
+          value: {{ .Values.port | quote }}
         - name: REGISTRY_URL
           value: "http://registry-api/v0"
         - name: REGISTRY_READ_ONLY_URL

deploy/magda-csv-semantic-indexer/values.yaml

@@ -10,11 +10,11 @@ global:
       numberOfReplicas: 0
       knnVectorFieldConfig:
         # -- Vector workload mode: `on_disk` or `in_memory`.
-        mode: "in_memory"
+        mode: "on_disk"
         # -- Dimension of the embedding vectors.
         dimension: 768
         # -- The compression_level mapping parameter selects a quantization encoder that reduces vector memory consumption by the given factor.
-        compressionLevel: null
+        compressionLevel: 32x
         # Supported values: l1, l2, innerProduct, cosine, linf
         spaceType: "l2"
         # -- Similar to efSearch but used during index construction. Higher values improve search quality but increase index build time.
@@ -24,10 +24,10 @@ global:
         # -- The maximum number of graph edges per vector. Higher values increase memory usage but may improve search quality.
         m: 16
         # -- FAISS Encoder configuration (If compressionLevel is set, encoder will be ignored).
-        encoder:
-          name: "sq"
-          type: "fp16"
-          clip: false
+        # encoder: null
+          # name: "sq"
+          # type: "fp16"
+          # clip: false
     defaultDatasetBucket: "magda-datasets"
 
 opensearchURL: http://opensearch:9200

package.json

@@ -52,7 +52,7 @@
         "@langchain/core": "^0.3.61",
         "@langchain/textsplitters": "https://gitpkg.vercel.app/JHZhang2736/langchainjs/libs/langchain-textsplitters?textsplitters-build",
         "@magda/esm-utils": "^1.0.1",
-        "@magda/semantic-indexer-sdk": "6.0.0-alpha.6",
+        "@magda/semantic-indexer-sdk": "6.0.0-alpha.8",
         "csv-parse": "^6.1.0",
         "fs-extra": "^11.3.0",
         "yargs": "17.6.2"