Merge pull request #49 from Paulanerus/dev

Dev

Author: Paulanerus

api/build.gradle.kts

@@ -13,8 +13,4 @@ dependencies {
     implementation(kotlin("stdlib"))
 
     implementation("com.fasterxml.jackson.core:jackson-annotations:${rootProject.extra["jackson.version"]}")
-}
-
-kotlin {
-    jvmToolchain(21)
 }

build.gradle.kts

@@ -1,4 +1,6 @@
+import org.gradle.kotlin.dsl.configure
 import org.jetbrains.compose.desktop.application.dsl.TargetFormat
+import org.jetbrains.kotlin.gradle.dsl.KotlinJvmProjectExtension
 
 plugins {
     kotlin("jvm")
@@ -23,8 +25,12 @@ dependencies {
     implementation(project(":ui"))
 }
 
-kotlin {
-    jvmToolchain(21)
+allprojects {
+    plugins.withId("org.jetbrains.kotlin.jvm") {
+        extensions.configure<KotlinJvmProjectExtension> {
+            jvmToolchain(21)
+        }
+    }
 }
 
 compose.desktop {

core/build.gradle.kts

@@ -52,8 +52,4 @@ dependencies {
 
 tasks.test {
     useJUnitPlatform()
-}
-
-kotlin {
-    jvmToolchain(21)
 }

core/src/main/kotlin/dev/paulee/core/data/FileService.kt

@@ -8,15 +8,16 @@ import org.slf4j.LoggerFactory.getLogger
 import java.nio.file.Path
 import kotlin.io.path.Path
 import kotlin.io.path.createDirectories
+import kotlin.io.path.listDirectoryEntries
 import kotlin.io.path.notExists
 
 internal object FileService {
 
-    enum class OperatingSystem {
+    enum class Platform {
         Windows, MacOS, Linux, Other;
 
         companion object {
-            val current: OperatingSystem by lazy {
+            val current: Platform by lazy {
                 val os = System.getProperty("os.name")?.lowercase() ?: return@lazy Other
 
                 when {
@@ -34,32 +35,47 @@ internal object FileService {
             val isLinux: Boolean get() = current == Linux
 
             val isOther: Boolean get() = current == Other
-        }
-    }
 
-    val isCuda12xInstalled: Boolean by lazy {
-        if (OperatingSystem.isMacOS) return@lazy false
+            val isCuda12xInstalled: Boolean by lazy {
+                if (isMacOS) return@lazy false
 
-        runCatching {
-            val process = ProcessBuilder().apply {
-                command("nvcc", "--version")
-                redirectErrorStream(true)
-            }.start()
+                runCatching {
+                    val process = ProcessBuilder().apply {
+                        command("nvcc", "--version")
+                        redirectErrorStream(true)
+                    }.start()
 
-            val output = process.inputStream.bufferedReader().use { it.readText() }
-            process.waitFor()
+                    val output = process.inputStream.bufferedReader().use { it.readText() }
+                    process.waitFor()
 
-            val versionRegex = Regex("""V(\d+)\.(\d+)\.(\d+)""")
-            val matchResult = versionRegex.find(output)
+                    val versionRegex = Regex("""V(\d+)\.(\d+)\.(\d+)""")
+                    val matchResult = versionRegex.find(output)
 
-            if (matchResult != null) {
-                val majorVersion = matchResult.groupValues[1].toIntOrNull() ?: 0
+                    if (matchResult != null) {
+                        val majorVersion = matchResult.groupValues[1].toIntOrNull() ?: 0
 
-                majorVersion == 12
-            } else {
-                false
+                        majorVersion == 12
+                    } else {
+                        false
+                    }
+                }.getOrDefault(false)
             }
-        }.getOrDefault(false)
+
+            val isCuDNNInstalled by lazy {
+                if (isMacOS) return@lazy false
+
+                // This is a temporary solution until JDK 25 with FFM API can be used.
+                return@lazy runCatching {
+                    if (isWindows) {
+                        System.getenv("PATH").contains("NVIDIA\\CUDNN\\v9")
+                    } else {
+                        val lib = Path("/usr/lib")
+
+                        lib.listDirectoryEntries("libcudnn.so.9*").isNotEmpty()
+                    }
+                }.getOrDefault(false)
+            }
+        }
     }
 
     val appDir: Path get() = ensureDir(".textexplorer", true)
@@ -75,7 +91,7 @@ internal object FileService {
     private val mapper = jacksonObjectMapper().apply { enable(SerializationFeature.INDENT_OUTPUT) }
 
     init {
-        logger.info("Operating system: ${OperatingSystem.current} | CUDA: $isCuda12xInstalled")
+        logger.info("Operating system: ${Platform.current} | CUDA: ${Platform.isCuda12xInstalled}, cuDNN: ${Platform.isCuDNNInstalled}")
     }
 
     fun toJson(dataInfo: DataInfo): String? = runCatching { this.mapper.writeValueAsString(dataInfo) }

core/src/main/kotlin/dev/paulee/core/data/provider/EmbeddingProvider.kt

@@ -42,24 +42,31 @@ internal object EmbeddingProvider {
 
     private val sessions = mutableMapOf<Embedding.Model, OrtSession?>()
 
-    private val availableProvider by lazy {
-        OrtEnvironment.getAvailableProviders().orEmpty().mapNotNull { it }.toSet()
+    private val availableProvider = OrtEnvironment.getAvailableProviders().orEmpty().mapNotNull { it }.toSet()
+
+    private val canCuda = with(FileService.Platform) {
+        OrtProvider.CUDA in availableProvider && isCuda12xInstalled && isCuDNNInstalled
     }
 
-    private val options = OrtSession.SessionOptions().apply {
-        setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT)
-        setMemoryPatternOptimization(true)
-        setInterOpNumThreads(Runtime.getRuntime().availableProcessors())
-        setIntraOpNumThreads(max(1, Runtime.getRuntime().availableProcessors() / 2))
+    private val lazyOptions = lazy {
+        runCatching {
+            OrtSession.SessionOptions().apply {
+                setOptimizationLevel(OrtSession.SessionOptions.OptLevel.ALL_OPT)
+                setMemoryPatternOptimization(true)
+                setInterOpNumThreads(Runtime.getRuntime().availableProcessors())
+                setIntraOpNumThreads(max(1, Runtime.getRuntime().availableProcessors() / 2))
+
+                if (!FileService.Platform.isMacOS && canCuda) {
+                    logger.info("Selecting CUDA provider.")
+                    addCUDA()
+                }
 
-        if (!FileService.OperatingSystem.isMacOS) {
-            if (OrtProvider.CUDA in availableProvider && FileService.isCuda12xInstalled) {
-                logger.info("Selecting CUDA provider.")
-                addCUDA()
+                addCPU(true)
             }
+        }.getOrElse {
+            logger.error("Failed to create session options.", it)
+            null
         }
-
-        addCPU(true)
     }
 
     private const val CACHE_SIZE = 60_000
@@ -241,7 +248,10 @@ internal object EmbeddingProvider {
 
         sessions.values.forEach { it?.close() }
 
-        options.close()
+        with(lazyOptions) {
+            if (isInitialized())
+                value?.close()
+        }
 
         env.close()
     }
@@ -394,10 +404,12 @@ internal object EmbeddingProvider {
     }
 
     private fun createSession(model: Embedding.Model): OrtSession? {
+        if (lazyOptions.value == null) return null
+
         return runCatching {
             env.createSession(
                 FileService.modelsDir.resolve(model.name).resolve(model.modelData.model).toString(),
-                options
+                lazyOptions.value
             )
         }.getOrElse {
             logger.error("Failed to create session for model ${model.name}", it)

docs/source/content/installation.md

@@ -20,12 +20,12 @@ Both plugins and data sets can be loaded directly within the application, as doc
 
 By default, the application runs embedding models for semantic queries on the CPU, which may result in longer loading times depending on the model, data size, and hardware. GPU acceleration generally reduces these loading times. Below is a list of supported platforms for GPU acceleration:
 
-### NVIDIA GPUs  
-GPU acceleration supported using CUDA. Requires CUDA 12.x installed. Available on Linux and Windows.
-For installation, see [CUDA Toolkit](https://developer.nvidia.com/cuda-downloads).
+### NVIDIA  
+GPU acceleration supported using CUDA. Requires[ CUDA 12.x](https://developer.nvidia.com/cuda-toolkit-archive) and [cuDNN 9.x](https://developer.nvidia.com/cudnn) installed. Available on Linux and Windows.
+For installation, see [CUDA Installation Guide](https://docs.nvidia.com/cuda/cuda-installation-guide-microsoft-windows/index.html) and [Installing cuDNN Backend](https://docs.nvidia.com/deeplearning/cudnn/installation/latest/windows.html).
 
-### AMD GPUs 
+### AMD 
 Currently not supported, but planned.
 
-### Apple GPUs 
+### Apple 
 Not supported, no plans soon.

gradle.properties

@@ -14,6 +14,6 @@ duckdb.version=1.4.0.0
 caffeine.version=3.2.2
 
 api.version=1.13.1
-core.version=1.16.11
-ui.version=1.17.1
-app.version=1.5.1
+core.version=1.16.14
+ui.version=1.17.2
+app.version=1.5.2

plugins/demo/build.gradle.kts

@@ -26,8 +26,4 @@ tasks.jar {
     manifest{
         attributes["Main-Class"] = "dev.paulee.demo.DemoPlugin"
     }
-}
-
-kotlin {
-    jvmToolchain(21)
 }

ui/build.gradle.kts

@@ -20,8 +20,4 @@ dependencies {
 
     implementation(compose.desktop.currentOs)
     implementation("org.jetbrains.compose.material:material-icons-core:${rootProject.extra["icons.version"]}")
-}
-
-kotlin {
-    jvmToolchain(21)
 }

ui/src/main/kotlin/dev/paulee/ui/TextExplorerUI.kt

@@ -49,6 +49,7 @@ enum class Window {
     PluginInfo,
     Settings,
     ModelManagement,
+    PoolManagement
 }
 
 class TextExplorerUI(
@@ -208,6 +209,7 @@ class TextExplorerUI(
                     "setting.load_data" -> openWindow = Window.LoadData
                     "plugin.title" -> openWindow = Window.PluginInfo
                     "model_management.title" -> openWindow = Window.ModelManagement
+                    "pools_management.title" -> openWindow = Window.PoolManagement
                     "settings.title" -> openWindow = Window.Settings
                 }
             }
@@ -680,6 +682,8 @@ class TextExplorerUI(
 
                 Window.ModelManagement -> ModelManagerWindow(dataService) { openWindow = Window.None }
 
+                Window.PoolManagement -> PoolManagerWindow(dataService) { openWindow = Window.None }
+
                 else -> {}
             }
         }