Merge pull request #268 from fwcd/symbol-index

Index global descriptors and provide completions for unimported symbols

Author: fwcd

gradle.properties

@@ -1,3 +1,4 @@
 projectVersion=0.9.1
-kotlinVersion=1.4.20-release-327
+kotlinVersion=1.4.30-RC-232
+exposedVersion=0.29.1
 javaVersion=11

server/build.gradle

@@ -27,6 +27,9 @@ startScripts {
 repositories {
     maven { url uri("$projectDir/lib") }
     maven { url 'https://jitpack.io' }
+    // TODO: Update once https://github.com/JetBrains/Exposed/issues/1160 is resolved
+    //       since Bintray will be shutting down soon
+    maven { url 'https://cache-redirector.jetbrains.com/dl.bintray.com/kotlin/exposed' }
 }
 
 dependencies {
@@ -38,9 +41,13 @@ dependencies {
     implementation "org.jetbrains.kotlin:kotlin-scripting-compiler-impl:$kotlinVersion"
     implementation "org.jetbrains.kotlin:kotlin-scripting-jvm-host-unshaded:$kotlinVersion"
     implementation "org.jetbrains.kotlin:kotlin-reflect:$kotlinVersion"
-    implementation "org.jetbrains.kotlin:ide-common-ij201:$kotlinVersion"
+    implementation "org.jetbrains.kotlin:ide-common-ij202:$kotlinVersion"
     // implementation("org.jetbrains.kotlin:kotlin-plugin-ij201:$kotlinVersion") { transitive = false }
     implementation 'org.jetbrains:fernflower:1.0'
+    implementation "org.jetbrains.exposed:exposed-core:$exposedVersion"
+    implementation "org.jetbrains.exposed:exposed-dao:$exposedVersion"
+    implementation "org.jetbrains.exposed:exposed-jdbc:$exposedVersion"
+    implementation 'com.h2database:h2:1.4.200'
     implementation 'com.github.fwcd:ktfmt:22bd538a1c'
     implementation 'com.beust:jcommander:1.78'
 

server/src/main/dist/licenseReport.html

@@ -16,6 +16,9 @@ <h3>Notice for packages:</h3>
       </li>
       <pre>GNU LESSER GENERAL PUBLIC LICENSE 2.1
 <a href='https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html'>https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html</a></pre>
+      <li>
+        <a href='#1544977522'>Joda-Time</a>
+      </li>
       <li>
         <a href='#1544977522'>Kotlinx-coroutines-core</a>
       </li>
@@ -428,10 +431,18 @@ <h3>Notice for packages:</h3>
    See the License for the specific language governing permissions and
    limitations under the License.
 </pre>
+      <li>
+        <a href='#76480'>Exposed</a>
+      </li>
       <li>
         <a href='#76480'>Ktfmt</a>
       </li>
       <pre>No license found</pre>
+      <li>
+        <a href='#1168029172'>H2 Database Engine</a>
+      </li>
+      <pre>MPL 2.0 or EPL 1.0
+<a href='https://h2database.com/html/license.html'>https://h2database.com/html/license.html</a></pre>
       <li>
         <a href='#-989315363'>Checker Qual</a>
       </li>
@@ -545,7 +556,7 @@ <h3>Notice for packages:</h3>
         <a href='#1288284111'>Kotlin Util Klib</a>
       </li>
       <li>
-        <a href='#1288284111'>Org.jetbrains.kotlin:ide-common-ij201</a>
+        <a href='#1288284111'>Org.jetbrains.kotlin:ide-common-ij202</a>
       </li>
       <a name='1288284111' />
       <pre>                                 Apache License
@@ -758,7 +769,10 @@ <h3>Notice for packages:</h3>
       <li>
         <a href='#-687391964'>Animal Sniffer Annotations</a>
       </li>
-      <pre>MIT license
+      <li>
+        <a href='#-687391964'>SLF4J API Module</a>
+      </li>
+      <pre>MIT License
 <a href='http://www.opensource.org/licenses/mit-license.php'>http://www.opensource.org/licenses/mit-license.php</a></pre>
       <li>
         <a href='#79718298'>LSP4J</a>

server/src/main/kotlin/org/javacs/kt/Configuration.kt

@@ -23,6 +23,11 @@ public data class CompilerConfiguration(
     val jvm: JVMConfiguration = JVMConfiguration()
 )
 
+public data class IndexingConfiguration(
+    /** Whether an index of global symbols should be built in the background. */
+    var enabled: Boolean = true
+)
+
 public data class ExternalSourcesConfiguration(
     /** Whether kls-URIs should be sent to the client to describe classes in JARs. */
     var useKlsScheme: Boolean = false,
@@ -34,5 +39,6 @@ public data class Configuration(
     val compiler: CompilerConfiguration = CompilerConfiguration(),
     val completion: CompletionConfiguration = CompletionConfiguration(),
     val linting: LintingConfiguration = LintingConfiguration(),
+    var indexing: IndexingConfiguration = IndexingConfiguration(),
     val externalSources: ExternalSourcesConfiguration = ExternalSourcesConfiguration()
 )

server/src/main/kotlin/org/javacs/kt/KotlinLanguageServer.kt

@@ -11,27 +11,35 @@ import org.javacs.kt.externalsources.JarClassContentProvider
 import org.javacs.kt.util.AsyncExecutor
 import org.javacs.kt.util.TemporaryDirectory
 import org.javacs.kt.util.parseURI
+import org.javacs.kt.progress.Progress
+import org.javacs.kt.progress.LanguageClientProgress
 import java.net.URI
 import java.io.Closeable
 import java.nio.file.Paths
 import java.util.concurrent.CompletableFuture
 import java.util.concurrent.CompletableFuture.completedFuture
 
 class KotlinLanguageServer : LanguageServer, LanguageClientAware, Closeable {
-    private val config = Configuration()
+    val config = Configuration()
     val classPath = CompilerClassPath(config.compiler)
 
     private val tempDirectory = TemporaryDirectory()
     private val uriContentProvider = URIContentProvider(JarClassContentProvider(config.externalSources, classPath, tempDirectory))
-    val sourcePath = SourcePath(classPath, uriContentProvider)
+    val sourcePath = SourcePath(classPath, uriContentProvider, config.indexing)
     val sourceFiles = SourceFiles(sourcePath, uriContentProvider)
 
     private val textDocuments = KotlinTextDocumentService(sourceFiles, sourcePath, config, tempDirectory, uriContentProvider)
     private val workspaces = KotlinWorkspaceService(sourceFiles, sourcePath, classPath, textDocuments, config)
     private val protocolExtensions = KotlinProtocolExtensionService(uriContentProvider)
 
     private lateinit var client: LanguageClient
+
     private val async = AsyncExecutor()
+    private var progressFactory: Progress.Factory = Progress.Factory.None
+        set(factory: Progress.Factory) {
+            field = factory
+            sourcePath.progressFactory = factory
+        }
 
     override fun connect(client: LanguageClient) {
         this.client = client
@@ -72,44 +80,26 @@ class KotlinLanguageServer : LanguageServer, LanguageClientAware, Closeable {
         val clientCapabilities = params.capabilities
         config.completion.snippets.enabled = clientCapabilities?.textDocument?.completion?.completionItem?.snippetSupport ?: false
 
-        val folders = params.workspaceFolders
-
-        fun reportProgress(notification: WorkDoneProgressNotification) {
-            params.workDoneToken?.let {
-                client.notifyProgress(ProgressParams(it, notification))
-            }
+        if (clientCapabilities?.window?.workDoneProgress ?: false) {
+            progressFactory = LanguageClientProgress.Factory(client)
         }
 
-        reportProgress(WorkDoneProgressBegin().apply {
-            title = "Adding Kotlin workspace folders"
-            percentage = 0
-        })
+        val folders = params.workspaceFolders
+        val progress = params.workDoneToken?.let { LanguageClientProgress("Workspace folders", it, client) }
 
         folders.forEachIndexed { i, folder ->
             LOG.info("Adding workspace folder {}", folder.name)
             val progressPrefix = "[${i + 1}/${folders.size}] ${folder.name}"
             val progressPercent = (100 * i) / folders.size
 
-            reportProgress(WorkDoneProgressReport().apply {
-                message = "$progressPrefix: Updating source path"
-                percentage = progressPercent
-            })
-
+            progress?.update("$progressPrefix: Updating source path", progressPercent)
             val root = Paths.get(parseURI(folder.uri))
             sourceFiles.addWorkspaceRoot(root)
 
-            reportProgress(WorkDoneProgressReport().apply {
-                message = "$progressPrefix: Updating class path"
-                percentage = progressPercent
-            })
-
+            progress?.update("$progressPrefix: Updating class path", progressPercent)
             val refreshed = classPath.addWorkspaceRoot(root)
             if (refreshed) {
-                reportProgress(WorkDoneProgressReport().apply {
-                    message = "$progressPrefix: Refreshing source path"
-                    percentage = progressPercent
-                })
-
+                progress?.update("$progressPrefix: Refreshing source path", progressPercent)
                 sourcePath.refresh()
             }
         }

server/src/main/kotlin/org/javacs/kt/KotlinTextDocumentService.kt

@@ -156,7 +156,7 @@ class KotlinTextDocumentService(
             LOG.info("Completing at {}", describePosition(position))
 
             val (file, cursor) = recover(position, Recompile.NEVER) // TODO: Investigate when to recompile
-            val completions = completions(file, cursor, config.completion)
+            val completions = completions(file, cursor, sp.index, config.completion)
             LOG.info("Found {} items", completions.items.size)
 
             Either.forRight<List<CompletionItem>, CompletionList>(completions)

server/src/main/kotlin/org/javacs/kt/KotlinWorkspaceService.kt

@@ -123,6 +123,14 @@ class KotlinWorkspaceService(
                 }
             }
 
+            // Update indexing options
+            get("indexing")?.asJsonObject?.apply {
+                val indexing = config.indexing
+                get("enabled")?.asBoolean?.let {
+                    indexing.enabled = it
+                }
+            }
+
             // Update options about external sources e.g. JAR files, decompilers, etc
             get("externalSources")?.asJsonObject?.apply {
                 val externalSources = config.externalSources

server/src/main/kotlin/org/javacs/kt/SourcePath.kt

@@ -1,12 +1,20 @@
 package org.javacs.kt
 
 import org.javacs.kt.compiler.CompilationKind
+import org.javacs.kt.util.AsyncExecutor
 import org.javacs.kt.util.fileExtension
 import org.javacs.kt.util.filePath
 import org.javacs.kt.util.describeURI
+import org.javacs.kt.index.SymbolIndex
+import org.javacs.kt.progress.Progress
+import org.javacs.kt.IndexingConfiguration
 import com.intellij.lang.Language
 import com.intellij.psi.PsiFile
+import com.intellij.openapi.fileTypes.FileType
+import com.intellij.openapi.fileTypes.LanguageFileType
 import org.jetbrains.kotlin.container.ComponentProvider
+import org.jetbrains.kotlin.container.getService
+import org.jetbrains.kotlin.descriptors.ModuleDescriptor
 import org.jetbrains.kotlin.psi.KtFile
 import org.jetbrains.kotlin.resolve.BindingContext
 import org.jetbrains.kotlin.resolve.CompositeBindingContext
@@ -18,13 +26,25 @@ import java.util.concurrent.locks.ReentrantLock
 
 class SourcePath(
     private val cp: CompilerClassPath,
-    private val contentProvider: URIContentProvider
+    private val contentProvider: URIContentProvider,
+    private val indexingConfig: IndexingConfiguration
 ) {
     private val files = mutableMapOf<URI, SourceFile>()
     private val parseDataWriteLock = ReentrantLock()
 
+    private val indexAsync = AsyncExecutor()
+    private var indexInitialized: Boolean = false
+    var indexEnabled: Boolean by indexingConfig::enabled
+    val index = SymbolIndex()
+
     var beforeCompileCallback: () -> Unit = {}
 
+    var progressFactory: Progress.Factory = Progress.Factory.None
+        set(factory: Progress.Factory) {
+            field = factory
+            index.progressFactory = factory
+        }
+
     private inner class SourceFile(
         val uri: URI,
         var content: String,
@@ -36,7 +56,7 @@ class SourcePath(
         val language: Language? = null,
         val isTemporary: Boolean = false // A temporary source file will not be returned by .all()
     ) {
-        val extension: String? = uri.fileExtension ?: language?.associatedFileType?.defaultExtension
+        val extension: String? = uri.fileExtension ?: "kt" // TODO: Use language?.associatedFileType?.defaultExtension again
         val isScript: Boolean = extension == "kts"
         val kind: CompilationKind =
             if (path?.fileName?.toString()?.endsWith(".gradle.kts") ?: false) CompilationKind.BUILD_SCRIPT
@@ -85,6 +105,8 @@ class SourcePath(
                 compiledContainer = container
                 compiledFile = parsed
             }
+
+            initializeIndexAsyncIfNeeded(container)
         }
 
         private fun doCompileIfChanged() {
@@ -190,6 +212,11 @@ class SourcePath(
                 }
             }
 
+            // Only index normal files, not build files
+            if (kind == CompilationKind.DEFAULT) {
+                initializeIndexAsyncIfNeeded(container)
+            }
+
             return context
         }
 
@@ -203,6 +230,18 @@ class SourcePath(
         return CompositeBindingContext.create(combined)
     }
 
+    /**
+     * Initialized the symbol index asynchronously, if not
+     * already done.
+     */
+    private fun initializeIndexAsyncIfNeeded(container: ComponentProvider) = indexAsync.execute {
+        if (indexEnabled && !indexInitialized) {
+            indexInitialized = true
+            val module = container.getService(ModuleDescriptor::class.java)
+            index.refresh(module)
+        }
+    }
+
     /**
      * Recompiles all source files that are initialized.
      */