ES|QL query log

distribution/src/config/log4j2.properties

@@ -129,3 +129,26 @@ logger.index_indexing_slowlog.name = index.indexing.slowlog.index
 logger.index_indexing_slowlog.level = trace
 logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling
 logger.index_indexing_slowlog.additivity = false
+
+
+######## ES|QL query log JSON ####################
+appender.esql_querylog_rolling.type = RollingFile
+appender.esql_querylog_rolling.name = esql_querylog_rolling
+appender.esql_querylog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
+  .cluster_name}_esql_querylog.json
+appender.esql_querylog_rolling.layout.type = ECSJsonLayout
+appender.esql_querylog_rolling.layout.dataset = elasticsearch.esql_querylog
+
+appender.esql_querylog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\
+  .cluster_name}_esql_querylog-%i.json.gz
+appender.esql_querylog_rolling.policies.type = Policies
+appender.esql_querylog_rolling.policies.size.type = SizeBasedTriggeringPolicy
+appender.esql_querylog_rolling.policies.size.size = 1GB
+appender.esql_querylog_rolling.strategy.type = DefaultRolloverStrategy
+appender.esql_querylog_rolling.strategy.max = 4
+#################################################
+
+logger.esql_querylog_rolling.name = esql.querylog
+logger.esql_querylog_rolling.level = trace
+logger.esql_querylog_rolling.appenderRef.esql_querylog_rolling.ref = esql_querylog_rolling
+logger.esql_querylog_rolling.additivity = false

docs/changelog/124094.yaml

@@ -0,0 +1,5 @@
+pr: 124094
+summary: ES|QL slow log
+area: ES|QL
+type: enhancement
+issues: []

server/src/main/java/org/elasticsearch/index/SlowLogFieldProvider.java

@@ -19,4 +19,9 @@ public interface SlowLogFieldProvider {
      * @param indexSettings settings for the index
      */
     SlowLogFields create(IndexSettings indexSettings);
+
+    /**
+     * Create a field provider without index level settings
+     */
+    SlowLogFields create();
 }

server/src/main/java/org/elasticsearch/index/SlowLogFields.java

@@ -27,4 +27,12 @@ public interface SlowLogFields {
      * @return map of field name to value
      */
     Map<String, String> searchFields();
+
+    /**
+     * Slow log fields for query
+     * @return map of field name to value
+     */
+    default Map<String, String> queryFields() {
+        return Map.of();
+    }
 }

server/src/main/java/org/elasticsearch/indices/IndicesServiceBuilder.java

@@ -83,7 +83,7 @@ public class IndicesServiceBuilder {
     QueryRewriteInterceptor queryRewriteInterceptor = null;
     SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
         @Override
-        public SlowLogFields create(IndexSettings indexSettings) {
+        public SlowLogFields create() {
             return new SlowLogFields() {
                 @Override
                 public Map<String, String> indexFields() {
@@ -96,6 +96,12 @@ public Map<String, String> searchFields() {
                 }
             };
         }
+
+        @Override
+        public SlowLogFields create(IndexSettings indexSettings) {
+            return create();
+        }
+
     };
 
     public IndicesServiceBuilder settings(Settings settings) {

server/src/main/java/org/elasticsearch/node/NodeConstruction.java

@@ -112,6 +112,7 @@
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.index.IndexSettingProvider;
 import org.elasticsearch.index.IndexSettingProviders;
+import org.elasticsearch.index.IndexSettings;
 import org.elasticsearch.index.IndexingPressure;
 import org.elasticsearch.index.SlowLogFieldProvider;
 import org.elasticsearch.index.SlowLogFields;
@@ -806,26 +807,65 @@ private void construct(
         List<? extends SlowLogFieldProvider> slowLogFieldProviders = pluginsService.loadServiceProviders(SlowLogFieldProvider.class);
         // NOTE: the response of index/search slow log fields below must be calculated dynamically on every call
         // because the responses may change dynamically at runtime
-        SlowLogFieldProvider slowLogFieldProvider = indexSettings -> {
-            final List<SlowLogFields> fields = new ArrayList<>();
-            for (var provider : slowLogFieldProviders) {
-                fields.add(provider.create(indexSettings));
-            }
-            return new SlowLogFields() {
-                @Override
-                public Map<String, String> indexFields() {
-                    return fields.stream()
-                        .flatMap(f -> f.indexFields().entrySet().stream())
-                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+        SlowLogFieldProvider slowLogFieldProvider = new SlowLogFieldProvider() {
+            public SlowLogFields create() {
+                final List<SlowLogFields> fields = new ArrayList<>();
+                for (var provider : slowLogFieldProviders) {
+                    fields.add(provider.create());
                 }
+                return new SlowLogFields() {
+                    @Override
+                    public Map<String, String> indexFields() {
+                        return fields.stream()
+                            .flatMap(f -> f.indexFields().entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    }
+
+                    @Override
+                    public Map<String, String> searchFields() {
+                        return fields.stream()
+                            .flatMap(f -> f.searchFields().entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    }
+
+                    @Override
+                    public Map<String, String> queryFields() {
+                        return fields.stream()
+                            .flatMap(f -> f.queryFields().entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    }
+                };
+            }
 
-                @Override
-                public Map<String, String> searchFields() {
-                    return fields.stream()
-                        .flatMap(f -> f.searchFields().entrySet().stream())
-                        .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+            public SlowLogFields create(IndexSettings indexSettings) {
+                final List<SlowLogFields> fields = new ArrayList<>();
+                for (var provider : slowLogFieldProviders) {
+                    fields.add(provider.create(indexSettings));
                 }
-            };
+                return new SlowLogFields() {
+                    @Override
+                    public Map<String, String> indexFields() {
+                        return fields.stream()
+                            .flatMap(f -> f.indexFields().entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    }
+
+                    @Override
+                    public Map<String, String> searchFields() {
+                        return fields.stream()
+                            .flatMap(f -> f.searchFields().entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    }
+
+                    @Override
+                    public Map<String, String> queryFields() {
+                        return fields.stream()
+                            .flatMap(f -> f.queryFields().entrySet().stream())
+                            .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+                    }
+                };
+            }
+
         };
 
         IndicesService indicesService = new IndicesServiceBuilder().settings(settings)
@@ -914,7 +954,8 @@ public Map<String, String> searchFields() {
             dataStreamGlobalRetentionSettings,
             documentParsingProvider,
             taskManager,
-            projectResolver
+            projectResolver,
+            slowLogFieldProvider
         );
 
         Collection<?> pluginComponents = pluginsService.flatMap(plugin -> {

server/src/main/java/org/elasticsearch/node/PluginServiceInstances.java

@@ -20,6 +20,7 @@
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
 import org.elasticsearch.features.FeatureService;
+import org.elasticsearch.index.SlowLogFieldProvider;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.plugins.Plugin;
@@ -53,5 +54,6 @@ public record PluginServiceInstances(
     DataStreamGlobalRetentionSettings dataStreamGlobalRetentionSettings,
     DocumentParsingProvider documentParsingProvider,
     TaskManager taskManager,
-    ProjectResolver projectResolver
+    ProjectResolver projectResolver,
+    SlowLogFieldProvider slowLogFieldProvider
 ) implements Plugin.PluginServices {}

server/src/main/java/org/elasticsearch/plugins/Plugin.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.features.FeatureService;
 import org.elasticsearch.index.IndexModule;
 import org.elasticsearch.index.IndexSettingProvider;
+import org.elasticsearch.index.SlowLogFieldProvider;
 import org.elasticsearch.indices.IndicesService;
 import org.elasticsearch.indices.SystemIndices;
 import org.elasticsearch.plugins.internal.DocumentParsingProvider;
@@ -180,6 +181,11 @@ public interface PluginServices {
          * The project resolver for the cluster. This should be used to determine the active project against which a request should execute
          */
         ProjectResolver projectResolver();
+
+        /**
+         * Provider for additional SlowLog fields
+         */
+        SlowLogFieldProvider slowLogFieldProvider();
     }
 
     /**

server/src/test/java/org/elasticsearch/indices/IndicesServiceTests.java

@@ -213,7 +213,7 @@ static void setFields(Map<String, String> fields) {
         }
 
         @Override
-        public SlowLogFields create(IndexSettings indexSettings) {
+        public SlowLogFields create() {
             return new SlowLogFields() {
                 @Override
                 public Map<String, String> indexFields() {
@@ -226,6 +226,12 @@ public Map<String, String> searchFields() {
                 }
             };
         }
+
+        @Override
+        public SlowLogFields create(IndexSettings indexSettings) {
+            return create();
+        }
+
     }
 
     public static class TestAnotherSlowLogFieldProvider implements SlowLogFieldProvider {
@@ -237,7 +243,7 @@ static void setFields(Map<String, String> fields) {
         }
 
         @Override
-        public SlowLogFields create(IndexSettings indexSettings) {
+        public SlowLogFields create() {
             return new SlowLogFields() {
                 @Override
                 public Map<String, String> indexFields() {
@@ -250,6 +256,11 @@ public Map<String, String> searchFields() {
                 }
             };
         }
+
+        @Override
+        public SlowLogFields create(IndexSettings indexSettings) {
+            return create();
+        }
     }
 
     @Override

test/test-clusters/src/main/java/org/elasticsearch/test/cluster/LogType.java

@@ -15,6 +15,7 @@ public enum LogType {
     AUDIT("%s_audit.json"),
     SEARCH_SLOW("%s_index_search_slowlog.json"),
     INDEXING_SLOW("%s_index_indexing_slowlog.json"),
+    ESQL_SLOW("%s_esql_slowlog.json"),
     DEPRECATION("%s_deprecation.json");
 
     private final String filenameFormat;

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/MockAppender.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+package org.elasticsearch.xpack.esql;
+
+import org.apache.logging.log4j.core.LogEvent;
+import org.apache.logging.log4j.core.appender.AbstractAppender;
+import org.apache.logging.log4j.core.filter.RegexFilter;
+import org.apache.logging.log4j.message.Message;
+
+public class MockAppender extends AbstractAppender {
+    public LogEvent lastEvent;
+
+    public MockAppender(final String name) throws IllegalAccessException {
+        super(name, RegexFilter.createFilter(".*(\n.*)*", new String[0], false, null, null), null, false);
+    }
+
+    @Override
+    public void append(LogEvent event) {
+        lastEvent = event.toImmutable();
+    }
+
+    public Message lastMessage() {
+        return lastEvent.getMessage();
+    }
+
+    public LogEvent lastEvent() {
+        return lastEvent;
+    }
+
+    public LogEvent getLastEventAndReset() {
+        LogEvent toReturn = lastEvent;
+        lastEvent = null;
+        return toReturn;
+    }
+}