Fix issue 2489

DEVELOPER_GUIDE.rst

@@ -231,6 +231,8 @@ Most of the time you just need to run ./gradlew build which will make sure you p
      - Run all unit tests.
    * - ./gradlew :integ-test:integTest
      - Run all integration test (this takes time).
+   * - ./gradlew :integ-test:yamlRestTest
+     - Run rest integration test.
    * - ./gradlew :doctest:doctest
      - Run doctests
    * - ./gradlew build

core/src/main/java/org/opensearch/sql/utils/DateTimeFormatters.java

@@ -109,13 +109,6 @@ public class DateTimeFormatters {
   public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT =
       DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
 
-  public static final DateTimeFormatter DATE_TIME_FORMATTER =
-      new DateTimeFormatterBuilder()
-          .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT)
-          .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER)
-          .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
-          .toFormatter();
-
   /** todo. only support timestamp in format yyyy-MM-dd HH:mm:ss. */
   public static final DateTimeFormatter DATE_TIME_FORMATTER_WITHOUT_NANO =
       SQL_LITERAL_DATE_TIME_FORMAT;

integ-test/build.gradle

@@ -40,6 +40,7 @@ plugins {
 
 apply plugin: 'opensearch.build'
 apply plugin: 'opensearch.rest-test'
+apply plugin: 'opensearch.yaml-rest-test'
 apply plugin: 'java'
 apply plugin: 'io.freefair.lombok'
 apply plugin: 'com.wiredforcode.spawn'
@@ -258,6 +259,13 @@ testClusters {
         plugin ":opensearch-sql-plugin"
         setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
     }
+    yamlRestTest {
+        testDistribution = 'archive'
+        plugin(getJobSchedulerPlugin())
+        plugin(getGeoSpatialPlugin())
+        plugin ":opensearch-sql-plugin"
+        setting "plugins.query.datasources.encryption.masterkey", "1234567812345678"
+    }
     remoteCluster {
         testDistribution = 'archive'
         plugin(getJobSchedulerPlugin())
@@ -411,6 +419,10 @@ task integTestWithSecurity(type: RestIntegTestTask) {
     }
 }
 
+yamlRestTest {
+    systemProperty 'tests.security.manager', 'false'
+}
+
 // Run PPL ITs and new, legacy and comparison SQL ITs with new SQL engine enabled
 integTest {
     useCluster testClusters.remoteCluster

integ-test/src/yamlRestTest/java/org/opensearch/sql/rest/RestHandlerClientYamlTestSuiteIT.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.sql.rest;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.opensearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.opensearch.test.rest.yaml.OpenSearchClientYamlSuiteTestCase;
+
+public class RestHandlerClientYamlTestSuiteIT extends OpenSearchClientYamlSuiteTestCase {
+
+  public RestHandlerClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+    super(testCandidate);
+  }
+
+  @ParametersFactory
+  public static Iterable<Object[]> parameters() throws Exception {
+    return OpenSearchClientYamlSuiteTestCase.createParameters();
+  }
+}

integ-test/src/yamlRestTest/resources/rest-api-spec/api/ppl.json

@@ -0,0 +1,22 @@
+{
+  "ppl": {
+    "documentation": {
+      "url": "https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/index.rst",
+      "description": "OpenSearch PPL Reference Manual"
+    },
+    "stability" : "stable",
+    "url": {
+      "paths": [
+        {
+          "path" : "/_plugins/_ppl",
+          "methods" : ["POST"]
+        }
+      ]
+    },
+    "params": {},
+    "body": {
+      "description": "PPL Query",
+      "required":true
+    }
+  }
+}

integ-test/src/yamlRestTest/resources/rest-api-spec/api/query.settings.json

@@ -0,0 +1,22 @@
+{
+  "query.settings": {
+    "documentation": {
+      "url": "https://github.com/opensearch-project/sql/blob/main/docs/user/admin/settings.rst",
+      "description": "Query Settings"
+    },
+    "stability" : "stable",
+    "url": {
+      "paths": [
+        {
+          "path" : "/_plugins/_query/settings",
+          "methods" : ["PUT"]
+        }
+      ]
+    },
+    "params": {},
+    "body": {
+      "description": "Query Settings",
+      "required":true
+    }
+  }
+}

integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/2489.yml

@@ -0,0 +1,50 @@
+setup:
+  - do:
+      indices.create:
+        index: test
+        body:
+          settings:
+            number_of_shards: 1
+            number_of_replicas: 0
+          mappings:
+            properties:
+              timestamp:
+                type: date
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : true
+            plugins.calcite.fallback.allowed : false
+
+---
+teardown:
+  - do:
+      query.settings:
+        body:
+          transient:
+            plugins.calcite.enabled : false
+            plugins.calcite.fallback.allowed : true
+
+---
+"Handle epoch field in string format":
+  - skip:
+      features:
+        - headers
+  - do:
+      bulk:
+        index: test
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"timestamp": "1705642934886"}'
+  - do:
+      headers:
+        Content-Type: 'application/json'
+      ppl:
+        body:
+          query: 'source=test | fields timestamp'
+  - match: {"total": 1}
+  - match: {"schema": [{"name": "timestamp", "type": "timestamp"}]}
+  - match: {"datarows": [["2024-01-19 05:42:14.886"]]}
+

opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java

@@ -17,7 +17,6 @@
 import static org.opensearch.sql.data.type.ExprCoreType.STRUCT;
 import static org.opensearch.sql.data.type.ExprCoreType.TIME;
 import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP;
-import static org.opensearch.sql.utils.DateTimeFormatters.DATE_TIME_FORMATTER;
 import static org.opensearch.sql.utils.DateTimeFormatters.STRICT_HOUR_MINUTE_SECOND_FORMATTER;
 import static org.opensearch.sql.utils.DateTimeFormatters.STRICT_YEAR_MONTH_DAY_FORMATTER;
 
@@ -44,6 +43,7 @@
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.time.DateFormatters;
 import org.opensearch.common.time.FormatNames;
+import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.sql.data.model.ExprBooleanValue;
 import org.opensearch.sql.data.model.ExprByteValue;
 import org.opensearch.sql.data.model.ExprCollectionValue;
@@ -262,9 +262,10 @@ private static ExprValue parseDateTimeString(String value, OpenSearchDateType da
               DateFormatters.from(STRICT_YEAR_MONTH_DAY_FORMATTER.parse(value)).toLocalDate());
         default:
           return new ExprTimestampValue(
-              DateFormatters.from(DATE_TIME_FORMATTER.parse(value)).toInstant());
+              DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(value))
+                  .toInstant());
       }
-    } catch (DateTimeParseException ignored) {
+    } catch (DateTimeParseException | IllegalArgumentException ignored) {
       // ignored
     }
 

opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java

@@ -78,6 +78,7 @@ class OpenSearchExprValueFactoryTest {
           .put("dateV", OpenSearchDateType.of(DATE))
           .put("timeV", OpenSearchDateType.of(TIME))
           .put("timestampV", OpenSearchDateType.of(TIMESTAMP))
+          .put("timeonlyV", OpenSearchDateType.of("HH:mm:ss"))
           .put("datetimeDefaultV", OpenSearchDateType.of())
           .put("dateStringV", OpenSearchDateType.of("date"))
           .put("timeStringV", OpenSearchDateType.of("time"))
@@ -312,10 +313,6 @@ public void constructDatetime() {
             assertEquals(
                 new ExprTimestampValue("2015-01-01 12:10:30"),
                 tupleValue("{\"timestampV\":\"2015-01-01T12:10:30\"}").get("timestampV")),
-        () ->
-            assertEquals(
-                new ExprTimestampValue("2015-01-01 12:10:30"),
-                tupleValue("{\"timestampV\":\"2015-01-01 12:10:30\"}").get("timestampV")),
         () ->
             assertEquals(
                 new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)),
@@ -348,10 +345,6 @@ public void constructDatetime() {
             assertEquals(
                 new ExprTimestampValue("1984-05-10 20:30:40"),
                 tupleValue("{ \"dateTimeCustomV\" : 19840510203040 }").get("dateTimeCustomV")),
-        () ->
-            assertEquals(
-                new ExprTimestampValue("2015-01-01 12:10:30"),
-                constructFromObject("timestampV", "2015-01-01 12:10:30")),
         () ->
             assertEquals(
                 new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)),
@@ -361,7 +354,7 @@ public void constructDatetime() {
         () ->
             assertEquals(
                 new ExprTimeValue("19:36:22"),
-                tupleValue("{\"timestampV\":\"19:36:22\"}").get("timestampV")),
+                tupleValue("{\"timeonlyV\":\"19:36:22\"}").get("timeonlyV")),
 
         // case: timestamp-formatted field, but it only gets a date: should match a date
         () ->
@@ -384,10 +377,6 @@ public void constructDatetime_fromCustomFormat() {
         "Construct TIMESTAMP from \"2015-01-01 12-10-30\" failed, unsupported format.",
         exception.getMessage());
 
-    assertEquals(
-        new ExprTimestampValue("2015-01-01 12:10:30"),
-        constructFromObject("customAndEpochMillisV", "2015-01-01 12:10:30"));
-
     assertEquals(
         new ExprTimestampValue("2015-01-01 12:10:30"),
         constructFromObject("customAndEpochMillisV", "2015-01-01-12-10-30"));
@@ -700,7 +689,7 @@ public void constructArrayOfCustomEpochMillisReturnsAll() {
             List.of(
                 new ExprTimestampValue("2015-01-01 12:10:30"),
                 new ExprTimestampValue("1999-11-09 01:09:44"))),
-        tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01 12:10:30\",\"1999-11-09 01:09:44\"]}")
+        tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01-12-10-30\",\"1999-11-09-01-09-44\"]}")
             .get("customAndEpochMillisV"));
   }