Add custom DDL parser

config/checkstyle/suppressions.xml

@@ -4,4 +4,5 @@
     "https://checkstyle.org/dtds/suppressions_1_2.dtd">
 <suppressions>
   <suppress files=".*models.*" checks=".*"/>
+  <suppress files=".*ddl.*" checks=".*"/>
 </suppressions>

config/spotbugs/exclude.xml

@@ -7,6 +7,9 @@
   <Match>
     <Class name="~.*models.*"/>
   </Match>
+  <Match>
+    <Class name="~.*ddl.*"/>
+  </Match>
 
   <!--
     Allow a class to hold or return mutable objects. While this has obvious risks, it is much too
@@ -24,4 +27,4 @@
     <Class name="~.*PipelineRules.*"/>
     <Bug code="CT"/>
   </Match>
-</FindBugsFilter>
+</FindBugsFilter>

gradle/libs.versions.toml

@@ -2,10 +2,11 @@
 assertj = "org.assertj:assertj-core:3.26.3"
 avro = "org.apache.avro:avro:1.11.4"
 calcite-avatica = "org.apache.calcite.avatica:avatica:1.27.0"
-calcite-core = "org.apache.calcite:calcite-core:1.40.0"
-calcite-server = "org.apache.calcite:calcite-server:1.40.0"
+calcite-core = "org.apache.calcite:calcite-core:1.41.0"
+calcite-server = "org.apache.calcite:calcite-server:1.41.0"
 commons-cli = "commons-cli:commons-cli:1.4"
 commons-dbcp2 = "org.apache.commons:commons-dbcp2:2.11.0"
+findbugs = "com.google.code.findbugs:jsr305:3.0.0"
 flink-clients = "org.apache.flink:flink-clients:1.18.1"
 flink-connector-base = "org.apache.flink:flink-connector-base:1.18.1"
 flink-connector-kafka = "org.apache.flink:flink-sql-connector-kafka:3.2.0-1.18"

hoptimator-cli/src/main/java/sqlline/HoptimatorAppConfig.java

@@ -18,7 +18,6 @@
 import org.apache.calcite.schema.Table;
 import org.apache.calcite.sql.SqlKind;
 import org.apache.calcite.sql.SqlNode;
-import org.apache.calcite.sql.ddl.SqlCreateMaterializedView;
 import org.apache.calcite.sql.dialect.CalciteSqlDialect;
 import org.apache.calcite.util.Pair;
 import org.jline.reader.Completer;
@@ -29,6 +28,7 @@
 import com.linkedin.hoptimator.jdbc.HoptimatorConnection;
 import com.linkedin.hoptimator.jdbc.HoptimatorDriver;
 import com.linkedin.hoptimator.jdbc.ResolvedTable;
+import com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView;
 import com.linkedin.hoptimator.util.DeploymentService;
 import com.linkedin.hoptimator.util.planner.PipelineRel;
 

hoptimator-jdbc/build.gradle

@@ -13,6 +13,7 @@ dependencies {
   implementation libs.calcite.server
   implementation libs.slf4j.api
   implementation libs.commons.dbcp2
+  compileOnly libs.findbugs 
 
   testFixturesImplementation libs.quidem
   testFixturesImplementation libs.calcite.core

hoptimator-jdbc/src/main/codegen/README.md

@@ -0,0 +1,24 @@
+## Custom DDL Parser Codegen
+
+These files are copy-pasted from Apache Calcite's `./server` module, with custom modifications.
+There is a lot of complex codegen machinery in the Calcite project. Rather than replicating all
+that here, it is easier to simply copy-paste these files back into the Calcite source tree and
+build from there.
+
+One way to do that:
+
+  1. Checkout Apache Calcite, using whatever release branch Hoptimator is built against.
+  2. Copy `./server/build.gradle.kts` into a new module, e.g. `./hoptimator-ddl/build.gradle.kts`.
+  3. Modify the build script to use our package name:
+
+    packageName.set("com.linkedin.hoptimator.jdbc.ddl")
+
+  4. Update `settings.gradle` to include the new module.
+  5. Run `./gradlew -PskipJavadoc assemble`.
+  6. Copy the resulting Java source code back to Hoptimator:
+
+    cp calcite/hoptimator-ddl/src/main/java/com/linkedin/hoptimator/jdbc/ddl/* Hoptimator/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ddl
+    cp calcite/hoptimator-ddl/build/javacc/javaCCMain/com/linkedin/hoptimator/jdbc/ddl/* Hoptimator/hoptimator-jdbc/src/main/java/com/linkedin/hoptimator/jdbc/ddl
+
+The resulting Java classes can be used within `HoptimatorDdlExecutor` etc.
+

hoptimator-jdbc/src/main/codegen/config.fmpp

@@ -0,0 +1,113 @@
+#
+# Copy-pasted from Apache Calcite with minor modifications.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to you under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data: {
+  # Data declarations for this parser.
+  #
+  # Default declarations are in default_config.fmpp; if you do not include a
+  # declaration ('imports' or 'nonReservedKeywords', for example) in this file,
+  # FMPP will use the declaration from default_config.fmpp.
+  parser: {
+    # Generated parser implementation class package and name
+    package: "com.linkedin.hoptimator.jdbc.ddl",
+    class: "HoptimatorDdlParserImpl",
+
+    # List of import statements.
+    imports: [
+      "org.apache.calcite.schema.ColumnStrategy"
+      "org.apache.calcite.sql.SqlCreate"
+      "org.apache.calcite.sql.SqlDrop"
+      "org.apache.calcite.sql.SqlTruncate"
+      "org.apache.calcite.sql.ddl.SqlCreateTableLike"
+      "org.apache.calcite.sql.ddl.SqlDdlNodes"
+      "com.linkedin.hoptimator.jdbc.ddl.SqlCreateFunction"
+      "com.linkedin.hoptimator.jdbc.ddl.SqlCreateMaterializedView"
+      "com.linkedin.hoptimator.jdbc.ddl.SqlCreateTrigger"
+    ]
+
+    # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is
+    # not a reserved keyword, add it to the 'nonReservedKeywords' section.
+    keywords: [
+      "IF"
+      "MATERIALIZED"
+      "STORED"
+      "VIRTUAL"
+      "JAR"
+      "FILE"
+      "ARCHIVE"
+      "REFRESHED"
+      "SCHEDULED"
+    ]
+
+    # List of non-reserved keywords to add;
+    # items in this list become non-reserved
+    nonReservedKeywordsToAdd: [
+      # not in core, added in server
+      "IF"
+      "MATERIALIZED"
+      "STORED"
+      "VIRTUAL"
+      "JAR"
+      "FILE"
+      "ARCHIVE"
+      "REFRESHED"
+      "SCHEDULED"
+    ]
+
+    # List of methods for parsing extensions to "CREATE [OR REPLACE]" calls.
+    # Each must accept arguments "(SqlParserPos pos, boolean replace)".
+    # Example: "SqlCreateForeignSchema".
+    createStatementParserMethods: [
+      "SqlCreateMaterializedView"
+      "SqlCreateTrigger"
+      "SqlCreateTable"
+      "SqlCreateView"
+      "SqlCreateFunction"
+    ]
+
+    # List of methods for parsing extensions to "DROP" calls.
+    # Each must accept arguments "(SqlParserPos pos)".
+    # Example: "SqlDropSchema".
+    dropStatementParserMethods: [
+      "SqlDropMaterializedView"
+      "SqlDropTrigger"
+      "SqlDropTable"
+      "SqlDropView"
+      "SqlDropFunction"
+    ]
+
+  # List of methods for parsing extensions to "TRUNCATE" calls.
+  # Each must accept arguments "(SqlParserPos pos)".
+  # Example: "SqlTruncateTable".
+  truncateStatementParserMethods: [
+  ]
+
+    # List of files in @includes directory that have parser method
+    # implementations for parsing custom SQL statements, literals or types
+    # given as part of "statementParserMethods", "literalParserMethods" or
+    # "dataTypeParserMethods".
+    # Example: "parserImpls.ftl".
+    implementationFiles: [
+      "parserImpls.ftl"
+    ]
+  }
+}
+
+freemarkerLinks: {
+  includes: includes/
+}