perf: Optimize wide table write performance

integration-test/src/test/java/org/apache/iotdb/db/it/selectinto/IoTDBSelectIntoIT.java

@@ -19,6 +19,7 @@
 
 package org.apache.iotdb.db.it.selectinto;
 
+import org.apache.iotdb.db.it.utils.TSDataTypeTestUtils;
 import org.apache.iotdb.it.env.EnvFactory;
 import org.apache.iotdb.it.framework.IoTDBTestRunner;
 import org.apache.iotdb.itbase.category.ClusterIT;
@@ -104,13 +105,11 @@ public class IoTDBSelectIntoIT {
   static {
     SELECT_INTO_SQL_LIST.add("CREATE DATABASE root.sg_type");
     for (int deviceId = 0; deviceId < 6; deviceId++) {
-      for (TSDataType dataType : TSDataType.values()) {
-        if (!dataType.equals(TSDataType.VECTOR) && !dataType.equals(TSDataType.UNKNOWN)) {
-          SELECT_INTO_SQL_LIST.add(
-              String.format(
-                  "CREATE TIMESERIES root.sg_type.d_%d.s_%s %s",
-                  deviceId, dataType.name().toLowerCase(), dataType));
-        }
+      for (TSDataType dataType : TSDataTypeTestUtils.getSupportedTypes()) {
+        SELECT_INTO_SQL_LIST.add(
+            String.format(
+                "CREATE TIMESERIES root.sg_type.d_%d.s_%s %s",
+                deviceId, dataType.name().toLowerCase(), dataType));
       }
     }
     for (int time = 0; time < 12; time++) {

integration-test/src/test/java/org/apache/iotdb/db/it/utils/TSDataTypeTestUtils.java

@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.it.utils;
+
+import org.apache.tsfile.enums.TSDataType;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Utility class for TSDataType operations in integration tests. This class provides helper methods
+ * to filter out unsupported data types that should not be used in tests.
+ *
+ * <p>Usage in IT tests:
+ *
+ * <pre>{@code
+ * Set<TSDataType> dataTypes = TSDataTypeTestUtils.getSupportedTypes();
+ * for (TSDataType from : dataTypes) {
+ *   for (TSDataType to : dataTypes) {
+ *     // test logic
+ *   }
+ * }
+ * }</pre>
+ *
+ * <p>To find this utility class quickly, search for: "TSDataTypeTestUtils" or "getSupportedTypes"
+ */
+public class TSDataTypeTestUtils {
+
+  private TSDataTypeTestUtils() {
+    // utility class
+  }
+
+  /**
+   * Get the set of unsupported TSDataType values that should be filtered out in tests.
+   *
+   * <p>Currently includes: VECTOR, UNKNOWN, OBJECT
+   *
+   * @return Set of unsupported TSDataType values
+   */
+  public static Set<TSDataType> getUnsupportedTypes() {
+    Set<TSDataType> unsupported = new HashSet<>();
+    unsupported.add(TSDataType.VECTOR);
+    unsupported.add(TSDataType.UNKNOWN);
+    unsupported.add(TSDataType.OBJECT);
+    return unsupported;
+  }
+
+  /**
+   * Check if a TSDataType is supported for general use (not an internal type).
+   *
+   * @param dataType the TSDataType to check
+   * @return true if the type is supported, false otherwise
+   */
+  public static boolean isSupportedType(TSDataType dataType) {
+    return !getUnsupportedTypes().contains(dataType);
+  }
+
+  /**
+   * Get all supported TSDataType values (filters out unsupported types).
+   *
+   * <p>This method filters out VECTOR, UNKNOWN, and any other types returned by {@link
+   * #getUnsupportedTypes()}.
+   *
+   * @return Set of supported TSDataType values
+   */
+  public static Set<TSDataType> getSupportedTypes() {
+    Set<TSDataType> allTypes = new HashSet<>(Arrays.asList(TSDataType.values()));
+    allTypes.removeAll(getUnsupportedTypes());
+    return allTypes;
+  }
+
+  /**
+   * Get all supported TSDataType values as a List (filters out unsupported types).
+   *
+   * @return List of supported TSDataType values
+   */
+  public static List<TSDataType> getSupportedTypesList() {
+    return Arrays.stream(TSDataType.values())
+        .filter(TSDataTypeTestUtils::isSupportedType)
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * Filter a collection of TSDataType values to only include supported types.
+   *
+   * @param dataTypes collection of TSDataType values to filter
+   * @return Set containing only supported types
+   */
+  public static Set<TSDataType> filterSupportedTypes(Set<TSDataType> dataTypes) {
+    Set<TSDataType> result = new HashSet<>(dataTypes);
+    result.removeAll(getUnsupportedTypes());
+    return result;
+  }
+}

integration-test/src/test/java/org/apache/iotdb/pipe/it/dual/tablemodel/TableModelUtils.java

@@ -108,7 +108,7 @@ public static void insertData(
     for (int i = startInclusive; i < endExclusive; ++i) {
       list.add(
           String.format(
-              "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
+              "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
               tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i));
     }
     list.add("flush");
@@ -128,7 +128,7 @@ public static void insertData(
       for (int i = startInclusive; i < endExclusive; ++i) {
         list.add(
             String.format(
-                "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
+                "insert into %s (s0, s1, s2, s3, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
                 tableName,
                 deviceIndex,
                 deviceIndex,
@@ -208,7 +208,7 @@ public static boolean insertDataNotThrowError(
     for (int i = start; i < end; ++i) {
       list.add(
           String.format(
-              "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
+              "insert into %s (s0, s1, s2, s3,  s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
               tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i));
     }
     try {
@@ -230,7 +230,7 @@ public static boolean insertData(
     for (int i = start; i < end; ++i) {
       list.add(
           String.format(
-              "insert into %s (s0, s3, s2, s1, s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
+              "insert into %s (s0, s1, s2, s3,  s4, s5, s6, s7, s8, s9, s10, s11, time) values ('t%s','t%s','t%s','t%s','%s', %s.0, %s, %s, %d, %d.0, '%s', '%s', %s)",
               tableName, i, i, i, i, i, i, i, i, i, i, getDateStr(i), i, i));
     }
     list.add("flush");

integration-test/src/test/java/org/apache/iotdb/relational/it/db/it/IoTDBInsertTableIT.java

@@ -294,6 +294,7 @@ public void testPartialInsertTablet() {
         }
         Assert.assertEquals(200, cnt);
       }
+      session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='true'");
     } catch (Exception e) {
       e.printStackTrace();
       fail(e.getMessage());
@@ -505,6 +506,7 @@ public void testInsertMultiRowWithMisMatchDataType() throws SQLException {
   public void testInsertMultiRowWithNull() throws SQLException {
     try (Connection connection = EnvFactory.getEnv().getConnection(BaseEnv.TABLE_SQL_DIALECT);
         Statement st1 = connection.createStatement()) {
+      st1.execute("SET CONFIGURATION enable_auto_create_schema='false'");
       st1.execute("use \"test\"");
       st1.execute(
           "create table wf14 (tag1 string tag, status boolean field, temperature float field)");
@@ -519,6 +521,8 @@ public void testInsertMultiRowWithNull() throws SQLException {
         st2.execute("CREATE TABLE wf15 (wt string tag, s1 double field, s2 double field)");
         st2.execute(
             "INSERT INTO wf15(wt, time, s1) VALUES ('1', 6, 10),('1', 7,12),('1', 8,14),('1', 9,160),('1', 10,null),('1', 11,58)");
+
+        st2.execute("SET CONFIGURATION enable_auto_create_schema='true'");
       } catch (SQLException e) {
         fail(e.getMessage());
       }
@@ -829,6 +833,7 @@ public void testInsertCaseSensitivity()
   @Test
   public void testInsertKeyword() throws IoTDBConnectionException, StatementExecutionException {
     try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+      session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='true'");
       session.executeNonQueryStatement("USE \"test\"");
       session.executeNonQueryStatement(
           "create table table20 ("
@@ -1066,6 +1071,7 @@ public void testInsertTabletWithTTL()
   public void testInsertUnsequenceData()
       throws IoTDBConnectionException, StatementExecutionException {
     try (ITableSession session = EnvFactory.getEnv().getTableSessionConnection()) {
+      session.executeNonQueryStatement("SET CONFIGURATION enable_auto_create_schema='true'");
       session.executeNonQueryStatement("USE \"test\"");
       // the table is missing column "m2"
       session.executeNonQueryStatement(

integration-test/src/test/java/org/apache/iotdb/relational/it/session/IoTDBSessionRelationalIT.java

@@ -19,6 +19,7 @@
 package org.apache.iotdb.relational.it.session;
 
 import org.apache.iotdb.commons.schema.table.column.TsTableColumnCategory;
+import org.apache.iotdb.db.it.utils.TSDataTypeTestUtils;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertRowsNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
 import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
@@ -64,7 +65,6 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
@@ -1626,11 +1626,7 @@ private Object genValue(TSDataType dataType, int i) {
   public void insertRelationalTabletWithAutoCastTest()
       throws IoTDBConnectionException, StatementExecutionException {
     int testNum = 14;
-    Set<TSDataType> dataTypes = new HashSet<>();
-    Collections.addAll(dataTypes, TSDataType.values());
-    dataTypes.remove(TSDataType.VECTOR);
-    dataTypes.remove(TSDataType.UNKNOWN);
-    dataTypes.remove(TSDataType.OBJECT);
+    Set<TSDataType> dataTypes = TSDataTypeTestUtils.getSupportedTypes();
 
     try {
       for (TSDataType from : dataTypes) {
@@ -1717,11 +1713,7 @@ public void dropTableOfTheSameNameTest()
   public void insertRelationalRowWithAutoCastTest()
       throws IoTDBConnectionException, StatementExecutionException {
     int testNum = 17;
-    Set<TSDataType> dataTypes = new HashSet<>();
-    Collections.addAll(dataTypes, TSDataType.values());
-    dataTypes.remove(TSDataType.VECTOR);
-    dataTypes.remove(TSDataType.UNKNOWN);
-    dataTypes.remove(TSDataType.OBJECT);
+    Set<TSDataType> dataTypes = TSDataTypeTestUtils.getSupportedTypes();
 
     for (TSDataType from : dataTypes) {
       for (TSDataType to : dataTypes) {

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -1117,8 +1117,6 @@ public class IoTDBConfig {
 
   private boolean loadActiveListeningEnable = true;
 
-  private long loadTableSchemaCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
-
   private long loadMeasurementIdCacheSizeInBytes = 2 * 1024 * 1024L; // 2MB
 
   private int loadTsFileSpiltPartitionMaxSize = 10;
@@ -4025,14 +4023,6 @@ public void setLoadActiveListeningEnable(boolean loadActiveListeningEnable) {
     this.loadActiveListeningEnable = loadActiveListeningEnable;
   }
 
-  public long getLoadTableSchemaCacheSizeInBytes() {
-    return loadTableSchemaCacheSizeInBytes;
-  }
-
-  public void setLoadTableSchemaCacheSizeInBytes(long loadTableSchemaCacheSizeInBytes) {
-    this.loadTableSchemaCacheSizeInBytes = loadTableSchemaCacheSizeInBytes;
-  }
-
   public long getLoadMeasurementIdCacheSizeInBytes() {
     return loadMeasurementIdCacheSizeInBytes;
   }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java

@@ -2342,12 +2342,6 @@ private void loadLoadTsFileProps(TrimProperties properties) {
             ? conf.getLoadActiveListeningCheckIntervalSeconds()
             : loadActiveListeningCheckIntervalSeconds);
 
-    conf.setLoadTableSchemaCacheSizeInBytes(
-        Long.parseLong(
-            properties.getProperty(
-                "load_table_schema_cache_size_in_bytes",
-                Long.toString(conf.getLoadTableSchemaCacheSizeInBytes()))));
-
     conf.setLoadMeasurementIdCacheSizeInBytes(
         Long.parseLong(
             properties.getProperty(

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/analyzer/StatementAnalyzer.java

@@ -744,9 +744,7 @@ private Scope visitInsert(WrappedInsertStatement insert, Optional<Scope> scope)
 
       final MPPQueryContext context = insert.getContext();
       InsertBaseStatement innerInsert = insert.getInnerTreeStatement();
-
-      innerInsert.semanticCheck();
-      innerInsert.toLowerCase();
+      innerInsert.toLowerCaseForDevicePath();
 
       innerInsert =
           AnalyzeUtils.analyzeInsert(

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/Metadata.java

@@ -22,6 +22,7 @@
 import org.apache.iotdb.commons.partition.DataPartition;
 import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
 import org.apache.iotdb.commons.partition.SchemaPartition;
+import org.apache.iotdb.commons.schema.table.InsertNodeMeasurementInfo;
 import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinWindowFunction;
 import org.apache.iotdb.db.exception.load.LoadAnalyzeTableColumnDisorderException;
 import org.apache.iotdb.db.exception.sql.SemanticException;
@@ -30,6 +31,7 @@
 import org.apache.iotdb.db.queryengine.plan.analyze.IModelFetcher;
 import org.apache.iotdb.db.queryengine.plan.analyze.IPartitionFetcher;
 import org.apache.iotdb.db.queryengine.plan.relational.function.OperatorType;
+import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.TableHeaderSchemaValidator;
 import org.apache.iotdb.db.queryengine.plan.relational.security.AccessControl;
 import org.apache.iotdb.db.queryengine.plan.relational.sql.ast.Expression;
 import org.apache.iotdb.db.queryengine.plan.relational.type.TypeNotFoundException;
@@ -125,6 +127,14 @@ Optional<TableSchema> validateTableHeaderSchema(
       final boolean isStrictIdColumn)
       throws LoadAnalyzeTableColumnDisorderException;
 
+  void validateInsertNodeMeasurements(
+      final String database,
+      final InsertNodeMeasurementInfo measurementInfo,
+      final MPPQueryContext context,
+      final boolean allowCreateTable,
+      final TableHeaderSchemaValidator.MeasurementValidator measurementValidator,
+      final TableHeaderSchemaValidator.TagColumnHandler tagColumnHandler);
+
   /**
    * This method is used for table device validation and should be invoked after column validation.
    *

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/metadata/TableMetadataImpl.java

@@ -24,6 +24,7 @@
 import org.apache.iotdb.commons.partition.DataPartition;
 import org.apache.iotdb.commons.partition.DataPartitionQueryParam;
 import org.apache.iotdb.commons.partition.SchemaPartition;
+import org.apache.iotdb.commons.schema.table.InsertNodeMeasurementInfo;
 import org.apache.iotdb.commons.schema.table.TreeViewSchema;
 import org.apache.iotdb.commons.schema.table.TsTable;
 import org.apache.iotdb.commons.udf.builtin.relational.TableBuiltinAggregationFunction;
@@ -1397,6 +1398,24 @@ public Optional<TableSchema> validateTableHeaderSchema(
             database, tableSchema, context, allowCreateTable, isStrictTagColumn);
   }
 
+  @Override
+  public void validateInsertNodeMeasurements(
+      final String database,
+      final InsertNodeMeasurementInfo measurementInfo,
+      final MPPQueryContext context,
+      final boolean allowCreateTable,
+      final TableHeaderSchemaValidator.MeasurementValidator measurementValidator,
+      final TableHeaderSchemaValidator.TagColumnHandler tagColumnHandler) {
+    TableHeaderSchemaValidator.getInstance()
+        .validateInsertNodeMeasurements(
+            database,
+            measurementInfo,
+            context,
+            allowCreateTable,
+            measurementValidator,
+            tagColumnHandler);
+  }
+
   @Override
   public void validateDeviceSchema(
       ITableDeviceSchemaValidation schemaValidation, MPPQueryContext context) {