support variable offset sizes for real

Author: TYoungSL

CMakeLists.txt

@@ -103,6 +103,23 @@ option (FLATCC_ALLOW_WERROR "allow -Werror to be configured" ON)
 # try using this option.
 option (FLATCC_IGNORE_CONST_COND "silence const condition warnings" OFF)
 
+# Enforces parsing of identifier size per the spec of 4 characters.
+# FLATBUFFERS_STRICT_IDENTIFIER_SIZE has no effect if the identifier
+# size is 4.
+# Note: Encoded identifiers will still be padded when FLATCC_OFFSET_SIZE
+# is greater than 4 and truncated when offset size is less than 4.
+option (FLATBUFFERS_STRICT_IDENTIFIER_SIZE
+        "enforce identifier size being offset size" OFF)
+
+# Relaxes the size specification for file identifier sizes to allow them
+# to be padded if shorter or truncated if longer.
+# The restriction prohibits interpreting schema oriented at other offset
+# sizes, so this is implied when FLATCC_OFFSET_SIZE is not 4.
+# This option requires FLATBUFFERS_STRICT_IDENTIFIER_SIZE to be OFF.
+cmake_dependent_option (FLATBUFFERS_RELAXED_IDENTIFIER_SIZE
+        "identifier will be padded if too short or truncated if too long"
+        OFF "FLATBUFFERS_STRICT_IDENTIFIER_SIZE" OFF)
+
 if (FLATCC_RTONLY)
     set(FLATCC_TEST off)
 endif()
@@ -179,7 +196,7 @@ if (CMAKE_C_COMPILER_ID MATCHES "Clang")
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -pedantic -Wall -Wextra")
     # Fix broken C++ alignas - either will do
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
-    #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPORTABLE_PATCH_CPLUSPLUS_STDALIGN") 
+    #set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DPORTABLE_PATCH_CPLUSPLUS_STDALIGN")
     if (FLATCC_ALLOW_WERROR)
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror")
     endif()
@@ -219,7 +236,7 @@ elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU")
             # structs, but these are valid as zero-paddded, not zero terminated.
             #
             # -Wno-format-overflow:
-            # GCC 9 warns on mistakenly assumed NULL string when 
+            # GCC 9 warns on mistakenly assumed NULL string when
             # printing from a required FlatBuffer string field.
             #
             message(STATUS "Disabling GNU C compiler warnings: -Wstringop-truncation -Wno-format-overflow")
@@ -294,11 +311,20 @@ if (FLATCC_PORTABLE)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DFLATCC_PORTABLE")
 endif()
 
-
 if (DEFINED FLATCC_OFFSET_SIZE)
     set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DFLATCC_OFFSET_SIZE=${FLATCC_OFFSET_SIZE}")
 endif()
 
+if (DEFINED FLATCC_VOFFSET_SIZE)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DFLATCC_VOFFSET_SIZE=${FLATCC_VOFFSET_SIZE}")
+endif()
+
+if (DEFINED FLATCC_OFFSET_SIZE AND NOT FLATCC_OFFSET_SIZE EQUAL 4 AND NOT DEFINED FLATBUFFERS_STRICT_IDENTIFIER_SIZE)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DFLATBUFFERS_RELAXED_IDENTIFIER_SIZE")
+elseif(DEFINED FLATBUFFERS_RELAXED_IDENTIFIER_SIZE)
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DFLATBUFFERS_RELAXED_IDENTIFIER_SIZE")
+endif()
+
 if (CLANG_VERSION)
     message(STATUS "CLANG_VERSION: ${CLANG_VERSION}")
 endif()

include/flatcc/flatcc_identifier.h

@@ -66,19 +66,41 @@ static inline void flatbuffers_identifier_from_type_hash(flatbuffers_thash_t typ
     out_identifier[0] = (char)(type_hash & 0xff);
     type_hash >>= 8;
     out_identifier[1] = (char)(type_hash & 0xff);
+
+#if FLATBUFFERS_THASH_WIDTH > 16
     type_hash >>= 8;
     out_identifier[2] = (char)(type_hash & 0xff);
     type_hash >>= 8;
     out_identifier[3] = (char)(type_hash & 0xff);
+#endif
+
+#if FLATBUFFERS_THASH_WIDTH > 32
+    type_hash >>= 8;
+    out_identifier[4] = (char)(type_hash & 0xff);
+    type_hash >>= 8;
+    out_identifier[5] = (char)(type_hash & 0xff);
+    type_hash >>= 8;
+    out_identifier[6] = (char)(type_hash & 0xff);
+    type_hash >>= 8;
+    out_identifier[7] = (char)(type_hash & 0xff);
+#endif
 }
 
 /* Native integer encoding of file identifier. */
 static inline flatbuffers_thash_t flatbuffers_type_hash_from_identifier(const flatbuffers_fid_t identifier)
 {
     uint8_t *p = (uint8_t *)identifier;
 
-    return identifier ?
-        (uint32_t)p[0] + (((uint32_t)p[1]) << 8) + (((uint32_t)p[2]) << 16) + (((uint32_t)p[3]) << 24) : 0;
+    return !identifier ? 0 :
+
+#if FLATBUFFERS_THASH_WIDTH == 16
+        (uint16_t)p[0] + (((uint16_t)p[1]) << 8);
+#elif FLATBUFFERS_THASH_WIDTH == 32
+        (uint32_t)p[0] + (((uint32_t)p[1]) << 8) + (((uint32_t)p[2]) << 16) + (((uint32_t)p[3]) << 24);
+#elif FLATBUFFERS_THASH_WIDTH == 64
+        (uint64_t)p[0] + (((uint64_t)p[1]) << 8) + (((uint64_t)p[2]) << 16) + (((uint64_t)p[3]) << 24) +
+        (((uint64_t)p[4]) << 32) + (((uint64_t)p[5]) << 40) + (((uint64_t)p[6]) << 48) + (((uint64_t)p[7]) << 56);
+#endif
 }
 
 /*
@@ -91,14 +113,31 @@ static inline flatbuffers_thash_t flatbuffers_type_hash_from_string(const char *
     flatbuffers_thash_t h = 0;
     const uint8_t *p = (const uint8_t *)identifier;
 
+    if (!p) return 0;
+
     if (!p[0]) return h;
     h += ((flatbuffers_thash_t)p[0]);
     if (!p[1]) return h;
     h += ((flatbuffers_thash_t)p[1]) << 8;
+
+#if FLATBUFFERS_THASH_WIDTH > 16
     if (!p[2]) return h;
     h += ((flatbuffers_thash_t)p[2]) << 16;
-    /* No need to test for termination here. */
+    if (!p[3]) return h;
     h += ((flatbuffers_thash_t)p[3]) << 24;
+#endif
+
+#if FLATBUFFERS_THASH_WIDTH > 32
+    if (!p[4]) return h;
+    h += ((flatbuffers_thash_t)p[4]) << 32;
+    if (!p[5]) return h;
+    h += ((flatbuffers_thash_t)p[5]) << 40;
+    if (!p[6]) return h;
+    h += ((flatbuffers_thash_t)p[6]) << 48;
+    if (!p[7]) return h;
+    h += ((flatbuffers_thash_t)p[7]) << 56;
+#endif
+
     return h;
 }
 
@@ -125,21 +164,31 @@ static inline void flatbuffers_identifier_from_name(const char *name, flatbuffer
  * additional information and just complicates matters. Furthermore, the
  * unmodified type hash has the benefit that it can seed a child namespace.
  */
-static inline uint32_t flatbuffers_disperse_type_hash(flatbuffers_thash_t type_hash)
+static inline flatbuffers_thash_t flatbuffers_disperse_type_hash(flatbuffers_thash_t type_hash)
 {
+    flatbuffers_thash_t x = type_hash;
+
+#if FLATBUFFERS_THASH_WIDTH == 32
     /* http://stackoverflow.com/a/12996028 */
-    uint32_t x = type_hash;
 
     x = ((x >> 16) ^ x) * UINT32_C(0x45d9f3b);
     x = ((x >> 16) ^ x) * UINT32_C(0x45d9f3b);
     x = ((x >> 16) ^ x);
+#elif FLATBUFFERS_THASH_WIDTH == 64
+    /* http://stackoverflow.com/a/12996028 */
+
+    x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9);
+    x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb);
+    x = x ^ (x >> 31);
+#endif
+
     return x;
 }
 
 
 /* We have hardcoded assumptions about identifier size. */
-static_assert(sizeof(flatbuffers_fid_t) == 4, "unexpected file identifier size");
-static_assert(sizeof(flatbuffers_thash_t) == 4, "unexpected type hash size");
+//static_assert(sizeof(flatbuffers_fid_t) == 4, "unexpected file identifier size");
+//static_assert(sizeof(flatbuffers_thash_t) == 4, "unexpected type hash size");
 
 #ifdef __cplusplus
 }

include/flatcc/flatcc_json_parser.h

@@ -168,7 +168,7 @@ static inline const char *flatcc_json_parser_string_end(flatcc_json_parser_t *ct
  * and raise errors according to overflow/underflow runtime flags. Zero
  * and truncate as needed. A trailing zero is not inserted if the input
  * is at least the same length as the char array.
- * 
+ *
  * Runtime flags: `skip_array_overflow`, `pad_array_underflow`.
  */
 const char *flatcc_json_parser_char_array(flatcc_json_parser_t *ctx,
@@ -875,15 +875,15 @@ const char *flatcc_json_parser_union_type_vector(flatcc_json_parser_t *ctx,
  * `flags` default to 0. See also `flatcc_json_parser_flags`.
  */
 int flatcc_json_parser_table_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx,
-        const char *buf, size_t bufsiz, int flags, const char *fid,
+        const char *buf, size_t bufsiz, int flags, const flatbuffers_fid_t fid,
         flatcc_json_parser_table_f *parser);
 
 /*
  * Similar to `flatcc_json_parser_table_as_root` but parses a struct as
  * root.
  */
 int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx,
-        const char *buf, size_t bufsiz, int flags, const char *fid,
+        const char *buf, size_t bufsiz, int flags, const flatbuffers_fid_t fid,
         flatcc_json_parser_struct_f *parser);
 
 #include "flatcc/portable/pdiagnostic_pop.h"

include/flatcc/flatcc_json_printer.h

@@ -679,11 +679,11 @@ void flatcc_json_printer_uint8_vector_base64_field(flatcc_json_printer_t *ctx,
  * require aligned memory addresses (as always for flatbuffers).
  */
 int flatcc_json_printer_table_as_root(flatcc_json_printer_t *ctx,
-        const void *buf, size_t bufsiz, const char *fid,
+        const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         flatcc_json_printer_table_f *pf);
 
 int flatcc_json_printer_struct_as_root(flatcc_json_printer_t *ctx,
-        const void *buf, size_t bufsiz, const char *fid,
+        const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         flatcc_json_printer_struct_f *pf);
 
 /*
@@ -756,13 +756,13 @@ void flatcc_json_printer_union_vector_field(flatcc_json_printer_t *ctx,
 void flatcc_json_printer_struct_as_nested_root(flatcc_json_printer_t *ctx,
         flatcc_json_printer_table_descriptor_t *td,
         int id, const char *name, size_t len,
-        const char *fid,
+        const flatbuffers_fid_t fid,
         flatcc_json_printer_struct_f *pf);
 
 void flatcc_json_printer_table_as_nested_root(flatcc_json_printer_t *ctx,
         flatcc_json_printer_table_descriptor_t *td,
         int id, const char *name, size_t len,
-        const char *fid,
+        const flatbuffers_fid_t fid,
         flatcc_json_printer_table_f pf);
 
 void flatcc_json_printer_union_field(flatcc_json_printer_t *ctx,

include/flatcc/flatcc_types.h

@@ -11,6 +11,8 @@ extern "C" {
 #include <stdint.h>
 #endif
 
+#include "../../config/config.h"
+
 /*
  * This should match generated type declaratios in
  * `flatbuffers_common_reader.h` (might have different name prefix).
@@ -42,52 +44,95 @@ extern "C" {
 #define flatbuffers_utype_t_defined
 #define flatbuffers_bool_t_defined
 #define flatbuffers_thash_t_defined
-#define flatbuffers_fid_t_defined
-
-/* uoffset_t is also used for vector and string headers. */
-#define FLATBUFFERS_UOFFSET_MAX UINT32_MAX
-#define FLATBUFFERS_SOFFSET_MAX INT32_MAX
-#define FLATBUFFERS_SOFFSET_MIN INT32_MIN
-#define FLATBUFFERS_VOFFSET_MAX UINT16_MAX
-#define FLATBUFFERS_UTYPE_MAX UINT8_MAX
-/* Well - the max of the underlying type. */
-#define FLATBUFFERS_BOOL_MAX UINT8_MAX
-#define FLATBUFFERS_THASH_MAX UINT32_MAX
 
 #define FLATBUFFERS_ID_MAX (FLATBUFFERS_VOFFSET_MAX / sizeof(flatbuffers_voffset_t) - 3)
 /* Vectors of empty structs can yield div by zero, so we must guard against this. */
 #define FLATBUFFERS_COUNT_MAX(elem_size) (FLATBUFFERS_UOFFSET_MAX/((elem_size) == 0 ? 1 : (elem_size)))
 
-#define FLATBUFFERS_UOFFSET_WIDTH 32
-#define FLATBUFFERS_COUNT_WIDTH 32
-#define FLATBUFFERS_SOFFSET_WIDTH 32
-#define FLATBUFFERS_VOFFSET_WIDTH 16
-#define FLATBUFFERS_UTYPE_WIDTH 8
-#define FLATBUFFERS_BOOL_WIDTH 8
-#define FLATBUFFERS_THASH_WIDTH 32
 
 #define FLATBUFFERS_TRUE 1
 #define FLATBUFFERS_FALSE 0
 
 #define FLATBUFFERS_PROTOCOL_IS_LE 1
 #define FLATBUFFERS_PROTOCOL_IS_BE 0
 
+/* uoffset_t is also used for vector and string headers. */
+#if FLATCC_OFFSET_SIZE == 4
 typedef uint32_t flatbuffers_uoffset_t;
 typedef int32_t flatbuffers_soffset_t;
+typedef uint32_t flatbuffers_thash_t;
+#define FLATBUFFERS_UOFFSET_MAX UINT32_MAX
+#define FLATBUFFERS_SOFFSET_MAX INT32_MAX
+#define FLATBUFFERS_SOFFSET_MIN INT32_MIN
+#define FLATBUFFERS_THASH_MAX UINT32_MAX
+#define FLATBUFFERS_UOFFSET_WIDTH 32
+#define FLATBUFFERS_SOFFSET_WIDTH 32
+#define FLATBUFFERS_THASH_WIDTH 32
+#elif FLATCC_OFFSET_SIZE == 8
+typedef uint64_t flatbuffers_uoffset_t;
+typedef int64_t flatbuffers_soffset_t;
+typedef uint64_t flatbuffers_thash_t;
+#define FLATBUFFERS_UOFFSET_MAX UINT64_MAX
+#define FLATBUFFERS_SOFFSET_MAX INT64_MAX
+#define FLATBUFFERS_SOFFSET_MIN INT64_MIN
+#define FLATBUFFERS_THASH_MAX UINT64_MAX
+#define FLATBUFFERS_UOFFSET_WIDTH 64
+#define FLATBUFFERS_SOFFSET_WIDTH 64
+#define FLATBUFFERS_THASH_WIDTH 64
+#elif FLATCC_OFFSET_SIZE == 2
+typedef uint16_t flatbuffers_uoffset_t;
+typedef int16_t flatbuffers_soffset_t;
+typedef uint16_t flatbuffers_thash_t;
+#define FLATBUFFERS_UOFFSET_MAX UINT16_MAX
+#define FLATBUFFERS_SOFFSET_MAX INT16_MAX
+#define FLATBUFFERS_SOFFSET_MIN INT16_MIN
+#define FLATBUFFERS_THASH_MAX UINT16_MAX
+#define FLATBUFFERS_UOFFSET_WIDTH 16
+#define FLATBUFFERS_SOFFSET_WIDTH 16
+#define FLATBUFFERS_THASH_WIDTH 16
+#else
+#error FLATCC_OFFSET_SIZE must be defined.
+#endif
+
+#if FLATCC_VOFFSET_SIZE == 2
 typedef uint16_t flatbuffers_voffset_t;
+#define FLATBUFFERS_VOFFSET_MAX UINT16_MAX
+#define FLATBUFFERS_VOFFSET_WIDTH 16
+#elif FLATCC_VOFFSET_SIZE == 8
+typedef uint64_t flatbuffers_voffset_t;
+#define FLATBUFFERS_VOFFSET_MAX UINT64_MAX
+#define FLATBUFFERS_VOFFSET_WIDTH 64
+#elif FLATCC_VOFFSET_SIZE == 4
+typedef uint32_t flatbuffers_voffset_t;
+#define FLATBUFFERS_VOFFSET_MAX UINT32_MAX
+#define FLATBUFFERS_VOFFSET_WIDTH 32
+#else
+#error FLATCC_VOFFSET_SIZE must be defined.
+#endif
+
+#define FLATBUFFERS_UTYPE_MAX UINT8_MAX
+#define FLATBUFFERS_UTYPE_WIDTH 8
 typedef uint8_t flatbuffers_utype_t;
+
+/* Well - the max of the underlying type. */
+#define FLATBUFFERS_BOOL_MAX UINT8_MAX
+#define FLATBUFFERS_BOOL_WIDTH 8
 typedef uint8_t flatbuffers_bool_t;
-typedef uint32_t flatbuffers_thash_t;
+
 /* Public facing type operations. */
 typedef flatbuffers_utype_t flatbuffers_union_type_t;
 
 static const flatbuffers_bool_t flatbuffers_true = FLATBUFFERS_TRUE;
 static const flatbuffers_bool_t flatbuffers_false = FLATBUFFERS_FALSE;
 
-#define FLATBUFFERS_IDENTIFIER_SIZE (FLATBUFFERS_THASH_WIDTH / 8)
+#ifndef flatbuffers_fid_t_defined
+#define FLATBUFFERS_IDENTIFIER_SIZE FLATCC_OFFSET_SIZE
 
 typedef char flatbuffers_fid_t[FLATBUFFERS_IDENTIFIER_SIZE];
 
+#define flatbuffers_fid_t_defined
+#endif
+
 #endif /* flatbuffers_types_defined */
 
 #ifdef __cplusplus

include/flatcc/flatcc_verifier.h

@@ -164,13 +164,13 @@ typedef int flatcc_union_verifier_f(flatcc_union_verifier_descriptor_t *ud);
  * require aligned memory addresses. The buffer pointers alignment is
  * not significant to internal verification of the buffer.
  */
-int flatcc_verify_struct_as_root(const void *buf, size_t bufsiz, const char *fid,
+int flatcc_verify_struct_as_root(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         size_t size, uint16_t align);
 
 int flatcc_verify_struct_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash,
         size_t size, uint16_t align);
 
-int flatcc_verify_table_as_root(const void *buf, size_t bufsiz, const char *fid,
+int flatcc_verify_table_as_root(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         flatcc_table_verifier_f *root_tvf);
 
 int flatcc_verify_table_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash,
@@ -179,7 +179,7 @@ int flatcc_verify_table_as_typed_root(const void *buf, size_t bufsiz, flatbuffer
  * The buffer header is verified by any of the `_as_root` verifiers, but
  * this function may be used as a quick sanity check.
  */
-int flatcc_verify_buffer_header(const void *buf, size_t bufsiz, const char *fid);
+int flatcc_verify_buffer_header(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid);
 
 int flatcc_verify_typed_buffer_header(const void *buf, size_t bufsiz, flatbuffers_thash_t type_hash);
 
@@ -204,10 +204,10 @@ int flatcc_verify_table_vector_field(flatcc_table_verifier_descriptor_t *td,
     flatbuffers_voffset_t id, int required, flatcc_table_verifier_f tvf);
 /* Table verifiers pass 0 as fid. */
 int flatcc_verify_struct_as_nested_root(flatcc_table_verifier_descriptor_t *td,
-        flatbuffers_voffset_t id, int required, const char *fid,
+        flatbuffers_voffset_t id, int required, const flatbuffers_fid_t fid,
         size_t size, uint16_t align);
 int flatcc_verify_table_as_nested_root(flatcc_table_verifier_descriptor_t *td,
-        flatbuffers_voffset_t id, int required, const char *fid,
+        flatbuffers_voffset_t id, int required, const flatbuffers_fid_t fid,
         uint16_t align, flatcc_table_verifier_f tvf);
 
 /*