support variable offset sizes for real

Author: TYoungSL

include/flatcc/flatcc_identifier.h

@@ -66,19 +66,41 @@ static inline void flatbuffers_identifier_from_type_hash(flatbuffers_thash_t typ
     out_identifier[0] = (char)(type_hash & 0xff);
     type_hash >>= 8;
     out_identifier[1] = (char)(type_hash & 0xff);
+
+#if FLATBUFFERS_THASH_WIDTH > 16
     type_hash >>= 8;
     out_identifier[2] = (char)(type_hash & 0xff);
     type_hash >>= 8;
     out_identifier[3] = (char)(type_hash & 0xff);
+#endif
+
+#if FLATBUFFERS_THASH_WIDTH > 32
+    type_hash >>= 8;
+    out_identifier[4] = (char)(type_hash & 0xff);
+    type_hash >>= 8;
+    out_identifier[5] = (char)(type_hash & 0xff);
+    type_hash >>= 8;
+    out_identifier[6] = (char)(type_hash & 0xff);
+    type_hash >>= 8;
+    out_identifier[7] = (char)(type_hash & 0xff);
+#endif
 }
 
 /* Native integer encoding of file identifier. */
 static inline flatbuffers_thash_t flatbuffers_type_hash_from_identifier(const flatbuffers_fid_t identifier)
 {
     uint8_t *p = (uint8_t *)identifier;
 
-    return identifier ?
-        (uint32_t)p[0] + (((uint32_t)p[1]) << 8) + (((uint32_t)p[2]) << 16) + (((uint32_t)p[3]) << 24) : 0;
+    return !identifier ? 0 :
+
+#if FLATBUFFERS_THASH_WIDTH == 16
+        (uint16_t)p[0] + (((uint16_t)p[1]) << 8);
+#elif FLATBUFFERS_THASH_WIDTH == 32
+        (uint32_t)p[0] + (((uint32_t)p[1]) << 8) + (((uint32_t)p[2]) << 16) + (((uint32_t)p[3]) << 24);
+#elif FLATBUFFERS_THASH_WIDTH == 64
+        (uint64_t)p[0] + (((uint64_t)p[1]) << 8) + (((uint64_t)p[2]) << 16) + (((uint64_t)p[3]) << 24) +
+        (((uint64_t)p[4]) << 32) + (((uint64_t)p[5]) << 40) + (((uint64_t)p[6]) << 48) + (((uint64_t)p[7]) << 56);
+#endif
 }
 
 /*
@@ -95,10 +117,25 @@ static inline flatbuffers_thash_t flatbuffers_type_hash_from_string(const char *
     h += ((flatbuffers_thash_t)p[0]);
     if (!p[1]) return h;
     h += ((flatbuffers_thash_t)p[1]) << 8;
+
+#if FLATBUFFERS_THASH_WIDTH > 16
     if (!p[2]) return h;
     h += ((flatbuffers_thash_t)p[2]) << 16;
-    /* No need to test for termination here. */
+    if (!p[3]) return h;
     h += ((flatbuffers_thash_t)p[3]) << 24;
+#endif
+
+#if FLATBUFFERS_THASH_WIDTH > 32
+    if (!p[4]) return h;
+    h += ((flatbuffers_thash_t)p[4]) << 32;
+    if (!p[5]) return h;
+    h += ((flatbuffers_thash_t)p[5]) << 40;
+    if (!p[6]) return h;
+    h += ((flatbuffers_thash_t)p[6]) << 48;
+    if (!p[7]) return h;
+    h += ((flatbuffers_thash_t)p[7]) << 56;
+#endif
+
     return h;
 }
 
@@ -125,21 +162,31 @@ static inline void flatbuffers_identifier_from_name(const char *name, flatbuffer
  * additional information and just complicates matters. Furthermore, the
  * unmodified type hash has the benefit that it can seed a child namespace.
  */
-static inline uint32_t flatbuffers_disperse_type_hash(flatbuffers_thash_t type_hash)
+static inline flatbuffers_thash_t flatbuffers_disperse_type_hash(flatbuffers_thash_t type_hash)
 {
+    flatbuffers_thash_t x = type_hash;
+
+#if FLATBUFFERS_THASH_WIDTH == 32
     /* http://stackoverflow.com/a/12996028 */
-    uint32_t x = type_hash;
 
     x = ((x >> 16) ^ x) * UINT32_C(0x45d9f3b);
     x = ((x >> 16) ^ x) * UINT32_C(0x45d9f3b);
     x = ((x >> 16) ^ x);
+#elif FLATBUFFERS_THASH_WIDTH == 64
+    /* http://stackoverflow.com/a/12996028 */
+
+    x = (x ^ (x >> 30)) * UINT64_C(0xbf58476d1ce4e5b9);
+    x = (x ^ (x >> 27)) * UINT64_C(0x94d049bb133111eb);
+    x = x ^ (x >> 31);
+#endif
+
     return x;
 }
 
 
 /* We have hardcoded assumptions about identifier size. */
-static_assert(sizeof(flatbuffers_fid_t) == 4, "unexpected file identifier size");
-static_assert(sizeof(flatbuffers_thash_t) == 4, "unexpected type hash size");
+//static_assert(sizeof(flatbuffers_fid_t) == 4, "unexpected file identifier size");
+//static_assert(sizeof(flatbuffers_thash_t) == 4, "unexpected type hash size");
 
 #ifdef __cplusplus
 }

include/flatcc/flatcc_json_parser.h

@@ -168,7 +168,7 @@ static inline const char *flatcc_json_parser_string_end(flatcc_json_parser_t *ct
  * and raise errors according to overflow/underflow runtime flags. Zero
  * and truncate as needed. A trailing zero is not inserted if the input
  * is at least the same length as the char array.
- * 
+ *
  * Runtime flags: `skip_array_overflow`, `pad_array_underflow`.
  */
 const char *flatcc_json_parser_char_array(flatcc_json_parser_t *ctx,
@@ -875,15 +875,15 @@ const char *flatcc_json_parser_union_type_vector(flatcc_json_parser_t *ctx,
  * `flags` default to 0. See also `flatcc_json_parser_flags`.
  */
 int flatcc_json_parser_table_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx,
-        const char *buf, size_t bufsiz, int flags, const char *fid,
+        const char *buf, size_t bufsiz, int flags, const flatbuffers_fid_t fid,
         flatcc_json_parser_table_f *parser);
 
 /*
  * Similar to `flatcc_json_parser_table_as_root` but parses a struct as
  * root.
  */
 int flatcc_json_parser_struct_as_root(flatcc_builder_t *B, flatcc_json_parser_t *ctx,
-        const char *buf, size_t bufsiz, int flags, const char *fid,
+        const char *buf, size_t bufsiz, int flags, const flatbuffers_fid_t fid,
         flatcc_json_parser_struct_f *parser);
 
 #include "flatcc/portable/pdiagnostic_pop.h"

include/flatcc/flatcc_json_printer.h

@@ -679,11 +679,11 @@ void flatcc_json_printer_uint8_vector_base64_field(flatcc_json_printer_t *ctx,
  * require aligned memory addresses (as always for flatbuffers).
  */
 int flatcc_json_printer_table_as_root(flatcc_json_printer_t *ctx,
-        const void *buf, size_t bufsiz, const char *fid,
+        const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         flatcc_json_printer_table_f *pf);
 
 int flatcc_json_printer_struct_as_root(flatcc_json_printer_t *ctx,
-        const void *buf, size_t bufsiz, const char *fid,
+        const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         flatcc_json_printer_struct_f *pf);
 
 /*
@@ -756,13 +756,13 @@ void flatcc_json_printer_union_vector_field(flatcc_json_printer_t *ctx,
 void flatcc_json_printer_struct_as_nested_root(flatcc_json_printer_t *ctx,
         flatcc_json_printer_table_descriptor_t *td,
         int id, const char *name, size_t len,
-        const char *fid,
+        const flatbuffers_fid_t fid,
         flatcc_json_printer_struct_f *pf);
 
 void flatcc_json_printer_table_as_nested_root(flatcc_json_printer_t *ctx,
         flatcc_json_printer_table_descriptor_t *td,
         int id, const char *name, size_t len,
-        const char *fid,
+        const flatbuffers_fid_t fid,
         flatcc_json_printer_table_f pf);
 
 void flatcc_json_printer_union_field(flatcc_json_printer_t *ctx,

include/flatcc/flatcc_types.h

@@ -11,6 +11,8 @@ extern "C" {
 #include <stdint.h>
 #endif
 
+#include "../../config/config.h"
+
 /*
  * This should match generated type declaratios in
  * `flatbuffers_common_reader.h` (might have different name prefix).
@@ -44,40 +46,80 @@ extern "C" {
 #define flatbuffers_thash_t_defined
 #define flatbuffers_fid_t_defined
 
-/* uoffset_t is also used for vector and string headers. */
-#define FLATBUFFERS_UOFFSET_MAX UINT32_MAX
-#define FLATBUFFERS_SOFFSET_MAX INT32_MAX
-#define FLATBUFFERS_SOFFSET_MIN INT32_MIN
-#define FLATBUFFERS_VOFFSET_MAX UINT16_MAX
-#define FLATBUFFERS_UTYPE_MAX UINT8_MAX
-/* Well - the max of the underlying type. */
-#define FLATBUFFERS_BOOL_MAX UINT8_MAX
-#define FLATBUFFERS_THASH_MAX UINT32_MAX
-
 #define FLATBUFFERS_ID_MAX (FLATBUFFERS_VOFFSET_MAX / sizeof(flatbuffers_voffset_t) - 3)
 /* Vectors of empty structs can yield div by zero, so we must guard against this. */
 #define FLATBUFFERS_COUNT_MAX(elem_size) (FLATBUFFERS_UOFFSET_MAX/((elem_size) == 0 ? 1 : (elem_size)))
 
-#define FLATBUFFERS_UOFFSET_WIDTH 32
-#define FLATBUFFERS_COUNT_WIDTH 32
-#define FLATBUFFERS_SOFFSET_WIDTH 32
-#define FLATBUFFERS_VOFFSET_WIDTH 16
-#define FLATBUFFERS_UTYPE_WIDTH 8
-#define FLATBUFFERS_BOOL_WIDTH 8
-#define FLATBUFFERS_THASH_WIDTH 32
 
 #define FLATBUFFERS_TRUE 1
 #define FLATBUFFERS_FALSE 0
 
 #define FLATBUFFERS_PROTOCOL_IS_LE 1
 #define FLATBUFFERS_PROTOCOL_IS_BE 0
 
+/* uoffset_t is also used for vector and string headers. */
+#if FLATCC_OFFSET_SIZE == 4
 typedef uint32_t flatbuffers_uoffset_t;
 typedef int32_t flatbuffers_soffset_t;
+typedef uint32_t flatbuffers_thash_t;
+#define FLATBUFFERS_UOFFSET_MAX UINT32_MAX
+#define FLATBUFFERS_SOFFSET_MAX INT32_MAX
+#define FLATBUFFERS_SOFFSET_MIN INT32_MIN
+#define FLATBUFFERS_THASH_MAX UINT32_MAX
+#define FLATBUFFERS_UOFFSET_WIDTH 32
+#define FLATBUFFERS_SOFFSET_WIDTH 32
+#define FLATBUFFERS_THASH_WIDTH 32
+#elif FLATCC_OFFSET_SIZE == 8
+typedef uint64_t flatbuffers_uoffset_t;
+typedef int64_t flatbuffers_soffset_t;
+typedef uint64_t flatbuffers_thash_t;
+#define FLATBUFFERS_UOFFSET_MAX UINT64_MAX
+#define FLATBUFFERS_SOFFSET_MAX INT64_MAX
+#define FLATBUFFERS_SOFFSET_MIN INT64_MIN
+#define FLATBUFFERS_THASH_MAX UINT64_MAX
+#define FLATBUFFERS_UOFFSET_WIDTH 64
+#define FLATBUFFERS_SOFFSET_WIDTH 64
+#define FLATBUFFERS_THASH_WIDTH 64
+#elif FLATCC_OFFSET_SIZE == 2
+typedef uint16_t flatbuffers_uoffset_t;
+typedef int16_t flatbuffers_soffset_t;
+typedef uint16_t flatbuffers_thash_t;
+#define FLATBUFFERS_UOFFSET_MAX UINT16_MAX
+#define FLATBUFFERS_SOFFSET_MAX INT16_MAX
+#define FLATBUFFERS_SOFFSET_MIN INT16_MIN
+#define FLATBUFFERS_THASH_MAX UINT16_MAX
+#define FLATBUFFERS_UOFFSET_WIDTH 16
+#define FLATBUFFERS_SOFFSET_WIDTH 16
+#define FLATBUFFERS_THASH_WIDTH 16
+#else
+#error FLATCC_OFFSET_SIZE must be defined.
+#endif
+
+#if FLATCC_VOFFSET_SIZE == 2
 typedef uint16_t flatbuffers_voffset_t;
+#define FLATBUFFERS_VOFFSET_MAX UINT16_MAX
+#define FLATBUFFERS_VOFFSET_WIDTH 16
+#elif FLATCC_VOFFSET_SIZE == 8
+typedef uint64_t flatbuffers_voffset_t;
+#define FLATBUFFERS_VOFFSET_MAX UINT64_MAX
+#define FLATBUFFERS_VOFFSET_WIDTH 64
+#elif FLATCC_VOFFSET_SIZE == 4
+typedef uint32_t flatbuffers_voffset_t;
+#define FLATBUFFERS_VOFFSET_MAX UINT32_MAX
+#define FLATBUFFERS_VOFFSET_WIDTH 32
+#else
+#error FLATCC_VOFFSET_SIZE must be defined.
+#endif
+
+#define FLATBUFFERS_UTYPE_MAX UINT8_MAX
+#define FLATBUFFERS_UTYPE_WIDTH 8
 typedef uint8_t flatbuffers_utype_t;
+
+/* Well - the max of the underlying type. */
+#define FLATBUFFERS_BOOL_MAX UINT8_MAX
+#define FLATBUFFERS_BOOL_WIDTH 8
 typedef uint8_t flatbuffers_bool_t;
-typedef uint32_t flatbuffers_thash_t;
+
 /* Public facing type operations. */
 typedef flatbuffers_utype_t flatbuffers_union_type_t;
 

include/flatcc/flatcc_verifier.h

@@ -164,13 +164,13 @@ typedef int flatcc_union_verifier_f(flatcc_union_verifier_descriptor_t *ud);
  * require aligned memory addresses. The buffer pointers alignment is
  * not significant to internal verification of the buffer.
  */
-int flatcc_verify_struct_as_root(const void *buf, size_t bufsiz, const char *fid,
+int flatcc_verify_struct_as_root(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         size_t size, uint16_t align);
 
 int flatcc_verify_struct_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash,
         size_t size, uint16_t align);
 
-int flatcc_verify_table_as_root(const void *buf, size_t bufsiz, const char *fid,
+int flatcc_verify_table_as_root(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid,
         flatcc_table_verifier_f *root_tvf);
 
 int flatcc_verify_table_as_typed_root(const void *buf, size_t bufsiz, flatbuffers_thash_t thash,
@@ -179,7 +179,7 @@ int flatcc_verify_table_as_typed_root(const void *buf, size_t bufsiz, flatbuffer
  * The buffer header is verified by any of the `_as_root` verifiers, but
  * this function may be used as a quick sanity check.
  */
-int flatcc_verify_buffer_header(const void *buf, size_t bufsiz, const char *fid);
+int flatcc_verify_buffer_header(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid);
 
 int flatcc_verify_typed_buffer_header(const void *buf, size_t bufsiz, flatbuffers_thash_t type_hash);
 
@@ -204,10 +204,10 @@ int flatcc_verify_table_vector_field(flatcc_table_verifier_descriptor_t *td,
     flatbuffers_voffset_t id, int required, flatcc_table_verifier_f tvf);
 /* Table verifiers pass 0 as fid. */
 int flatcc_verify_struct_as_nested_root(flatcc_table_verifier_descriptor_t *td,
-        flatbuffers_voffset_t id, int required, const char *fid,
+        flatbuffers_voffset_t id, int required, const flatbuffers_fid_t fid,
         size_t size, uint16_t align);
 int flatcc_verify_table_as_nested_root(flatcc_table_verifier_descriptor_t *td,
-        flatbuffers_voffset_t id, int required, const char *fid,
+        flatbuffers_voffset_t id, int required, const flatbuffers_fid_t fid,
         uint16_t align, flatcc_table_verifier_f tvf);
 
 /*

include/flatcc/reflection/flatbuffers_common_reader.h

@@ -529,7 +529,7 @@ __flatbuffers_define_scan_by_scalar_field(N, NK, T)
 static inline T N ## _ ## NK ## _get(N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; }\
 static inline T N ## _ ## NK (N ## _struct_t t__tmp) { return t__tmp ? &(t__tmp->NK) : 0; }
 /* If fid is null, the function returns true without testing as buffer is not expected to have any id. */
-static inline int flatbuffers_has_identifier(const void *buffer, const char *fid)
+static inline int flatbuffers_has_identifier(const void *buffer, const flatbuffers_fid_t fid)
 { flatbuffers_thash_t id, id2 = 0; if (fid == 0) { return 1; };
   id2 = flatbuffers_type_hash_from_string(fid);
   id = __flatbuffers_thash_read_from_pe(((flatbuffers_uoffset_t *)buffer) + 1);

include/flatcc/reflection/reflection_verifier.h

@@ -39,7 +39,7 @@ static inline int reflection_Type_verify_as_typed_root(const void *buf, size_t b
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_Type_type_identifier, &reflection_Type_verify_table);
 }
 
-static inline int reflection_Type_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_Type_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Type_verify_table);
 }
@@ -67,7 +67,7 @@ static inline int reflection_KeyValue_verify_as_typed_root(const void *buf, size
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_KeyValue_type_identifier, &reflection_KeyValue_verify_table);
 }
 
-static inline int reflection_KeyValue_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_KeyValue_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_KeyValue_verify_table);
 }
@@ -98,7 +98,7 @@ static inline int reflection_EnumVal_verify_as_typed_root(const void *buf, size_
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_EnumVal_type_identifier, &reflection_EnumVal_verify_table);
 }
 
-static inline int reflection_EnumVal_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_EnumVal_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_EnumVal_verify_table);
 }
@@ -130,7 +130,7 @@ static inline int reflection_Enum_verify_as_typed_root(const void *buf, size_t b
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_Enum_type_identifier, &reflection_Enum_verify_table);
 }
 
-static inline int reflection_Enum_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_Enum_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Enum_verify_table);
 }
@@ -168,7 +168,7 @@ static inline int reflection_Field_verify_as_typed_root(const void *buf, size_t
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_Field_type_identifier, &reflection_Field_verify_table);
 }
 
-static inline int reflection_Field_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_Field_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Field_verify_table);
 }
@@ -201,7 +201,7 @@ static inline int reflection_Object_verify_as_typed_root(const void *buf, size_t
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_Object_type_identifier, &reflection_Object_verify_table);
 }
 
-static inline int reflection_Object_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_Object_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Object_verify_table);
 }
@@ -232,7 +232,7 @@ static inline int reflection_RPCCall_verify_as_typed_root(const void *buf, size_
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_RPCCall_type_identifier, &reflection_RPCCall_verify_table);
 }
 
-static inline int reflection_RPCCall_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_RPCCall_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_RPCCall_verify_table);
 }
@@ -262,7 +262,7 @@ static inline int reflection_Service_verify_as_typed_root(const void *buf, size_
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_Service_type_identifier, &reflection_Service_verify_table);
 }
 
-static inline int reflection_Service_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_Service_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Service_verify_table);
 }
@@ -294,7 +294,7 @@ static inline int reflection_Schema_verify_as_typed_root(const void *buf, size_t
     return flatcc_verify_table_as_root(buf, bufsiz, reflection_Schema_type_identifier, &reflection_Schema_verify_table);
 }
 
-static inline int reflection_Schema_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const char *fid)
+static inline int reflection_Schema_verify_as_root_with_identifier(const void *buf, size_t bufsiz, const flatbuffers_fid_t fid)
 {
     return flatcc_verify_table_as_root(buf, bufsiz, fid, &reflection_Schema_verify_table);
 }