Metadata table v2

Makefile

@@ -5,7 +5,7 @@ CCAN_HDRS := ccan/ccan/utf8/utf8.h ccan/ccan/container_of/container_of.h ccan/cc
 HEADERS = deps/lmdb/lmdb.h deps/secp256k1/include/secp256k1.h src/nostrdb.h src/cursor.h src/hex.h src/jsmn.h src/config.h src/random.h src/memchr.h src/cpu.h src/nostr_bech32.h src/block.h src/str_block.h src/print_util.h $(C_BINDINGS) $(CCAN_HDRS) $(BOLT11_HDRS)
 FLATCC_SRCS=deps/flatcc/src/runtime/json_parser.c deps/flatcc/src/runtime/verifier.c deps/flatcc/src/runtime/builder.c deps/flatcc/src/runtime/emitter.c deps/flatcc/src/runtime/refmap.c
 BOLT11_SRCS = src/bolt11/bolt11.c src/bolt11/bech32.c src/bolt11/amount.c src/bolt11/hash_u5.c
-SRCS = src/nostrdb.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c $(BOLT11_SRCS) $(FLATCC_SRCS) $(CCAN_SRCS)
+SRCS = src/nostrdb.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c src/binmoji.c src/metadata.c $(BOLT11_SRCS) $(FLATCC_SRCS) $(CCAN_SRCS)
 LDS = $(OBJS) $(ARS) 
 OBJS = $(SRCS:.c=.o)
 DEPS = $(OBJS) $(HEADERS) $(ARS)

TODO

@@ -0,0 +1 @@
+test write +metadata

docs/metadata.md

@@ -0,0 +1,95 @@
+
+# Note Metadata
+
+nostrdb supports a flexible metadata system which allows you to store additional information about a nostr note.
+
+metadata is stored as a sorted list of TVs (tag, value). The lengths are a fixed size but can reference a data table.
+
+The type follows the "it's ok to be odd" rule. odd tags are opaque, user defined types that can store data of any kind.
+
+## Binary format
+
+The binary format starts with a header containing the number of metadata entries. The count is followed by a sorted, aligned list of these entries:
+
+```c
+
+// 16 bytes
+struct ndb_note_meta {
+	// 4 bytes
+	uint8_t version;
+	uint8_t padding;
+	uint16_t count;
+
+	// 4 bytes
+	uint32_t data_table_size;
+
+	// 8 bytes
+	uint64_t flags;
+};
+
+// 16 bytes
+// 16 bytes
+struct ndb_note_meta_entry {
+	// 4 byte entry header
+	uint16_t type;
+	uint16_t flags;
+
+	// additional 4 bytes of aux storage for payloads that are >8 bytes
+	//
+	// for reactions types, this is used for counts
+	// normally this would have been padding but we make use of it
+	// in our manually packed structure
+	union {
+		uint32_t value;
+
+		/* if this is a thread root, this counts the total replies in the thread */
+		uint32_t total_reactions;
+	} aux;
+
+	// 8 byte metadata payload
+	union {
+		uint64_t value;
+
+		struct {
+			uint32_t offset;
+			uint32_t padding;
+		} offset;
+
+		struct {
+			/* number of direct replies */
+			uint16_t direct_replies;
+			uint16_t quotes;
+
+			/* number of replies in this thread */
+			uint32_t thread_replies;
+		} counts;
+
+		// the reaction binmoji[1] for reaction, count is stored in aux
+		union ndb_reaction_str reaction_str;
+	} payload;
+};
+
+```
+
+The offset is to a chunk of potentially unaligned data:
+
+```
+size : varint
+data : u8[size]
+```
+
+### Rationale
+
+We want the following properties:
+
+* The ability to quickly iterate/skip over different metadata fields.  This is achieved by a fixed table format, without needing to encode/decode like blocks
+* The ability to quickly update metadata fields.  To update flags or counts can be done via race-safe mutation operations. The mutation can be done inplace (memcpy + poke memory address)
+* The table entries can be inplace sorted for binary search lookups on large metadata tables
+
+## Write thread mutation operations
+
+We want to support many common operations:
+
+* Increase reaction count for a specific reaction type
+
+[binmoji]: https://github.com/jb55/binmoji

ndb.c

@@ -28,6 +28,7 @@ static int usage()
 	printf("	print-tag-keys\n");
 	printf("	print-relay-kind-index-keys\n");
 	printf("	print-author-kind-index-keys\n");
+	printf("	print-note-metadata\n");
 	printf("	import <line-delimited json file>\n\n");
 
 	printf("settings\n\n");
@@ -108,6 +109,7 @@ int ndb_print_kind_keys(struct ndb_txn *txn);
 int ndb_print_tag_index(struct ndb_txn *txn);
 int ndb_print_relay_kind_index(struct ndb_txn *txn);
 int ndb_print_author_kind_index(struct ndb_txn *txn);
+int ndb_print_note_metadata(struct ndb_txn *txn);
 
 static void print_note(struct ndb_note *note)
 {
@@ -323,7 +325,32 @@ int main(int argc, char *argv[])
 
 				argv += 2;
 				argc -= 2;
-			} else if (!strcmp(argv[0], "-a") || !strcmp(argv[0], "--author")) {
+			} else if (!strcmp(argv[0], "-q")) {
+				if (current_field != 'q') {
+					if (!ndb_filter_start_tag_field(f, 'q')) {
+						fprintf(stderr, "field already started\n");
+						res = 44;
+						goto cleanup;
+					}
+				}
+				current_field = 'q';
+
+				if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) {
+					fprintf(stderr, "invalid hex id\n");
+					res = 42;
+					goto cleanup;
+				}
+
+				if (!ndb_filter_add_id_element(f, tmp_id)) {
+					fprintf(stderr, "too many event ids\n");
+					res = 43;
+					goto cleanup;
+				}
+
+				argv += 2;
+				argc -= 2;
+			} 
+			else if (!strcmp(argv[0], "-a") || !strcmp(argv[0], "--author")) {
 				if (current_field != NDB_FILTER_AUTHORS) {
 					ndb_filter_end_field(f);
 					ndb_filter_start_field(f, NDB_FILTER_AUTHORS);
@@ -418,6 +445,10 @@ int main(int argc, char *argv[])
 		ndb_begin_query(ndb, &txn);
 		ndb_print_author_kind_index(&txn);
 		ndb_end_query(&txn);
+	} else if (argc == 2 && !strcmp(argv[1], "print-note-metadata")) {
+		ndb_begin_query(ndb, &txn);
+		ndb_print_note_metadata(&txn);
+		ndb_end_query(&txn);
 	} else if (argc == 3 && !strcmp(argv[1], "note-relays")) {
 		struct ndb_note_relay_iterator iter;
 		const char *relay;