migration: migrate full reaction stats for all kind1 notes

Signed-off-by: William Casarin <[email protected]>

Author: jb55

TODO

@@ -1,4 +1 @@
 test write +metadata
-(B) migrate reply stats +metadata
-(B) migrate quote stats +metadata
-(B) migrate reaction stats +metadata

ndb.c

@@ -28,6 +28,7 @@ static int usage()
 	printf("	print-tag-keys\n");
 	printf("	print-relay-kind-index-keys\n");
 	printf("	print-author-kind-index-keys\n");
+	printf("	print-note-metadata\n");
 	printf("	import <line-delimited json file>\n\n");
 
 	printf("settings\n\n");
@@ -108,6 +109,7 @@ int ndb_print_kind_keys(struct ndb_txn *txn);
 int ndb_print_tag_index(struct ndb_txn *txn);
 int ndb_print_relay_kind_index(struct ndb_txn *txn);
 int ndb_print_author_kind_index(struct ndb_txn *txn);
+int ndb_print_note_metadata(struct ndb_txn *txn);
 
 static void print_note(struct ndb_note *note)
 {
@@ -443,6 +445,10 @@ int main(int argc, char *argv[])
 		ndb_begin_query(ndb, &txn);
 		ndb_print_author_kind_index(&txn);
 		ndb_end_query(&txn);
+	} else if (argc == 2 && !strcmp(argv[1], "print-note-metadata")) {
+		ndb_begin_query(ndb, &txn);
+		ndb_print_note_metadata(&txn);
+		ndb_end_query(&txn);
 	} else if (argc == 3 && !strcmp(argv[1], "note-relays")) {
 		struct ndb_note_relay_iterator iter;
 		const char *relay;

src/metadata.c

@@ -221,7 +221,7 @@ uint16_t *ndb_note_meta_entry_type(struct ndb_note_meta_entry *entry)
 }
 
 /* find a metadata entry, optionally matching a payload */
-struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload)
+static struct ndb_note_meta_entry *ndb_note_meta_find_entry_impl(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload, int sorted)
 {
 	struct ndb_note_meta_entry *entries, *entry;
 	int i;
@@ -232,6 +232,8 @@ struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta,
 	entries = ndb_note_meta_entries(meta);
 	assert(((intptr_t)entries - (intptr_t)meta) == 16);
 
+	/* TODO(jb55): do bsearch for large sorted entries */
+
 	for (i = 0; i < meta->count; i++) {
 		entry = &entries[i];
 		assert(((uintptr_t)entry % 8) == 0);
@@ -250,6 +252,22 @@ struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta,
 	return NULL;
 }
 
+struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload)
+{
+	int sorted = 1;
+	return ndb_note_meta_find_entry_impl(meta, type, payload, sorted);
+}
+
+struct ndb_note_meta_entry *ndb_note_meta_builder_find_entry(
+		struct ndb_note_meta_builder *builder,
+		uint16_t type,
+		uint64_t *payload)
+{
+	/* meta building in progress is not necessarily sorted */
+	int sorted = 0;
+	return ndb_note_meta_find_entry_impl((struct ndb_note_meta *)builder->cursor.start, type, payload, sorted);
+}
+
 void ndb_note_meta_reaction_set(struct ndb_note_meta_entry *entry, uint32_t count, union ndb_reaction_str str)
 {
 	entry->type = NDB_NOTE_META_REACTION;
@@ -389,3 +407,35 @@ union ndb_reaction_str ndb_note_meta_reaction_str(struct ndb_note_meta_entry *en
 {
 	return entry->payload.reaction_str;
 }
+
+void print_note_meta(struct ndb_note_meta *meta)
+{
+	int count, i;
+	struct ndb_note_meta_entry *entries, *entry;
+	union ndb_reaction_str reaction;
+	char strbuf[128];
+
+	count = ndb_note_meta_entries_count(meta);
+	entries = ndb_note_meta_entries(meta);
+
+	for (i = 0; i < count; i++) {
+		entry = &entries[i];
+		switch (entry->type) {
+		case NDB_NOTE_META_REACTION:
+			reaction = ndb_note_meta_reaction_str(entry);
+
+			ndb_reaction_to_str(&reaction, strbuf);
+			printf("%s%d ", strbuf, *ndb_note_meta_reaction_count(entry));
+			break;
+		case NDB_NOTE_META_COUNTS:
+			printf("quotes %d\treplies %d\tall_replies %d\treactions %d\t",
+					*ndb_note_meta_counts_quotes(entry),
+					*ndb_note_meta_counts_direct_replies(entry),
+					*ndb_note_meta_counts_thread_replies(entry),
+					*ndb_note_meta_counts_total_reactions(entry));
+			break;
+		}
+	}
+
+	printf("\n");
+}

src/nostrdb.c

@@ -2178,7 +2178,7 @@ int ndb_count_replies(struct ndb_txn *txn, const unsigned char *note_id, uint16_
 }
 
 /* count all of the reactions for a note */
-int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint32_t *count)
+int ndb_rebuild_reaction_metadata(struct ndb_txn *txn, const unsigned char *note_id, struct ndb_note_meta_builder *builder, uint32_t *count)
 {
 	MDB_val k, v;
 	MDB_cursor *cur;
@@ -2189,6 +2189,8 @@ int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint3
 	size_t size;
 	struct ndb_note *note;
 	unsigned char *keybuf, *last_id;
+	struct ndb_note_meta_entry *entry;
+	union ndb_reaction_str reaction_str;
 	char buffer[41]; /* 1 + 32 + 8 */
 	*count = 0;
 
@@ -2227,6 +2229,21 @@ int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint3
 			continue;
 		if (memcmp(last_id, note_id, 32))
 			continue;
+
+		if (builder) {
+			if (!ndb_reaction_set(&reaction_str, ndb_note_content(note)))
+				ndb_reaction_set(&reaction_str, "+");
+
+			if ((entry = ndb_note_meta_builder_find_entry(builder, NDB_NOTE_META_REACTION, &reaction_str.binmoji))) {
+				(*ndb_note_meta_reaction_count(entry))++;
+			} else if ((entry = ndb_note_meta_add_entry(builder))) {
+				ndb_note_meta_reaction_set(entry, 1, reaction_str);
+			} else {
+				/* couldn't add reaction entry ? */
+				ndb_debug("ndb_rebuild_note_indices: couldn't add reaction count entry to metadata builder\n");
+			}
+		}
+
 		(*count)++;
 	} while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0);
 
@@ -2301,19 +2318,22 @@ static int ndb_note_meta_builder_counts(struct ndb_txn *txn,
 	thread_replies = 0;
 	total_reactions = 0;
 
-	if (!(entry = ndb_note_meta_add_entry(builder)))
-		return 0;
-
-	rcs[0] = ndb_count_reactions(txn, note_id, &total_reactions);
+	rcs[0] = ndb_rebuild_reaction_metadata(txn, note_id, builder, &total_reactions);
 	rcs[1] = ndb_count_quotes(txn, note_id, &quotes);
 	rcs[2] = ndb_count_replies(txn, note_id, &direct_replies, &thread_replies);
 
-	if (!rcs[0] && !rcs[1] && !rcs[2])
+	if (!rcs[0] && !rcs[1] && !rcs[2]) {
 		return 0;
+	}
 
 	/* no entry needed */
-	if (quotes == 0 && direct_replies == 0 && thread_replies == 0 && quotes == 0)
-		return 1;
+	if (quotes == 0 && direct_replies == 0 && thread_replies == 0 && quotes == 0) {
+		return 0;
+	}
+
+	if (!(entry = ndb_note_meta_add_entry(builder))) {
+		return 0;
+	}
 
 	ndb_note_meta_counts_set(entry, total_reactions, quotes, direct_replies, thread_replies);
 
@@ -2334,49 +2354,58 @@ static int ndb_migrate_metadata(struct ndb_txn *txn)
 {
 	MDB_val k, k2, v, v2;
 	MDB_cursor *cur;
-	MDB_dbi db;
+	MDB_dbi note_db, meta_db;
 	unsigned char *id;
-	unsigned char buffer[4096];
-	int rc;
+	size_t scratch_size = 1024 * 1024;
+	unsigned char *buffer = malloc(scratch_size);
+	int rc, count;
 	struct ndb_note_meta_builder builder;
+	struct ndb_note *note;
 	struct ndb_note_meta *meta;
 
-	db = txn->lmdb->dbs[NDB_DB_META];
+	meta_db = txn->lmdb->dbs[NDB_DB_META];
+	note_db = txn->lmdb->dbs[NDB_DB_NOTE];
 
-	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) {
+	if ((rc = mdb_cursor_open(txn->mdb_txn, note_db, &cur))) {
 		fprintf(stderr, "ndb_migrate_reaction_stats: mdb_cursor_open failed, error %d\n", rc);
 		return -1;
 	}
 
+	count = 0;
+
 	/* loop through every metadata entry */
 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
-		ndb_note_meta_builder_init(&builder, buffer, sizeof(buffer));
+		ndb_note_meta_builder_init(&builder, buffer, scratch_size);
 
-		id = (unsigned char *)k.mv_data;
-		ndb_note_meta_builder_count_reactions(txn, &builder);
-		ndb_note_meta_builder_counts(txn, id, &builder);
-		ndb_note_meta_build(&builder, &meta);
+		note = (struct ndb_note *)v.mv_data;
+		id = ndb_note_id(note);
+		k2.mv_data = (unsigned char *)id;
+		k2.mv_size = 32;
 
-		/* no counts found, just delete this entry */
-		if (ndb_note_meta_entries_count(meta) == 0) {
-			if ((rc = mdb_del(txn->mdb_txn, db, &k, &v))) {
-				ndb_debug("delete old metadata entry failed: %s\n", mdb_strerror(rc));
-				return -1;
-			}
+		rc = ndb_note_meta_builder_counts(txn, id, &builder);
+		if (!rc) {
+			mdb_del(txn->mdb_txn, meta_db, &k2, NULL);
 			continue;
 		}
 
-		k2.mv_data = (unsigned char *)id;
-		k2.mv_size = 32;
+		ndb_note_meta_build(&builder, &meta);
+		assert(ndb_note_meta_entries(meta)->type != 0);
+
 		v2.mv_data = meta;
 		v2.mv_size = ndb_note_meta_total_size(meta);
 
 		/* set entry */
-		if ((rc = mdb_put(txn->mdb_txn, db, &k2, &v2, 0))) {
+		if ((rc = mdb_put(txn->mdb_txn, meta_db, &k2, &v2, 0))) {
 			ndb_debug("migrate metadata entry failed on write: %s\n", mdb_strerror(rc));
 		}
+
+		count++;
 	}
 
+	fprintf(stderr, "nostrdb: migrated %d metadata entries\n", count);
+
+	free(buffer);
+	mdb_cursor_close(cur);
 	return 1;
 }
 
@@ -2713,7 +2742,7 @@ static struct ndb_migration MIGRATIONS[] = {
 	{ .fn = ndb_migrate_lower_user_search_indices },
 	{ .fn = ndb_migrate_utf8_profile_names },
 	{ .fn = ndb_migrate_profile_indices },
-	//{ .fn = ndb_migrate_metadata },
+	{ .fn = ndb_migrate_metadata },
 };
 
 
@@ -8515,6 +8544,29 @@ void ndb_config_set_ingest_filter(struct ndb_config *config,
 	config->filter_context = filter_ctx;
 }
 
+int ndb_print_note_metadata(struct ndb_txn *txn)
+{
+	MDB_cursor *cur;
+	MDB_val k, v;
+	int i;
+
+	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &cur))
+		return 0;
+
+	i = 1;
+	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
+		print_hex(k.mv_data, 32);
+		printf("\t");
+		print_note_meta((struct ndb_note_meta*)v.mv_data);
+		i++;
+	}
+
+	mdb_cursor_close(cur);
+
+	return i;
+}
+
+
 int ndb_print_author_kind_index(struct ndb_txn *txn)
 {
 	MDB_cursor *cur;

src/nostrdb.h

@@ -49,8 +49,8 @@ struct bolt11;
  */
 enum ndb_metadata_type {
     NDB_NOTE_META_RESERVED = 0, /* not used */
-    NDB_NOTE_META_COUNTS   = 2, /* replies, quotes, etc */
-    NDB_NOTE_META_REACTION = 4, /* count of all the reactions on a post, grouped by different reaction strings */
+    NDB_NOTE_META_COUNTS   = 100, /* replies, quotes, etc */
+    NDB_NOTE_META_REACTION = 200, /* count of all the reactions on a post, grouped by different reaction strings */
 };
 
 // some bindings like swift needs help with forward declared pointers
@@ -658,6 +658,7 @@ struct ndb_note_meta *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char
 int ndb_set_note_meta(struct ndb *ndb, const unsigned char *id, struct ndb_note_meta *meta);
 size_t ndb_note_meta_total_size(struct ndb_note_meta *header);
 int ndb_note_meta_builder_init(struct ndb_note_meta_builder *builder, unsigned char *, size_t);
+struct ndb_note_meta_entry *ndb_note_meta_builder_find_entry(struct ndb_note_meta_builder *builder, uint16_t type, uint64_t *payload);
 void ndb_note_meta_build(struct ndb_note_meta_builder *builder, struct ndb_note_meta **meta);
 uint16_t ndb_note_meta_entries_count(struct ndb_note_meta *meta);
 struct ndb_note_meta_entry *ndb_note_meta_entries(struct ndb_note_meta *meta);
@@ -673,6 +674,7 @@ uint16_t *ndb_note_meta_counts_direct_replies(struct ndb_note_meta_entry *entry)
 uint32_t *ndb_note_meta_counts_thread_replies(struct ndb_note_meta_entry *entry);
 uint16_t *ndb_note_meta_entry_type(struct ndb_note_meta_entry *entry);
 struct ndb_note_meta_entry *ndb_note_meta_entry_at(struct ndb_note_meta *meta, int ind);
+void print_note_meta(struct ndb_note_meta *meta);
 
 // META STRINGS
 int ndb_reaction_set(union ndb_reaction_str *reaction, const char *str);

test.c

@@ -30,7 +30,7 @@ static void delete_test_db() {
     unlink(TEST_DIR "/data.lock");
 }
 
-int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint32_t *count);
+int ndb_rebuild_reaction_metadata(struct ndb_txn *txn, const unsigned char *note_id, struct ndb_note_meta_builder *builder, uint32_t *count);
 int ndb_count_replies(struct ndb_txn *txn, const unsigned char *note_id, uint16_t *direct_replies, uint32_t *thread_replies);
 
 static void db_load_events(struct ndb *ndb, const char *filename)
@@ -141,7 +141,7 @@ static void test_count_metadata()
 	ndb_begin_query(ndb, &txn);
 	/* this is used in the migration code,
 	 * let's make sure it matches the online logic */
-	ndb_count_reactions(&txn, id, &reactions);
+	ndb_rebuild_reaction_metadata(&txn, id, NULL, &reactions);
 	printf("\t# after-counted reactions %d\n", reactions);
 	assert(reactions == total_reactions);