Skip to content

Commit f8aeb7c

Browse files
jb55jb55
committed
metadata: add initial metadata table api
Signed-off-by: William Casarin <[email protected]>
1 parent c7a2f24 commit f8aeb7c

File tree

4 files changed

+400
-9
lines changed

4 files changed

+400
-9
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ CCAN_HDRS := ccan/ccan/utf8/utf8.h ccan/ccan/container_of/container_of.h ccan/cc
55
HEADERS = deps/lmdb/lmdb.h deps/secp256k1/include/secp256k1.h src/nostrdb.h src/cursor.h src/hex.h src/jsmn.h src/config.h src/random.h src/memchr.h src/cpu.h src/nostr_bech32.h src/block.h src/str_block.h src/print_util.h $(C_BINDINGS) $(CCAN_HDRS) $(BOLT11_HDRS)
66
FLATCC_SRCS=deps/flatcc/src/runtime/json_parser.c deps/flatcc/src/runtime/verifier.c deps/flatcc/src/runtime/builder.c deps/flatcc/src/runtime/emitter.c deps/flatcc/src/runtime/refmap.c
77
BOLT11_SRCS = src/bolt11/bolt11.c src/bolt11/bech32.c src/bolt11/amount.c src/bolt11/hash_u5.c
8-
SRCS = src/nostrdb.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c src/binmoji.c $(BOLT11_SRCS) $(FLATCC_SRCS) $(CCAN_SRCS)
8+
SRCS = src/nostrdb.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c src/binmoji.c src/metadata.c $(BOLT11_SRCS) $(FLATCC_SRCS) $(CCAN_SRCS)
99
LDS = $(OBJS) $(ARS)
1010
OBJS = $(SRCS:.c=.o)
1111
DEPS = $(OBJS) $(HEADERS) $(ARS)

‎src/metadata.c‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
2+
#include "nostrdb.h"
3+
#include "binmoji.h"
4+
5+
// these must be byte-aligned, they are directly accessing the serialized data
6+
// representation
7+
#pragma pack(push, 1)
8+
9+
// 16 bytes
10+
struct ndb_note_meta_entry {
11+
// 4 byte entry header
12+
uint16_t type;
13+
uint16_t flags;
14+
15+
// additional 4 bytes of aux storage for payloads that are >8 bytes
16+
//
17+
// for reactions types, this is used for counts
18+
// normally this would have been padding but we make use of it
19+
// in our manually packed structure
20+
uint32_t aux;
21+
22+
// 8 byte metadata payload
23+
union {
24+
uint64_t value;
25+
26+
struct {
27+
uint32_t offset;
28+
uint32_t padding;
29+
} offset;
30+
31+
// the reaction binmoji[1] for reaction, count is stored in aux
32+
union ndb_reaction_str reaction_str;
33+
} payload;
34+
};
35+
STATIC_ASSERT(sizeof(struct ndb_note_meta_entry) == 16, note_meta_entry_should_be_16_bytes);
36+
37+
/* newtype wrapper around the header entry */
38+
struct ndb_note_meta {
39+
// 4 bytes
40+
uint8_t version;
41+
uint8_t padding;
42+
uint16_t count;
43+
44+
// 4 bytes
45+
uint32_t data_table_size;
46+
47+
// 8 bytes
48+
uint64_t flags;
49+
};
50+
STATIC_ASSERT(sizeof(struct ndb_note_meta) == 16, note_meta_entry_should_be_16_bytes);
51+
52+
#pragma pack(pop)
53+
54+
int ndb_reaction_str_is_emoji(union ndb_reaction_str str)
55+
{
56+
return binmoji_get_user_flag(str.binmoji) == 0;
57+
}
58+
59+
uint16_t ndb_note_meta_entries_count(struct ndb_note_meta *meta)
60+
{
61+
return meta->count;
62+
}
63+
64+
int ndb_reaction_set_emoji(union ndb_reaction_str *str, const char *emoji)
65+
{
66+
struct binmoji binmoji;
67+
/* TODO: parse failures? */
68+
binmoji_parse(emoji, &binmoji);
69+
str->binmoji = binmoji_encode(&binmoji);
70+
return 1;
71+
}
72+
73+
int ndb_reaction_set_str(union ndb_reaction_str *reaction, const char *str)
74+
{
75+
int i;
76+
char c;
77+
78+
/* this is like memset'ing the packed string to all 0s as well */
79+
reaction->binmoji = 0;
80+
81+
/* set the binmoji user flag so we can catch corrupt binmojis */
82+
/* this is in the LSB so it will only touch reaction->packed.flag */
83+
reaction->binmoji = binmoji_set_user_flag(reaction->binmoji, 1);
84+
assert(reaction->packed.flag != 0);
85+
86+
for (i = 0; i < 7; i++) {
87+
c = str[i];
88+
/* string is too big */
89+
if (i == 6 && c != '\0')
90+
return 0;
91+
reaction->packed.str[i] = c;
92+
if (c == '\0')
93+
return 1;
94+
}
95+
96+
return 0;
97+
}
98+
99+
static void ndb_note_meta_header_init(struct ndb_note_meta *meta)
100+
{
101+
meta->version = 1;
102+
meta->flags = 0;
103+
meta->count = 0;
104+
meta->data_table_size = 0;
105+
}
106+
107+
static inline size_t ndb_note_meta_entries_size(struct ndb_note_meta *meta)
108+
{
109+
return (sizeof(struct ndb_note_meta_entry) * meta->count);
110+
}
111+
112+
void *ndb_note_meta_data_table(struct ndb_note_meta *meta, size_t *size)
113+
{
114+
return meta + ndb_note_meta_entries_size(meta);
115+
}
116+
117+
size_t ndb_note_meta_total_size(struct ndb_note_meta *header)
118+
{
119+
size_t total_size = sizeof(*header) + header->data_table_size + ndb_note_meta_entries_size(header);
120+
assert((total_size % 8) == 0);
121+
return total_size;
122+
}
123+
124+
struct ndb_note_meta_entry *ndb_note_meta_add_entry(struct ndb_note_meta_builder *builder)
125+
{
126+
struct ndb_note_meta *header = (struct ndb_note_meta *)builder->cursor.start;
127+
struct ndb_note_meta_entry *entry = NULL;
128+
129+
assert(builder->cursor.p != builder->cursor.start);
130+
131+
if (!(entry = cursor_malloc(&builder->cursor, sizeof(*entry))))
132+
return NULL;
133+
134+
/* increase count entry count */
135+
header->count++;
136+
137+
return entry;
138+
}
139+
140+
int ndb_note_meta_builder_init(struct ndb_note_meta_builder *builder, unsigned char *buf, size_t bufsize)
141+
{
142+
make_cursor(buf, buf + bufsize, &builder->cursor);
143+
144+
/* allocate some space for the header */
145+
if (!cursor_malloc(&builder->cursor, sizeof(struct ndb_note_meta)))
146+
return 0;
147+
148+
ndb_note_meta_header_init((struct ndb_note_meta*)builder->cursor.start);
149+
150+
return 1;
151+
}
152+
153+
/* note flags are stored in the header entry */
154+
uint32_t ndb_note_meta_flags(struct ndb_note_meta *meta)
155+
{
156+
return meta->flags;
157+
}
158+
159+
/* note flags are stored in the header entry */
160+
void ndb_note_meta_set_flags(struct ndb_note_meta *meta, uint32_t flags)
161+
{
162+
meta->flags = flags;
163+
}
164+
165+
static int compare_entries(const void *a, const void *b)
166+
{
167+
struct ndb_note_meta_entry *entry_a, *entry_b;
168+
uint64_t binmoji_a, binmoji_b;
169+
int res;
170+
171+
entry_a = (struct ndb_note_meta_entry *)a;
172+
entry_b = (struct ndb_note_meta_entry *)b;
173+
174+
res = entry_a->type - entry_b->type;
175+
176+
if (res == 0 && entry_a->type == NDB_NOTE_META_REACTION) {
177+
/* we sort by reaction string for stability */
178+
binmoji_a = entry_a->payload.reaction_str.binmoji;
179+
binmoji_b = entry_b->payload.reaction_str.binmoji;
180+
181+
if (binmoji_a < binmoji_b) {
182+
return -1;
183+
} else if (binmoji_a > binmoji_b) {
184+
return 1;
185+
} else {
186+
return 0;
187+
}
188+
} else {
189+
return res;
190+
}
191+
}
192+
193+
struct ndb_note_meta_entry *ndb_note_meta_entries(struct ndb_note_meta *meta)
194+
{
195+
/* entries start at the end of the header record */
196+
return (struct ndb_note_meta_entry *)((uint8_t*)meta + sizeof(*meta));
197+
}
198+
199+
void ndb_note_meta_build(struct ndb_note_meta_builder *builder, struct ndb_note_meta **meta)
200+
{
201+
/* sort entries */
202+
struct ndb_note_meta_entry *entries;
203+
struct ndb_note_meta *header = (struct ndb_note_meta*)builder->cursor.start;
204+
205+
/* not initialized */
206+
assert(builder->cursor.start != builder->cursor.p);
207+
208+
if (header->count > 0) {
209+
entries = ndb_note_meta_entries(header);
210+
211+
/* ensure entries are always sorted so bsearch is possible for large metadata
212+
* entries. probably won't need that for awhile though */
213+
qsort(entries, header->count, sizeof(struct ndb_note_meta_entry), compare_entries);
214+
}
215+
216+
*meta = header;
217+
return;
218+
}
219+
220+
/* find a metadata entry, optionally matching a payload */
221+
struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload)
222+
{
223+
struct ndb_note_meta_entry *entries, *entry;
224+
int i;
225+
226+
if (meta->count == 0)
227+
return NULL;
228+
229+
entries = ndb_note_meta_entries(meta);
230+
231+
for (i = 0; i < meta->count; i++) {
232+
entry = &entries[i];
233+
if (entry->type != type)
234+
continue;
235+
if (payload && *payload != entry->payload.value)
236+
continue;
237+
return entry;
238+
}
239+
240+
return NULL;
241+
}
242+
243+
void ndb_note_meta_reaction_set(struct ndb_note_meta_entry *entry, uint32_t count, union ndb_reaction_str str)
244+
{
245+
entry->type = NDB_NOTE_META_REACTION;
246+
entry->flags = 0;
247+
entry->aux = count;
248+
entry->payload.reaction_str = str;
249+
}
250+
251+
/* sets the quote repost count for this note */
252+
void ndb_note_meta_quotes_set(struct ndb_note_meta_entry *entry, uint32_t count)
253+
{
254+
entry->type = NDB_NOTE_META_QUOTES;
255+
entry->flags = 0;
256+
entry->aux = count;
257+
/* unused */
258+
entry->payload.value = 0;
259+
}
260+
261+
uint32_t ndb_note_meta_reaction_count(struct ndb_note_meta_entry *entry)
262+
{
263+
return entry->aux;
264+
}
265+
266+
void ndb_note_meta_reaction_set_count(struct ndb_note_meta_entry *entry, uint32_t count)
267+
{
268+
entry->aux = count;
269+
}
270+
271+
union ndb_reaction_str ndb_note_meta_reaction_str(struct ndb_note_meta_entry *entry)
272+
{
273+
return entry->payload.reaction_str;
274+
}

‎src/nostrdb.c‎

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ enum ndb_writer_msgtype {
142142
NDB_WRITER_BLOCKS, // write parsed note blocks
143143
NDB_WRITER_MIGRATE, // migrate the database
144144
NDB_WRITER_NOTE_RELAY, // we already have the note, but we have more relays to write
145+
NDB_WRITER_NOTE_META, // write note metadata to the db
145146
};
146147

147148
// keys used for storing data in the NDB metadata database (NDB_DB_NDB_META)
@@ -2213,6 +2214,11 @@ struct ndb_writer_ndb_meta {
22132214
uint64_t version;
22142215
};
22152216

2217+
struct ndb_writer_note_meta {
2218+
unsigned char note_id[32];
2219+
struct ndb_note_meta *metadata;
2220+
};
2221+
22162222
// Used in the writer thread when writing ndb_profile_fetch_record's
22172223
// kv = pubkey: recor
22182224
struct ndb_writer_last_fetch {
@@ -2237,6 +2243,7 @@ struct ndb_writer_msg {
22372243
struct ndb_writer_ndb_meta ndb_meta;
22382244
struct ndb_writer_last_fetch last_fetch;
22392245
struct ndb_writer_blocks blocks;
2246+
struct ndb_writer_note_meta note_meta;
22402247
};
22412248
};
22422249

@@ -3334,6 +3341,42 @@ static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type)
33343341
return last;
33353342
}
33363343

3344+
int ndb_set_note_meta(struct ndb *ndb, const unsigned char *id, struct ndb_note_meta *meta)
3345+
{
3346+
struct ndb_writer_msg msg;
3347+
struct ndb_writer_note_meta *meta_msg = &msg.note_meta;
3348+
3349+
msg.type = NDB_WRITER_NOTE_META;
3350+
3351+
memcpy(meta_msg->note_id, id, 32);
3352+
meta_msg->metadata = meta;
3353+
3354+
return ndb_writer_queue_msg(&ndb->writer, &msg);
3355+
}
3356+
3357+
int ndb_writer_set_note_meta(struct ndb_txn *txn, const unsigned char *id, struct ndb_note_meta *meta)
3358+
{
3359+
int rc;
3360+
MDB_val k, v;
3361+
MDB_dbi note_meta_db;
3362+
3363+
// get dbs
3364+
note_meta_db = txn->lmdb->dbs[NDB_DB_META];
3365+
3366+
k.mv_data = (unsigned char *)id;
3367+
k.mv_size = 32;
3368+
3369+
v.mv_data = (unsigned char *)meta;
3370+
v.mv_size = ndb_note_meta_total_size(meta);
3371+
3372+
if ((rc = mdb_put(txn->mdb_txn, note_meta_db, &k, &v, 0))) {
3373+
ndb_debug("ndb_set_note_meta: write note metadata to db failed: %s\n", mdb_strerror(rc));
3374+
return 0;
3375+
}
3376+
3377+
return 1;
3378+
}
3379+
33373380
void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len)
33383381
{
33393382
MDB_val k, v;
@@ -5514,13 +5557,16 @@ static void *ndb_writer_thread(void *data)
55145557
for (i = 0 ; i < popped; i++) {
55155558
msg = &msgs[i];
55165559
switch (msg->type) {
5517-
case NDB_WRITER_NOTE: needs_commit = 1; break;
5518-
case NDB_WRITER_PROFILE: needs_commit = 1; break;
5519-
case NDB_WRITER_DBMETA: needs_commit = 1; break;
5520-
case NDB_WRITER_PROFILE_LAST_FETCH: needs_commit = 1; break;
5521-
case NDB_WRITER_BLOCKS: needs_commit = 1; break;
5522-
case NDB_WRITER_MIGRATE: needs_commit = 1; break;
5523-
case NDB_WRITER_NOTE_RELAY: needs_commit = 1; break;
5560+
case NDB_WRITER_NOTE:
5561+
case NDB_WRITER_NOTE_META:
5562+
case NDB_WRITER_PROFILE:
5563+
case NDB_WRITER_DBMETA:
5564+
case NDB_WRITER_PROFILE_LAST_FETCH:
5565+
case NDB_WRITER_BLOCKS:
5566+
case NDB_WRITER_MIGRATE:
5567+
case NDB_WRITER_NOTE_RELAY:
5568+
needs_commit = 1;
5569+
break;
55245570
case NDB_WRITER_QUIT: break;
55255571
}
55265572
}
@@ -5561,6 +5607,9 @@ static void *ndb_writer_thread(void *data)
55615607
ndb_debug("failed to write note\n");
55625608
}
55635609
break;
5610+
case NDB_WRITER_NOTE_META:
5611+
break;
5612+
55645613
case NDB_WRITER_NOTE:
55655614
note_nkey = ndb_write_note(&txn, &msg->note,
55665615
scratch,

0 commit comments

Comments
 (0)