Skip to content

Commit d3a98c3

Browse files
jb55jb55
committed
metadata: add initial metadata table api
Signed-off-by: William Casarin <[email protected]>
1 parent 47602df commit d3a98c3

File tree

4 files changed

+364
-9
lines changed

4 files changed

+364
-9
lines changed

‎Makefile‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ CCAN_HDRS := ccan/ccan/utf8/utf8.h ccan/ccan/container_of/container_of.h ccan/cc
55
HEADERS = deps/lmdb/lmdb.h deps/secp256k1/include/secp256k1.h src/nostrdb.h src/cursor.h src/hex.h src/jsmn.h src/config.h src/random.h src/memchr.h src/cpu.h src/nostr_bech32.h src/block.h src/str_block.h $(C_BINDINGS) $(CCAN_HDRS) $(BOLT11_HDRS)
66
FLATCC_SRCS=deps/flatcc/src/runtime/json_parser.c deps/flatcc/src/runtime/verifier.c deps/flatcc/src/runtime/builder.c deps/flatcc/src/runtime/emitter.c deps/flatcc/src/runtime/refmap.c
77
BOLT11_SRCS = src/bolt11/bolt11.c src/bolt11/bech32.c src/bolt11/amount.c src/bolt11/hash_u5.c
8-
SRCS = src/nostrdb.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c src/binmoji.c $(BOLT11_SRCS) $(FLATCC_SRCS) $(CCAN_SRCS)
8+
SRCS = src/nostrdb.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c src/binmoji.c src/metadata.c $(BOLT11_SRCS) $(FLATCC_SRCS) $(CCAN_SRCS)
99
LDS = $(OBJS) $(ARS)
1010
OBJS = $(SRCS:.c=.o)
1111
DEPS = $(OBJS) $(HEADERS) $(ARS)

‎src/metadata.c‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,240 @@
1+
2+
#include "nostrdb.h"
3+
#include "binmoji.h"
4+
5+
// these must be byte-aligned, they are directly accessing the serialized data
6+
// representation
7+
#pragma pack(push, 1)
8+
9+
// 16 bytes
10+
struct ndb_note_meta_entry {
11+
// 4 byte entry header
12+
uint16_t type;
13+
uint16_t flags;
14+
15+
// additional 4 bytes of aux storage for payloads that are >8 bytes
16+
//
17+
// for reactions types, this is used for counts
18+
// normally this would have been padding but we make use of it
19+
// in our manually packed structure
20+
//
21+
uint32_t aux;
22+
23+
// 8 byte metadata payload
24+
union {
25+
struct {
26+
uint32_t offset;
27+
uint32_t padding;
28+
} offset;
29+
30+
// the reaction binmoji[1] for reaction, count is stored in aux
31+
union ndb_reaction_str reaction_str;
32+
} payload;
33+
};
34+
STATIC_ASSERT(sizeof(struct ndb_note_meta_entry) == 16, note_meta_entry_should_be_16_bytes);
35+
36+
/* newtype wrapper around the header entry */
37+
struct ndb_note_meta {
38+
// 4 bytes
39+
uint8_t version;
40+
uint8_t padding;
41+
uint16_t count;
42+
43+
// 4 bytes
44+
uint32_t data_table_size;
45+
46+
// 8 bytes
47+
uint64_t flags;
48+
};
49+
STATIC_ASSERT(sizeof(struct ndb_note_meta) == 16, note_meta_entry_should_be_16_bytes);
50+
51+
#pragma pack(pop)
52+
53+
int ndb_reaction_str_is_emoji(union ndb_reaction_str str)
54+
{
55+
return binmoji_get_user_flag(str.binmoji) == 0;
56+
}
57+
58+
uint16_t ndb_note_meta_entries_count(struct ndb_note_meta *meta)
59+
{
60+
return meta->count;
61+
}
62+
63+
int ndb_reaction_set_emoji(union ndb_reaction_str *str, const char *emoji)
64+
{
65+
struct binmoji binmoji;
66+
/* TODO: parse failures? */
67+
binmoji_parse(emoji, &binmoji);
68+
str->binmoji = binmoji_encode(&binmoji);
69+
return 1;
70+
}
71+
72+
int ndb_reaction_set_str(union ndb_reaction_str *reaction, const char *str)
73+
{
74+
int i;
75+
char c;
76+
77+
/* this is like memset'ing the packed string to all 0s as well */
78+
reaction->binmoji = 0;
79+
80+
/* set the binmoji user flag so we can catch corrupt binmojis */
81+
/* this is in the LSB so it will only touch reaction->packed.flag */
82+
reaction->binmoji = binmoji_set_user_flag(reaction->binmoji, 1);
83+
assert(reaction->packed.flag != 0);
84+
85+
for (i = 0; i < 7; i++) {
86+
c = str[i];
87+
/* string is too big */
88+
if (i == 6 && c != '\0')
89+
return 0;
90+
reaction->packed.str[i] = c;
91+
if (c == '\0')
92+
return 1;
93+
}
94+
95+
return 0;
96+
}
97+
98+
static void ndb_note_meta_header_init(struct ndb_note_meta *meta)
99+
{
100+
meta->version = 1;
101+
meta->flags = 0;
102+
meta->count = 0;
103+
meta->data_table_size = 0;
104+
}
105+
106+
static inline size_t ndb_note_meta_entries_size(struct ndb_note_meta *meta)
107+
{
108+
return (sizeof(struct ndb_note_meta_entry) * meta->count);
109+
}
110+
111+
void *ndb_note_meta_data_table(struct ndb_note_meta *meta, size_t *size)
112+
{
113+
return meta + ndb_note_meta_entries_size(meta);
114+
}
115+
116+
size_t ndb_note_meta_total_size(struct ndb_note_meta *header)
117+
{
118+
size_t total_size = header->data_table_size + ndb_note_meta_entries_size(header);
119+
assert((total_size % 8) == 0);
120+
return total_size;
121+
}
122+
123+
struct ndb_note_meta_entry *ndb_note_meta_add_entry(struct ndb_note_meta_builder *builder)
124+
{
125+
struct ndb_note_meta *header = (struct ndb_note_meta *)builder->cursor.start;
126+
struct ndb_note_meta_entry *entry = NULL;
127+
128+
assert(builder->cursor.p != builder->cursor.start);
129+
130+
if (!(entry = cursor_malloc(&builder->cursor, sizeof(*entry))))
131+
return NULL;
132+
133+
/* increase count entry count */
134+
header->count++;
135+
136+
return entry;
137+
}
138+
139+
int ndb_note_meta_builder_init(struct ndb_note_meta_builder *builder, unsigned char *buf, size_t bufsize)
140+
{
141+
make_cursor(buf, buf + bufsize, &builder->cursor);
142+
143+
/* allocate some space for the header */
144+
if (!cursor_malloc(&builder->cursor, sizeof(struct ndb_note_meta)))
145+
return 0;
146+
147+
ndb_note_meta_header_init((struct ndb_note_meta*)builder->cursor.start);
148+
149+
return 1;
150+
}
151+
152+
/* note flags are stored in the header entry */
153+
uint32_t ndb_note_meta_flags(struct ndb_note_meta *meta)
154+
{
155+
return meta->flags;
156+
}
157+
158+
/* note flags are stored in the header entry */
159+
void ndb_note_meta_set_flags(struct ndb_note_meta *meta, uint32_t flags)
160+
{
161+
meta->flags = flags;
162+
}
163+
164+
static int compare_entries(const void *a, const void *b)
165+
{
166+
struct ndb_note_meta_entry *entry_a, *entry_b;
167+
uint64_t binmoji_a, binmoji_b;
168+
int res;
169+
170+
entry_a = (struct ndb_note_meta_entry *)a;
171+
entry_b = (struct ndb_note_meta_entry *)b;
172+
173+
res = entry_a->type - entry_b->type;
174+
175+
if (res == 0 && entry_a->type == NDB_NOTE_META_REACTION) {
176+
/* we sort by reaction string for stability */
177+
binmoji_a = entry_a->payload.reaction_str.binmoji;
178+
binmoji_b = entry_b->payload.reaction_str.binmoji;
179+
180+
if (binmoji_a < binmoji_b) {
181+
return -1;
182+
} else if (binmoji_a > binmoji_b) {
183+
return 1;
184+
} else {
185+
return 0;
186+
}
187+
} else {
188+
return res;
189+
}
190+
}
191+
192+
struct ndb_note_meta_entry *ndb_note_meta_entries(struct ndb_note_meta *meta)
193+
{
194+
/* entries start at the end of the header record */
195+
return (struct ndb_note_meta_entry *)((uint8_t*)meta + sizeof(*meta));
196+
}
197+
198+
void ndb_note_meta_build(struct ndb_note_meta_builder *builder, struct ndb_note_meta **meta)
199+
{
200+
/* sort entries */
201+
struct ndb_note_meta_entry *entries;
202+
struct ndb_note_meta *header = (struct ndb_note_meta*)builder->cursor.start;
203+
204+
/* not initialized */
205+
assert(builder->cursor.start != builder->cursor.p);
206+
207+
if (header->count > 0) {
208+
entries = ndb_note_meta_entries(header);
209+
210+
/* ensure entries are always sorted so bsearch is possible for large metadata
211+
* entries. probably won't need that for awhile though */
212+
qsort(entries, header->count, sizeof(struct ndb_note_meta_entry), compare_entries);
213+
}
214+
215+
*meta = header;
216+
return;
217+
}
218+
219+
void ndb_note_meta_reaction_set(struct ndb_note_meta_entry *entry, uint32_t count, union ndb_reaction_str str)
220+
{
221+
entry->type = NDB_NOTE_META_REACTION;
222+
entry->flags = 0;
223+
entry->aux = count;
224+
entry->payload.reaction_str = str;
225+
}
226+
227+
uint32_t ndb_note_meta_reaction_count(struct ndb_note_meta_entry *entry)
228+
{
229+
return entry->aux;
230+
}
231+
232+
void ndb_note_meta_reaction_set_count(struct ndb_note_meta_entry *entry, uint32_t count)
233+
{
234+
entry->aux = count;
235+
}
236+
237+
union ndb_reaction_str ndb_note_meta_reaction_str(struct ndb_note_meta_entry *entry)
238+
{
239+
return entry->payload.reaction_str;
240+
}

‎src/nostrdb.c‎

Lines changed: 55 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ enum ndb_writer_msgtype {
142142
NDB_WRITER_BLOCKS, // write parsed note blocks
143143
NDB_WRITER_MIGRATE, // migrate the database
144144
NDB_WRITER_NOTE_RELAY, // we already have the note, but we have more relays to write
145+
NDB_WRITER_NOTE_META, // write note metadata to the db
145146
};
146147

147148
// keys used for storing data in the NDB metadata database (NDB_DB_NDB_META)
@@ -2213,6 +2214,12 @@ struct ndb_writer_ndb_meta {
22132214
uint64_t version;
22142215
};
22152216

2217+
struct ndb_writer_note_meta {
2218+
unsigned char note_id[32];
2219+
void *metadata;
2220+
size_t metadata_len;
2221+
};
2222+
22162223
// Used in the writer thread when writing ndb_profile_fetch_record's
22172224
// kv = pubkey: recor
22182225
struct ndb_writer_last_fetch {
@@ -2237,6 +2244,7 @@ struct ndb_writer_msg {
22372244
struct ndb_writer_ndb_meta ndb_meta;
22382245
struct ndb_writer_last_fetch last_fetch;
22392246
struct ndb_writer_blocks blocks;
2247+
struct ndb_writer_note_meta note_meta;
22402248
};
22412249
};
22422250

@@ -3334,6 +3342,40 @@ static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type)
33343342
return last;
33353343
}
33363344

3345+
int ndb_set_note_meta(struct ndb *ndb, const unsigned char *id, void *meta, size_t *len)
3346+
{
3347+
struct ndb_writer_msg msg;
3348+
struct ndb_writer_note_meta *meta_msg = &msg.note_meta;
3349+
3350+
msg.type = NDB_WRITER_NOTE_META;
3351+
3352+
memcpy(meta_msg->note_id, id, 32);
3353+
3354+
return ndb_writer_queue_msg(&ndb->writer, &msg);
3355+
}
3356+
3357+
int ndb_writer_set_note_meta(struct ndb_txn *txn, const unsigned char *id, struct ndb_note_meta *meta)
3358+
{
3359+
MDB_val k, v;
3360+
MDB_dbi note_meta_db;
3361+
3362+
// get dbs
3363+
note_meta_db = txn->lmdb->dbs[NDB_DB_META];
3364+
3365+
k.mv_data = (unsigned char *)id;
3366+
k.mv_size = 32;
3367+
3368+
v.mv_data = (unsigned char *)meta;
3369+
v.mv_size = ndb_note_meta_total_size(meta);
3370+
3371+
if (mdb_put(txn->mdb_txn, note_meta_db, &k, &v, 0)) {
3372+
ndb_debug("ndb_set_note_meta: write note metadata to db failed: %s\n", mdb_strerror(rc));
3373+
return 0;
3374+
}
3375+
3376+
return 1;
3377+
}
3378+
33373379
void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len)
33383380
{
33393381
MDB_val k, v;
@@ -5514,13 +5556,16 @@ static void *ndb_writer_thread(void *data)
55145556
for (i = 0 ; i < popped; i++) {
55155557
msg = &msgs[i];
55165558
switch (msg->type) {
5517-
case NDB_WRITER_NOTE: needs_commit = 1; break;
5518-
case NDB_WRITER_PROFILE: needs_commit = 1; break;
5519-
case NDB_WRITER_DBMETA: needs_commit = 1; break;
5520-
case NDB_WRITER_PROFILE_LAST_FETCH: needs_commit = 1; break;
5521-
case NDB_WRITER_BLOCKS: needs_commit = 1; break;
5522-
case NDB_WRITER_MIGRATE: needs_commit = 1; break;
5523-
case NDB_WRITER_NOTE_RELAY: needs_commit = 1; break;
5559+
case NDB_WRITER_NOTE:
5560+
case NDB_WRITER_NOTE_META:
5561+
case NDB_WRITER_PROFILE:
5562+
case NDB_WRITER_DBMETA:
5563+
case NDB_WRITER_PROFILE_LAST_FETCH:
5564+
case NDB_WRITER_BLOCKS:
5565+
case NDB_WRITER_MIGRATE:
5566+
case NDB_WRITER_NOTE_RELAY:
5567+
needs_commit = 1;
5568+
break;
55245569
case NDB_WRITER_QUIT: break;
55255570
}
55265571
}
@@ -5561,6 +5606,9 @@ static void *ndb_writer_thread(void *data)
55615606
ndb_debug("failed to write note\n");
55625607
}
55635608
break;
5609+
case NDB_WRITER_NOTE_META:
5610+
break;
5611+
55645612
case NDB_WRITER_NOTE:
55655613
note_nkey = ndb_write_note(&txn, &msg->note,
55665614
scratch,

0 commit comments

Comments
 (0)