Add linked field to manifest (#679)

* remove unencrypted flag

* introduce linked field to manifest

* add flag for linked field

* review by @mafintosh

* fix typos

* simplify encoding

* throw on incompatible linked manifest

* default manifest version to 1

* explicit version wins

* linked is passed as null if optional

Author: chm-diederichs

index.js

@@ -241,7 +241,6 @@ class Hypercore extends EventEmitter {
 
   async setEncryption (encryption, opts) {
     if (!this.opened) await this.opening
-    if (this.core.unencrypted) return
 
     if (encryption === null) {
       this.encryption = encryption
@@ -329,7 +328,7 @@ class Hypercore extends EventEmitter {
 
     if (this.keyPair === null) this.keyPair = opts.keyPair || this.core.header.keyPair
 
-    if (!this.core.encryption && !this.core.unencrypted) {
+    if (!this.core.encryption) {
       const e = getEncryptionOption(opts)
 
       if (HypercoreEncryption.isHypercoreEncryption(e)) {
@@ -774,8 +773,8 @@ class Hypercore extends EventEmitter {
       block = b4a.from(block)
 
       if (this.encryption.compat !== this.core.compat) this._updateEncryption()
-      if (this.core.unencrypted) this.encryption = null
-      else await this.encryption.decrypt(index, block)
+
+      await this.encryption.decrypt(index, block)
     }
 
     return this._decode(encoding, block)
@@ -923,7 +922,7 @@ class Hypercore extends EventEmitter {
 
     blocks = Array.isArray(blocks) ? blocks : [blocks]
 
-    const preappend = this.core.unencrypted ? null : (this.encryption && this._preappend)
+    const preappend = this.encryption && this._preappend
     if (preappend) await this.encryption.ready()
 
     const buffers = this.encodeBatch !== null ? this.encodeBatch(blocks) : new Array(blocks.length)

lib/core.js

@@ -39,7 +39,6 @@ module.exports = class Core {
     this.verifier = null
     this.truncating = 0
     this.updating = false
-    this.unencrypted = false
     this.skipBitfield = null
     this.globalCache = opts.globalCache || null
     this.autoClose = opts.autoClose !== false
@@ -271,7 +270,6 @@ module.exports = class Core {
     // to unslab
     if (header.manifest) {
       header.manifest = Verifier.createManifest(header.manifest)
-      this.unencrypted = header.manifest.unencrypted
     }
 
     const verifier = header.manifest ? new Verifier(header.key, header.manifest, { crypto, legacy }) : null
@@ -326,7 +324,6 @@ module.exports = class Core {
     if (verifier.prologue) this.state.prologue = Object.assign({}, verifier.prologue)
 
     this.manifest = this.header.manifest = manifest
-    this.unencrypted = this.manifest.unencrypted
 
     tx.setAuth({
       key: this.header.key,

lib/messages.js

@@ -6,6 +6,10 @@ const unslab = require('unslab')
 
 const EMPTY = b4a.alloc(0)
 
+const MANIFEST_PATCH = 0b00000001
+const MANIFEST_PROLOGUE = 0b00000010
+const MANIFEST_LINKED = 0b00000100
+
 const hashes = {
   preencode (state, m) {
     state.end++ // small uint
@@ -137,7 +141,7 @@ const manifestv0 = {
           hash: c.fixed32.decode(state),
           length: 0
         },
-        unencrypted: false
+        linked: []
       }
     }
 
@@ -149,7 +153,7 @@ const manifestv0 = {
         quorum: 1,
         signers: [signer.decode(state)],
         prologue: null,
-        unencrypted: false
+        linked: []
       }
     }
 
@@ -162,11 +166,13 @@ const manifestv0 = {
       quorum: c.uint.decode(state),
       signers: signerArray.decode(state),
       prologue: null,
-      unencrypted: false
+      linked: []
     }
   }
 }
 
+const fixed32Array = c.array(c.fixed32)
+
 const manifest = exports.manifest = {
   preencode (state, m) {
     state.end++ // version
@@ -178,37 +184,53 @@ const manifest = exports.manifest = {
     c.uint.preencode(state, m.quorum)
     signerArray.preencode(state, m.signers)
     if (m.prologue) prologue.preencode(state, m.prologue)
+
+    if (m.linked) {
+      fixed32Array.preencode(state, m.linked)
+    }
   },
   encode (state, m) {
     c.uint.encode(state, m.version)
     if (m.version === 0) return manifestv0.encode(state, m)
 
-    c.uint.encode(state, (m.allowPatch ? 1 : 0) | (m.prologue ? 2 : 0) | (m.unencrypted ? 4 : 0))
+    let flags = 0
+    if (m.allowPatch) flags |= MANIFEST_PATCH
+    if (m.prologue) flags |= MANIFEST_PROLOGUE
+    if (m.linked) flags |= MANIFEST_LINKED
+
+    c.uint.encode(state, flags)
     hashes.encode(state, m.hash)
 
     c.uint.encode(state, m.quorum)
     signerArray.encode(state, m.signers)
     if (m.prologue) prologue.encode(state, m.prologue)
+
+    if (m.linked) {
+      fixed32Array.encode(state, m.linked)
+    }
   },
   decode (state) {
-    const v = c.uint.decode(state)
-    if (v === 0) return manifestv0.decode(state)
-    if (v !== 1) throw new Error('Unknown version: ' + v)
+    const version = c.uint.decode(state)
+    if (version === 0) return manifestv0.decode(state)
+    if (version > 2) throw new Error('Unknown version: ' + version)
 
     const flags = c.uint.decode(state)
     const hash = hashes.decode(state)
     const quorum = c.uint.decode(state)
     const signers = signerArray.decode(state)
-    const unencrypted = (flags & 4) !== 0
+
+    const hasPatch = (flags & MANIFEST_PATCH) !== 0
+    const hasPrologue = (flags & MANIFEST_PROLOGUE) !== 0
+    const hasLinked = (flags & MANIFEST_LINKED) !== 0
 
     return {
-      version: 1,
+      version,
       hash,
-      allowPatch: (flags & 1) !== 0,
+      allowPatch: hasPatch,
       quorum,
       signers,
-      prologue: (flags & 2) === 0 ? null : prologue.decode(state),
-      unencrypted
+      prologue: hasPrologue ? prologue.decode(state) : null,
+      linked: hasLinked ? fixed32Array.decode(state) : null
     }
   }
 }

lib/verifier.js

@@ -129,7 +129,7 @@ module.exports = class Verifier {
   }
 
   verify (batch, signature) {
-    if (this.version !== 1) {
+    if (this.version === 0) {
       return this._verifyCompat(batch, signature)
     }
 
@@ -182,7 +182,8 @@ module.exports = class Verifier {
         namespace: caps.DEFAULT_NAMESPACE,
         publicKey
       }],
-      prologue: null
+      prologue: null,
+      linked: null
     }
   }
 
@@ -195,13 +196,13 @@ module.exports = class Verifier {
     if (!inp) return null
 
     const manifest = {
-      version: typeof inp.version === 'number' ? inp.version : 1,
+      version: getManifestVersion(inp), // only v2 if linked are present
       hash: 'blake2b',
       allowPatch: !!inp.allowPatch,
       quorum: defaultQuorum(inp),
       signers: inp.signers ? inp.signers.map(parseSigner) : [],
       prologue: null,
-      unencrypted: !!inp.unencrypted
+      linked: null
     }
 
     if (inp.hash && inp.hash !== 'blake2b') throw BAD_ARGUMENT('Only Blake2b hashes are supported')
@@ -215,6 +216,18 @@ module.exports = class Verifier {
       manifest.prologue.hash = unslab(manifest.prologue.hash)
     }
 
+    if (inp.linked && inp.linked.length) {
+      if (manifest.version < 2) throw BAD_ARGUMENT('Invalid field')
+
+      for (const key of inp.linked) {
+        if (!(b4a.isBuffer(key) && key.byteLength === 32)) {
+          throw BAD_ARGUMENT('Invalid key')
+        }
+      }
+
+      manifest.linked = inp.linked
+    }
+
     return manifest
   }
 
@@ -311,3 +324,9 @@ function proofToVersion1 (proof) {
     patch: proof.patch ? proof.patch.length : 0
   }
 }
+
+function getManifestVersion (inp) {
+  if (typeof inp.version === 'number') return inp.version
+  if (inp.linked && inp.linked.length) return 2
+  return 1
+}

test/manifest.js

@@ -1343,7 +1343,7 @@ test('manifest encoding', t => {
       namespace: b4a.alloc(32, 1),
       publicKey: keyPair.publicKey
     }],
-    unencrypted: false
+    linked: []
   }
 
   t.alike(reencode(manifest), manifest)
@@ -1427,6 +1427,12 @@ test('manifest encoding', t => {
   manifest.allowPatch = false
   t.alike(reencode(manifest), manifest)
 
+  // with linked cores
+  manifest.version = 2
+  manifest.linked.push(b4a.alloc(32, 4))
+
+  t.alike(reencode(manifest), manifest)
+
   function reencode (m) {
     return c.decode(enc.manifest, c.encode(enc.manifest, m))
   }
@@ -1487,7 +1493,8 @@ function createMultiManifest (signers, prologue = null) {
       namespace: caps.DEFAULT_NAMESPACE,
       publicKey: s.manifest.signers[0].publicKey
     })),
-    prologue
+    prologue,
+    linked: []
   }
 }