spent utxo tracking added

Author: setavenger

README.md

@@ -7,7 +7,11 @@ matches [other implementations](https://github.com/bitcoin/bitcoin/pull/28241#is
 
 ## Setup
 
-The installation process is still very manual. Will be improved based on feedback and new findings.
+The installation process is still very manual. Will be improved based on feedback and new findings. It is advised to look at the example [blindbit.toml](blindbit.example.toml). As new config options appear they will be listed and explained there.
+
+## Breaking changes
+
+- Endpoints were expanded and have a slightly different syntax now [see endpoints](#endpoints)
 
 ### Requirements
 
@@ -39,9 +43,11 @@ $ <path/to/new/binary/file> --datadir <path/to/datadir/with/blindbit.toml>
 Note that the program will automatically default `--datadir` to `~/.blindbit-oracle` if not set.
 You still have to set up a config file in any case as the rpc users can't and **should** not be defaulted.
 
+You can now also decide which index you want to run. This setting can be set in the config file (blindbit.toml).
+
 ## Known Errors
 
-All known errors resolved.
+No known issues.
 
 ## Todos
 
@@ -69,6 +75,8 @@ All known errors resolved.
 - [ ] Remove unnecessary panics.
 - [ ] Convert hardcoded serialisation assertions into constants (?)
 - [x] Use x-only 32 byte public keys instead of scriptPubKey
+- [ ] Don't create all DBs by default, only those which are needed and activated
+- [ ] Check import paths (SilentPaymentBackend/.../...)
 
 ### Low Priority
 
@@ -80,8 +88,10 @@ All known errors resolved.
 ```text
 GET("/block-height")  // returns the height of the indexing server
 GET("/tweaks/:blockheight?dustLimit=<sat_amount>")  // returns tweak data (cut-through); optional parameter dustLimit can be omitted; filtering happens per request, so virtually any amount can be specified
-GET("/tweak-index/:blockheight")  // returns the full tweak index (no cut-through)
-GET("/filter/:blockheight") // returns a custom taproot only filter (the underlying data is subject to change; changing scriptPubKey to x-only pubKey) 
+GET("/tweak-index/:blockheight?dustLimit=<sat_amount>")  // returns the full tweak index (no cut-through); optional parameter dustLimit can be omitted; filtering happens per request, so virtually any amount can be specified
+GET("/spent-index/:blockheight")  // returns the spent outpoints index (see https://github.com/setavenger/BIP0352-light-client-specification?tab=readme-ov-file#spent-utxos)
+GET("/filter/spent/:blockheight") // returns a filter for shortened spent outpoint hashes (see https://github.com/setavenger/BIP0352-light-client-specification?tab=readme-ov-file#filters)
+GET("/filter/new-utxos/:blockheight") // returns a custom taproot only filter of x-only pubkey which received funds
 GET("/utxos/:blockheight")  // UTXO data for that block (cut down to the essentials needed to spend)
 ```
 

blindbit.example.toml

@@ -3,10 +3,10 @@
 host = "127.0.0.1:8000"
 
 # Defines on which chain the wallet runs. Allowed values: main, testnet, signet, regtest.
-# Default: signet
+# default: signet
 chain = "signet"
 
-# (defaults to http://127.0.0.1:8332)
+# default: http://127.0.0.1:8332
 rpc_endpoint = "http://127.0.0.1:18443"
 
 # required, unless rpc_user and rpc_pass are set
@@ -28,7 +28,23 @@ max_parallel_tweak_computations = 4
 # (depends on max-rpc-workers of the underlying full node)
 max_parallel_requests = 4
 
-# 
 # optional - will only generate tweaks (still both cut-through and full-index)
-# default: 0 set to 1 to activate
+# default: 0
 tweaks_only = 0
+
+# The base index. Only includes the tweaks. No dust filtering or cut-through possible
+# default: 1
+tweaks_full_basic = 1
+
+# if this is set a full non-cut-through index will be created.
+# This index can be used to filter for dust (?dustLimit=). If this is active the base index will not be created.
+# All full index queries will be served from this with or without (?dustLimit=) set in the query.
+# default 0
+tweaks_full_with_dust_filter = 0
+
+# This index applies cut-through and dust filtering.
+# Beware that it will be stored in addition to any full index (with or without dust) if activated.
+# It has more storage requirments than the simple indices.
+# Currently still requires tweaks_only=1.
+# default: 0
+tweaks_cut_through_with_dust_filter = 0

src/common/config.go

@@ -1,6 +1,9 @@
 package common
 
 import (
+	"errors"
+	"os"
+
 	"github.com/spf13/viper"
 )
 
@@ -23,23 +26,32 @@ func LoadConfigs(pathToConfig string) {
 	// RPC endpoint only. Fails if others are not set
 	viper.SetDefault("rpc_endpoint", RpcEndpoint)
 
-	//Others
+	//Tweaks
 	viper.SetDefault("tweaks_only", false)
+	viper.SetDefault("tweaks_full_basic", true)
+	viper.SetDefault("tweaks_full_with_dust_filter", false)
+	viper.SetDefault("tweaks_cut_through_with_dust_filter", false)
 
 	/* read and set config variables */
+	// General
+	SyncStartHeight = viper.GetUint32("sync_start_height")
 	Host = viper.GetString("host")
 
+	// Performance
 	MaxParallelRequests = viper.GetUint16("max_parallel_requests")
 	MaxParallelTweakComputations = viper.GetInt("max_parallel_tweak_computations")
 
+	// RPC
 	RpcEndpoint = viper.GetString("rpc_endpoint")
 	CookiePath = viper.GetString("cookie_path")
 	RpcPass = viper.GetString("rpc_pass")
 	RpcUser = viper.GetString("rpc_user")
 
-	SyncStartHeight = viper.GetUint32("sync_start_height")
-
+	// Tweaks
 	TweaksOnly = viper.GetBool("tweaks_only")
+	TweakIndexFullNoDust = viper.GetBool("tweaks_full_basic")
+	TweakIndexFullIncludingDust = viper.GetBool("tweaks_full_with_dust_filter")
+	TweaksCutThroughWithDust = viper.GetBool("tweaks_cut_through_with_dust_filter")
 
 	chainInput := viper.GetString("chain")
 
@@ -55,4 +67,21 @@ func LoadConfigs(pathToConfig string) {
 	default:
 		panic("chain undefined")
 	}
+
+	// todo print settings
+	InfoLogger.Printf("tweaks_only: %t\n", TweaksOnly)
+	InfoLogger.Printf("tweaks_full_basic: %t\n", TweakIndexFullNoDust)
+	InfoLogger.Printf("tweaks_full_with_dust_filter: %t\n", TweakIndexFullIncludingDust)
+	InfoLogger.Printf("tweaks_cut_through_with_dust_filter: %t\n", TweaksCutThroughWithDust)
+
+	if !TweakIndexFullNoDust && !TweakIndexFullIncludingDust && !TweaksCutThroughWithDust {
+		WarningLogger.Println("no tweaks are being collected, all tweak settings were set to 0")
+		WarningLogger.Println("make sure your configuration loaded correctly, check example blindbit.toml for configuration")
+	}
+
+	if TweaksCutThroughWithDust && TweaksOnly {
+		err := errors.New("cut through requires tweaks_only to be set to 1")
+		ErrorLogger.Println(err)
+		os.Exit(1)
+	}
 }

src/common/types/tweak.go

@@ -8,12 +8,14 @@ import (
 	"errors"
 )
 
+const TweakDataLength = 33
+
 type Tweak struct {
 	BlockHash string `json:"block_hash"`
 	// BlockHeight todo not really used at the moment, could be added on a per request basis in the API handler
-	BlockHeight uint32   `json:"block_height"`
-	Txid        string   `json:"txid"`
-	Data        [33]byte `json:"data"`
+	BlockHeight uint32                `json:"block_height"`
+	Txid        string                `json:"txid"`
+	TweakData   [TweakDataLength]byte `json:"tweak_data"`
 	// HighestValue indicates the value of the UTXO with the most value for a specific tweak
 	HighestValue uint64
 }
@@ -30,7 +32,7 @@ func (v *Tweak) SerialiseKey() ([]byte, error) {
 func (v *Tweak) SerialiseData() ([]byte, error) {
 	var buf bytes.Buffer
 
-	buf.Write(v.Data[:])
+	buf.Write(v.TweakData[:])
 
 	err := binary.Write(&buf, binary.BigEndian, v.HighestValue)
 	if err != nil {
@@ -53,19 +55,19 @@ func (v *Tweak) DeSerialiseKey(key []byte) error {
 }
 
 func (v *Tweak) DeSerialiseData(data []byte) error {
-	if len(data) != 33 && len(data) != 33+8 {
+	// todo why did we check both dusted and non dusted
+	if len(data) != TweakDataLength+8 {
 		common.ErrorLogger.Printf("wrong data length: %+v", data)
 		return errors.New("data is wrong length. should not happen")
 	}
-	copy(v.Data[:], data[:33])
+	copy(v.TweakData[:], data[:TweakDataLength])
 
 	// we only try to read HighestValue if the data is there
-	if len(data) == 33+8 {
-		err := binary.Read(bytes.NewReader(data[33:]), binary.BigEndian, &v.HighestValue)
-		if err != nil {
-			common.ErrorLogger.Println(err)
-			return err
-		}
+	// revoke: if the data is not there it seems like an implementation error. prior, where dust was an option it made sense
+	err := binary.Read(bytes.NewReader(data[TweakDataLength:]), binary.BigEndian, &v.HighestValue)
+	if err != nil {
+		common.ErrorLogger.Println(err)
+		return err
 	}
 
 	return nil

src/common/types/tweak_interface.go

@@ -0,0 +1,6 @@
+package types
+
+type TweakDusted interface {
+	HighestValue() uint64
+	Tweak() [33]byte // todo pointer to show none?
+}

src/common/types/tweakindex.go

@@ -11,9 +11,9 @@ import (
 // there is no transaction cut-through, so it will keep a full history.
 // The tweaks will most likely not be sorted in any meaningful way and have no metadata attached.
 type TweakIndex struct {
-	BlockHash   string     `json:"block_hash"`
-	BlockHeight uint32     `json:"block_height"`
-	Data        [][33]byte `json:"data"`
+	BlockHash   string                  `json:"block_hash"`
+	BlockHeight uint32                  `json:"block_height"`
+	Data        [][TweakDataLength]byte `json:"data"`
 }
 
 func PairFactoryTweakIndex() Pair {
@@ -29,7 +29,7 @@ func (v *TweakIndex) SerialiseKey() ([]byte, error) {
 func (v *TweakIndex) SerialiseData() ([]byte, error) {
 
 	// todo can this be made more efficiently?
-	totalLength := len(v.Data) * 33
+	totalLength := len(v.Data) * TweakDataLength
 	flattened := make([]byte, 0, totalLength)
 
 	for _, byteArray := range v.Data {
@@ -51,27 +51,31 @@ func (v *TweakIndex) DeSerialiseKey(key []byte) error {
 }
 
 func (v *TweakIndex) DeSerialiseData(data []byte) error {
-	if len(data)%33 != 0 {
+	if len(data)%TweakDataLength != 0 {
 		common.ErrorLogger.Printf("wrong data length: %+v", data)
 		return errors.New("data is wrong length. should not happen")
 	}
 
-	numArrays := len(data) / 33
-	v.Data = make([][33]byte, numArrays)
+	numArrays := len(data) / TweakDataLength
+	v.Data = make([][TweakDataLength]byte, numArrays)
 	// Iterate and copy segments from the flat slice into the new array of arrays
 	for i := 0; i < numArrays; i++ {
-		copy(v.Data[i][:], data[i*33:(i+1)*33])
+		copy(v.Data[i][:], data[i*TweakDataLength:(i+1)*TweakDataLength])
 	}
 	return nil
 }
 
 // TweakIndexFromTweakArray builds a TweakIndex from a slice of Tweak
 // comes without blockHash or height
-func TweakIndexFromTweakArray(tweaks []Tweak) *TweakIndex {
+func TweakIndexFromTweakArray(tweaksMap map[string]Tweak, block *Block) *TweakIndex {
+	// todo benchmark the sorting, should not create too much overhead,
+	//  seems more like a nice to have for comparisons across implementations
 	var index TweakIndex
 	// can only panic hence no error output
-	for _, tweak := range tweaks {
-		index.Data = append(index.Data, tweak.Data)
+	for _, tx := range block.Txs {
+		if tweak, exists := tweaksMap[tx.Txid]; exists {
+			index.Data = append(index.Data, tweak.TweakData)
+		}
 	}
 	return &index
 }
@@ -84,7 +88,7 @@ func (v *TweakIndex) ToTweakArray() (tweaks []Tweak) {
 			BlockHash:   v.BlockHash,
 			BlockHeight: v.BlockHeight,
 			Txid:        "",
-			Data:        data,
+			TweakData:   data,
 		})
 	}
 	return