Skip to content

Commit 3bd218a

Browse files
markopoloparadoxmarkopoloparadox
authored
Fix storage fetch performance (#23)
* Fixed storage fetch performance * Added more storage examples * Updated docs
1 parent 66ae0d9 commit 3bd218a

File tree

8 files changed

+177
-71
lines changed

8 files changed

+177
-71
lines changed

‎documentation/src/installation.md

+4-4
Original file line numberDiff line numberDiff line change
@@ -25,19 +25,19 @@ go version
2525
#### To Existing Project
2626

2727
```bash
28-
# Fetches Avail-GO SDK v0.2.2. This might not be the newest version so make sure to check out the latest github avail-go-sdk release.
28+
# Fetches Avail-GO SDK v0.2.3. This might not be the newest version so make sure to check out the latest github avail-go-sdk release.
2929
# Link to Github: https://github.com/availproject/avail-go-sdk/releases
30-
go get github.com/availproject/[email protected].2
30+
go get github.com/availproject/[email protected].3
3131
```
3232

3333
#### To A New Project
3434

3535
```bash
3636
# Creates a new project with name myproject
3737
go mod init myproject
38-
# Fetches Avail-GO SDK v0.2.2. This might not be the newest version so make sure to check out the latest github avail-go-sdk release.
38+
# Fetches Avail-GO SDK v0.2.3. This might not be the newest version so make sure to check out the latest github avail-go-sdk release.
3939
# Link to Github: https://github.com/availproject/avail-go-sdk/releases
40-
go get github.com/availproject/[email protected].2
40+
go get github.com/availproject/[email protected].3
4141
```
4242

4343
#### First Time Running

‎documentation/src/start.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
# SDK Version
22

3-
This Documentation is based upon avail-go version v0.2.2
3+
This Documentation is based upon avail-go version v0.2.3

‎examples/storage.go

+20
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package examples
33
import (
44
"fmt"
55

6+
daPallet "github.com/availproject/avail-go-sdk/metadata/pallets/data_availability"
67
idenPallet "github.com/availproject/avail-go-sdk/metadata/pallets/identity"
78
staPallet "github.com/availproject/avail-go-sdk/metadata/pallets/staking"
89
sysPallet "github.com/availproject/avail-go-sdk/metadata/pallets/system"
@@ -54,12 +55,30 @@ func RunStorage() {
5455
fmt.Println("Account Free Balance: ", val.Value.AccountData.Free.ToHuman())
5556
}
5657

58+
// Fetch Map Storage 2
59+
{
60+
storage := daPallet.StorageAppKeys{}
61+
value1, err := storage.Fetch(&blockStorage, []byte("ThisShouldNotExist"))
62+
PanicOnError(err)
63+
AssertEq(value1.IsNone(), true, "")
64+
65+
value2, err := storage.Fetch(&blockStorage, []byte("gohan"))
66+
PanicOnError(err)
67+
AssertEq(value2.IsSome(), true, "")
68+
val2 := value2.Unwrap()
69+
70+
fmt.Println("Key: ", string(val2.Key))
71+
fmt.Println("AppId: ", val2.Value.AppId)
72+
fmt.Println("Owner: ", val2.Value.Owner.ToSS58())
73+
}
74+
5775
// Fetch All Map Storage
5876
{
5977
storage := idenPallet.StorageIdentityOf{}
6078
val, err := storage.FetchAll(&blockStorage)
6179
PanicOnError(err)
6280
AssertTrue(len(val) > 0, "There need to be more than 0 values")
81+
AssertEq(len(val), 120, "")
6382

6483
for i := 0; i < len(val); i++ {
6584
fmt.Println("Identity Key: ", val[i].Key.ToHuman())
@@ -94,6 +113,7 @@ func RunStorage() {
94113
val, err := storage.FetchAll(&blockStorage, era)
95114
PanicOnError(err)
96115
AssertTrue(len(val) > 0, "There need to be more than 0 values")
116+
AssertEq(len(val), 80, "")
97117

98118
for i := 0; i < len(val); i++ {
99119
fmt.Println("Era: ", val[i].Key1)

‎interfaces/mod.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
package interfaces
22

3-
import ()
3+
import "github.com/availproject/avail-go-sdk/primitives"
44

55
type EventT interface {
66
PalletIndex() uint8
@@ -10,6 +10,6 @@ type EventT interface {
1010
}
1111

1212
type BlockStorageT interface {
13-
Fetch(storageEntryKey string) (string, error)
13+
Fetch(storageEntryKey string) (primitives.Option[string], error)
1414
FetchKeys(key string) ([]string, error)
1515
}

‎metadata/utils.go

+108-28
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ package metadata
33
import (
44
"strings"
55

6+
"sync"
7+
68
"github.com/availproject/avail-go-sdk/interfaces"
79
prim "github.com/availproject/avail-go-sdk/primitives"
810
)
@@ -161,12 +163,12 @@ func GenericFetch[V any, S StorageT](blockStorage interfaces.BlockStorageT, stor
161163
return prim.None[V](), err
162164
}
163165

164-
if encoded == "" {
166+
if encoded.IsNone() || encoded.Unwrap() == "" {
165167
return prim.None[V](), nil
166168
}
167169

168170
var t V
169-
decoder := prim.NewDecoder(prim.Hex.FromHex(encoded), 0)
171+
decoder := prim.NewDecoder(prim.Hex.FromHex(encoded.Unwrap()), 0)
170172
if err := decoder.Decode(&t); err != nil {
171173
return prim.None[V](), err
172174
}
@@ -181,12 +183,12 @@ func GenericMapFetch[V any, K any, S StorageMapT](blockStorage interfaces.BlockS
181183
return prim.None[StorageEntry[K, V]](), err
182184
}
183185

184-
if encoded == "" {
186+
if encoded.IsNone() || encoded.Unwrap() == "" {
185187
return prim.None[StorageEntry[K, V]](), nil
186188
}
187189

188190
var t V
189-
decoder := prim.NewDecoder(prim.Hex.FromHex(encoded), 0)
191+
decoder := prim.NewDecoder(prim.Hex.FromHex(encoded.Unwrap()), 0)
190192
if err := decoder.Decode(&t); err != nil {
191193
return prim.None[StorageEntry[K, V]](), err
192194
}
@@ -206,23 +208,62 @@ func GenericMapKeysFetch[V any, K any, S StorageMapT](blockStorage interfaces.Bl
206208
return nil, nil
207209
}
208210

211+
// Sync mechanism
212+
errors := []error{}
213+
mu := sync.Mutex{}
214+
wg := sync.WaitGroup{}
215+
209216
storageEntries := []StorageEntry[K, V]{}
210-
for i := range storageKeys {
211-
mapKey, err := storageMapKeyDecode[K](storageKeys[i], storage)
212-
if err != nil {
213-
return nil, err
217+
target := len(storageKeys)
218+
currentIndex := 0
219+
for {
220+
for i := 0; i < 150; i++ {
221+
wg.Add(1)
222+
go func(i int) {
223+
defer wg.Done()
224+
225+
mapKey, err := storageMapKeyDecode[K](storageKeys[i], storage)
226+
if err != nil {
227+
mu.Lock()
228+
errors = append(errors, err)
229+
mu.Unlock()
230+
return
231+
}
232+
233+
value, err := GenericMapFetch[V](blockStorage, mapKey, storage)
234+
if err != nil {
235+
mu.Lock()
236+
errors = append(errors, err)
237+
mu.Unlock()
238+
return
239+
}
240+
241+
if value.IsNone() {
242+
return
243+
}
244+
245+
mu.Lock()
246+
storageEntries = append(storageEntries, value.Unwrap())
247+
mu.Unlock()
248+
}(currentIndex)
249+
250+
currentIndex += 1
251+
if currentIndex >= target {
252+
break
253+
}
214254
}
215255

216-
value, err := GenericMapFetch[V](blockStorage, mapKey, storage)
217-
if err != nil {
218-
return nil, err
256+
wg.Wait()
257+
if len(errors) > 0 {
258+
break
219259
}
220-
221-
if value.IsNone() {
222-
continue
260+
if currentIndex >= (target - 1) {
261+
break
223262
}
263+
}
224264

225-
storageEntries = append(storageEntries, value.Unwrap())
265+
if len(errors) > 0 {
266+
return nil, errors[0]
226267
}
227268

228269
return storageEntries, nil
@@ -235,12 +276,12 @@ func GenericDoubleMapFetch[V any, K1 any, K2 any, S StorageDoubleMapT](blockStor
235276
return prim.None[StorageEntryDoubleMap[K1, K2, V]](), err
236277
}
237278

238-
if encoded == "" {
279+
if encoded.IsNone() || encoded.Unwrap() == "" {
239280
return prim.None[StorageEntryDoubleMap[K1, K2, V]](), nil
240281
}
241282

242283
var t V
243-
decoder := prim.NewDecoder(prim.Hex.FromHex(encoded), 0)
284+
decoder := prim.NewDecoder(prim.Hex.FromHex(encoded.Unwrap()), 0)
244285
if err := decoder.Decode(&t); err != nil {
245286
return prim.None[StorageEntryDoubleMap[K1, K2, V]](), err
246287
}
@@ -260,23 +301,62 @@ func GenericDoubleMapKeysFetch[V any, K1 any, K2 any, S StorageDoubleMapT](block
260301
return nil, nil
261302
}
262303

304+
// Sync mechanism
305+
errors := []error{}
306+
mu := sync.Mutex{}
307+
wg := sync.WaitGroup{}
308+
263309
storageEntries := []StorageEntryDoubleMap[K1, K2, V]{}
264-
for i := range storageKeys {
265-
mapKey1, mapKey2, err := storageDoubleMapKeyDecode[K1, K2](storageKeys[i], storage)
266-
if err != nil {
267-
return nil, err
310+
target := len(storageKeys)
311+
currentIndex := 0
312+
for {
313+
for i := 0; i < 150; i++ {
314+
wg.Add(1)
315+
go func(i int) {
316+
defer wg.Done()
317+
318+
mapKey1, mapKey2, err := storageDoubleMapKeyDecode[K1, K2](storageKeys[i], storage)
319+
if err != nil {
320+
mu.Lock()
321+
errors = append(errors, err)
322+
mu.Unlock()
323+
return
324+
}
325+
326+
value, err := GenericDoubleMapFetch[V](blockStorage, mapKey1, mapKey2, storage)
327+
if err != nil {
328+
mu.Lock()
329+
errors = append(errors, err)
330+
mu.Unlock()
331+
return
332+
}
333+
334+
if value.IsNone() {
335+
return
336+
}
337+
338+
mu.Lock()
339+
storageEntries = append(storageEntries, value.Unwrap())
340+
mu.Unlock()
341+
}(currentIndex)
342+
343+
currentIndex += 1
344+
if currentIndex >= target {
345+
break
346+
}
268347
}
269348

270-
value, err := GenericDoubleMapFetch[V](blockStorage, mapKey1, mapKey2, storage)
271-
if err != nil {
272-
return nil, err
349+
wg.Wait()
350+
if len(errors) > 0 {
351+
break
273352
}
274-
275-
if value.IsNone() {
276-
continue
353+
if currentIndex >= (target - 1) {
354+
break
277355
}
356+
}
278357

279-
storageEntries = append(storageEntries, value.Unwrap())
358+
if len(errors) > 0 {
359+
return nil, errors[0]
280360
}
281361

282362
return storageEntries, nil

‎sdk/client.go

+39-33
Original file line numberDiff line numberDiff line change
@@ -167,39 +167,7 @@ func (this *Client) RequestWithRetry(method string, params string) (string, erro
167167
}
168168

169169
func (this *Client) Request(method string, params string) (prim.Option[string], error) {
170-
rawJSON := []byte(`{
171-
"id": 1,
172-
"jsonrpc": "2.0",
173-
"method": "%s",
174-
"params": %s
175-
}`)
176-
177-
requestBodyString := fmt.Sprintf(string(rawJSON), method, params)
178-
requestBodyBytes := []byte(requestBodyString)
179-
180-
request, err := http.NewRequest("POST", this.endpoint, bytes.NewBuffer(requestBodyBytes))
181-
if err != nil {
182-
return prim.None[string](), err
183-
}
184-
185-
request.Header.Add("Content-Type", "application/json")
186-
response, err := this.client.Do(request)
187-
if err != nil {
188-
return prim.None[string](), newError(err, ErrorCode000)
189-
}
190-
191-
defer response.Body.Close()
192-
193-
responseBodyBytes, _ := io.ReadAll(response.Body)
194-
// fmt.Println("response Status:", response.Status)
195-
// fmt.Println("response Headers:", response.Header)
196-
// fmt.Println("response Body:", string(responseBodyBytes))
197-
198-
if response.StatusCode != http.StatusOK {
199-
err := ErrorCode001
200-
err.Message = fmt.Sprintf(`Status Code: %v`, response.StatusCode)
201-
return prim.None[string](), &err
202-
}
170+
responseBodyBytes, _ := this.RequestRaw(method, params)
203171

204172
var mappedData map[string]interface{}
205173
if err := json.Unmarshal(responseBodyBytes, &mappedData); err != nil {
@@ -236,6 +204,44 @@ func (this *Client) Request(method string, params string) (prim.Option[string],
236204
return prim.Some(result), nil
237205
}
238206

207+
func (this *Client) RequestRaw(method string, params string) ([]byte, error) {
208+
rawJSON := []byte(`{
209+
"id": 1,
210+
"jsonrpc": "2.0",
211+
"method": "%s",
212+
"params": %s
213+
}`)
214+
215+
requestBodyString := fmt.Sprintf(string(rawJSON), method, params)
216+
requestBodyBytes := []byte(requestBodyString)
217+
218+
request, err := http.NewRequest("POST", this.endpoint, bytes.NewBuffer(requestBodyBytes))
219+
if err != nil {
220+
return []byte{}, err
221+
}
222+
223+
request.Header.Add("Content-Type", "application/json")
224+
response, err := this.client.Do(request)
225+
if err != nil {
226+
return []byte{}, newError(err, ErrorCode000)
227+
}
228+
229+
defer response.Body.Close()
230+
231+
responseBodyBytes, _ := io.ReadAll(response.Body)
232+
// fmt.Println("response Status:", response.Status)
233+
// fmt.Println("response Headers:", response.Header)
234+
// fmt.Println("response Body:", string(responseBodyBytes))
235+
236+
if response.StatusCode != http.StatusOK {
237+
err := ErrorCode001
238+
err.Message = fmt.Sprintf(`Status Code: %v`, response.StatusCode)
239+
return []byte{}, &err
240+
}
241+
242+
return responseBodyBytes, nil
243+
}
244+
239245
func (this *Client) Send(tx prim.EncodedExtrinsic) (prim.H256, error) {
240246
return this.Rpc.Author.SubmitExtrinsic(tx.ToHexWith0x())
241247
}

‎sdk/rpc_state.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,14 @@ func (this *stateRPC) GetRuntimeVersion(blockHash prim.Option[prim.H256]) (prim.
2323
return prim.NewRuntimeVersionFromJson(value)
2424
}
2525

26-
func (this *stateRPC) GetStorage(key string, at prim.Option[prim.H256]) (string, error) {
26+
func (this *stateRPC) GetStorage(key string, at prim.Option[prim.H256]) (prim.Option[string], error) {
2727
params := RPCParams{}
2828
params.Add("\"" + key + "\"")
2929
if at.IsSome() {
3030
params.AddH256(at.Unwrap())
3131
}
3232

33-
return this.client.RequestWithRetry("state_getStorage", params.Build())
33+
return this.client.Request("state_getStorage", params.Build())
3434
}
3535

3636
func (this *stateRPC) GetKeys(key string, at prim.Option[prim.H256]) ([]string, error) {

0 commit comments

Comments
 (0)