Add Virtual Database Client + Unit Tests

README.md

@@ -27,13 +27,6 @@ There is also a test program dialout_server_cli for collecting data from dial-ou
 
 The binaries will be installed under $GOPATH/bin/, they may be copied to any SONiC switch and run there.
 
-You can also build a debian package and install it:
-
-    git clone https://github.com/Azure/sonic-telemetry.git
-    pushd sonic-telemetry
-    dpkg-buildpackage -rfakeroot -b -us -uc
-    popd
-
 ### Running
 * See [SONiC gRPC telemetry](./doc/grpc_telemetry.md) for how to run dial-in mode system telemetry server
 * See [SONiC telemetry in dial-out mode](./doc/dialout.md) for how to run dial-out mode system telemetry client

dialout/dialout_client/dialout_client.go

@@ -1,11 +1,14 @@
 package telemetry_dialout
 
 import (
-	// "encoding/json"
 	"crypto/tls"
 	"errors"
 	"fmt"
+
 	spb "github.com/Azure/sonic-telemetry/proto"
+
+	"net"
+
 	sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
 	"github.com/go-redis/redis"
 	log "github.com/golang/glog"
@@ -15,8 +18,7 @@ import (
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
-	"net"
-	//"reflect"
+
 	"strconv"
 	"strings"
 	"sync"

dialout/dialout_client/dialout_client_test.go

@@ -6,20 +6,16 @@ package telemetry_dialout
 import (
 	"crypto/tls"
 	"encoding/json"
+
 	"github.com/go-redis/redis"
-	//"github.com/golang/protobuf/proto"
+
 	testcert "github.com/Azure/sonic-telemetry/testdata/tls"
 
-	//"github.com/kylelemons/godebug/pretty"
-	//"github.com/openconfig/gnmi/client"
 	pb "github.com/openconfig/gnmi/proto/gnmi"
 	"github.com/openconfig/gnmi/value"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
-	//"google.golang.org/grpc/codes"
-	"google.golang.org/grpc/credentials"
-	//"google.golang.org/grpc/status"
-	//"fmt"
+
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -28,6 +24,8 @@ import (
 	"testing"
 	"time"
 
+	"google.golang.org/grpc/credentials"
+
 	sds "github.com/Azure/sonic-telemetry/dialout/dialout_server"
 	spb "github.com/Azure/sonic-telemetry/proto"
 	sdc "github.com/Azure/sonic-telemetry/sonic_data_client"

gnmi_server/client_subscribe.go

@@ -11,8 +11,9 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
 
-	//spb "github.com/Azure/sonic-telemetry/proto"
 	sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
+	vdc "github.com/Azure/sonic-telemetry/virtual_database_client"
+
 	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
 )
 
@@ -119,6 +120,8 @@ func (c *Client) Run(stream gnmipb.GNMI_SubscribeServer) (err error) {
 	var dc sdc.Client
 	if target == "OTHERS" {
 		dc, err = sdc.NewNonDbClient(paths, prefix)
+	} else if target == "SONIC_DB" {
+		dc, err = vdc.NewDbClient(paths, prefix)
 	} else {
 		dc, err = sdc.NewDbClient(paths, prefix)
 	}
@@ -226,6 +229,11 @@ func (c *Client) send(stream gnmipb.GNMI_SubscribeServer) error {
 				c.errors++
 				return err
 			}
+		case vdc.Value:
+			if resp, err = vdc.ValToResp(v); err != nil {
+				c.errors++
+				return err
+			}
 		default:
 			log.V(1).Infof("Unknown data type %v for %s in queue", items[0], c)
 			c.errors++

gnmi_server/server.go

@@ -16,6 +16,8 @@ import (
 	"google.golang.org/grpc/status"
 
 	sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
+	vdc "github.com/Azure/sonic-telemetry/virtual_database_client"
+
 	gnmipb "github.com/openconfig/gnmi/proto/gnmi"
 )
 
@@ -77,6 +79,11 @@ func (srv *Server) Serve() error {
 	return srv.s.Serve(srv.lis)
 }
 
+func (srv *Server) Stop() {
+	s := srv.s
+	s.Stop()
+}
+
 // Address returns the port the Server is listening to.
 func (srv *Server) Address() string {
 	addr := srv.lis.Addr().String()
@@ -174,6 +181,8 @@ func (s *Server) Get(ctx context.Context, req *gnmipb.GetRequest) (*gnmipb.GetRe
 	var dc sdc.Client
 	if target == "OTHERS" {
 		dc, err = sdc.NewNonDbClient(paths, prefix)
+	} else if target == "SONIC_DB" {
+		dc, err = vdc.NewDbClient(paths, prefix)
 	} else {
 		dc, err = sdc.NewDbClient(paths, prefix)
 	}
@@ -186,6 +195,10 @@ func (s *Server) Get(ctx context.Context, req *gnmipb.GetRequest) (*gnmipb.GetRe
 		return nil, status.Error(codes.NotFound, err.Error())
 	}
 
+	if target == "SONIC_DB" {
+		notifications = make([]*gnmipb.Notification, len(spbValues))
+	}
+
 	for index, spbValue := range spbValues {
 		update := &gnmipb.Update{
 			Path: spbValue.GetPath(),

gnmi_server/server_test.go

@@ -6,10 +6,18 @@ package gnmi
 import (
 	"crypto/tls"
 	"encoding/json"
+
 	testcert "github.com/Azure/sonic-telemetry/testdata/tls"
 	"github.com/go-redis/redis"
 	"github.com/golang/protobuf/proto"
 
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"reflect"
+	"testing"
+	"time"
+
 	"github.com/kylelemons/godebug/pretty"
 	"github.com/openconfig/gnmi/client"
 	pb "github.com/openconfig/gnmi/proto/gnmi"
@@ -19,15 +27,11 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/status"
-	"io/ioutil"
-	"os"
-	"os/exec"
-	"reflect"
-	"testing"
-	"time"
+
 	// Register supported client types.
 	spb "github.com/Azure/sonic-telemetry/proto"
 	sdc "github.com/Azure/sonic-telemetry/sonic_data_client"
+
 	gclient "github.com/jipanyang/gnmi/client/gnmi"
 )
 

sonic_data_client/db_client.go

@@ -93,6 +93,7 @@ func (val Value) Compare(other queue.Item) int {
 type DbClient struct {
 	prefix  *gnmipb.Path
 	pathG2S map[*gnmipb.Path][]tablePath
+
 	q       *queue.PriorityQueue
 	channel chan struct{}
 
@@ -112,6 +113,14 @@ func NewDbClient(paths []*gnmipb.Path, prefix *gnmipb.Path) (Client, error) {
 	if UseRedisLocalTcpPort {
 		useRedisTcpClient()
 	}
+
+	// TODO: Remove debug log
+	//for _, _path := range paths {
+	//	fmt.Printf("single path: %v\n", _path)
+	//}
+	//
+	//fmt.Printf("prefix: %v\n", prefix)
+
 	if prefix.GetTarget() == "COUNTERS_DB" {
 		err = initCountersPortNameMap()
 		if err != nil {
@@ -145,7 +154,7 @@ func NewDbClient(paths []*gnmipb.Path, prefix *gnmipb.Path) (Client, error) {
 // String returns the target the client is querying.
 func (c *DbClient) String() string {
 	// TODO: print gnmiPaths of this DbClient
-	return fmt.Sprintf("DbClient Prefix %v  sendMsg %v, recvMsg %v",
+	return fmt.Sprintf("DbClient Prefix %v	sendMsg %v, recvMsg %v",
 		c.prefix.GetTarget(), c.sendMsg, c.recvMsg)
 }
 
@@ -243,6 +252,7 @@ func (c *DbClient) Get(w *sync.WaitGroup) ([]*spb.Value, error) {
 	ts := time.Now()
 	for gnmiPath, tblPaths := range c.pathG2S {
 		val, err := tableData2TypedValue(tblPaths, nil)
+		//log.V(5).Infof("Val: %v\n", val)
 		if err != nil {
 			return nil, err
 		}
@@ -496,6 +506,7 @@ func makeJSON_redis(msi *map[string]interface{}, key *string, op *string, mfv ma
 		for f, v := range mfv {
 			(*msi)[f] = v
 		}
+
 		return nil
 	}
 
@@ -515,12 +526,12 @@ func makeJSON_redis(msi *map[string]interface{}, key *string, op *string, mfv ma
 		of[*op] = fp
 		(*msi)[*key] = of
 	}
+
 	return nil
 }
 
 // emitJSON marshalls map[string]interface{} to JSON byte stream.
 func emitJSON(v *map[string]interface{}) ([]byte, error) {
-	//j, err := json.MarshalIndent(*v, "", indentString)
 	j, err := json.Marshal(*v)
 	if err != nil {
 		return nil, fmt.Errorf("JSON marshalling error: %v", err)
@@ -589,6 +600,7 @@ func tableData2Msi(tblPath *tablePath, useKey bool, op *string, msi *map[string]
 			// Split dbkey string into two parts and second part is key in table
 			keys := strings.SplitN(dbkey, tblPath.delimitor, 2)
 			key = keys[1]
+
 			err = makeJSON_redis(msi, &key, op, fv)
 		}
 		if err != nil {
@@ -597,6 +609,7 @@ func tableData2Msi(tblPath *tablePath, useKey bool, op *string, msi *map[string]
 		}
 		log.V(6).Infof("Added idex %v fv %v ", idx, fv)
 	}
+
 	return nil
 }
 
@@ -644,6 +657,9 @@ func tableData2TypedValue(tblPaths []tablePath, op *string) (*gnmipb.TypedValue,
 			}
 		}
 
+		// Debug logging
+		log.V(5).Infof("tblPath: %v\n", tblPath)
+
 		err := tableData2Msi(&tblPath, useKey, nil, &msi)
 		if err != nil {
 			return nil, err

sonic_data_client/virtual_db.go

@@ -2,8 +2,9 @@ package client
 
 import (
 	"fmt"
-	log "github.com/golang/glog"
 	"strings"
+
+	log "github.com/golang/glog"
 )
 
 // virtual db is to Handle

telemetry/telemetry.go

@@ -11,6 +11,7 @@ import (
 	"google.golang.org/grpc/credentials"
 
 	gnmi "github.com/Azure/sonic-telemetry/gnmi_server"
+
 	testcert "github.com/Azure/sonic-telemetry/testdata/tls"
 )
 

testdata/Interfaces_Port_name_Ethernet68_1_BaseCounter.txt

@@ -0,0 +1,84 @@
+{
+	"SAI_PORT_STAT_ETHER_IN_PKTS_1024_TO_1518_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_128_TO_255_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_1519_TO_2047_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_2048_TO_4095_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_256_TO_511_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_4096_TO_9216_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_512_TO_1023_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_64_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_65_TO_127_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_IN_PKTS_9217_TO_16383_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_1024_TO_1518_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_128_TO_255_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_1519_TO_2047_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_2048_TO_4095_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_256_TO_511_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_4096_TO_9216_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_512_TO_1023_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_64_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_65_TO_127_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_OUT_PKTS_9217_TO_16383_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_RX_OVERSIZE_PKTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_BROADCAST_PKTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_COLLISIONS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_CRC_ALIGN_ERRORS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_DROP_EVENTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_FRAGMENTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_JABBERS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_MULTICAST_PKTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_OVERSIZE_PKTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_1024_TO_1518_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_128_TO_255_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_1519_TO_2047_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_2048_TO_4095_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_256_TO_511_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_4096_TO_9216_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_512_TO_1023_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_64_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_65_TO_127_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_PKTS_9217_TO_16383_OCTETS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_RX_NO_ERRORS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_TX_NO_ERRORS": "0",
+	"SAI_PORT_STAT_ETHER_STATS_UNDERSIZE_PKTS": "0",
+	"SAI_PORT_STAT_ETHER_TX_OVERSIZE_PKTS": "0",
+	"SAI_PORT_STAT_IF_IN_BROADCAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_IN_DISCARDS": "0",
+	"SAI_PORT_STAT_IF_IN_ERRORS": "0",
+	"SAI_PORT_STAT_IF_IN_MULTICAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_IN_NON_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_IN_OCTETS": "0",
+	"SAI_PORT_STAT_IF_IN_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_IN_UNKNOWN_PROTOS": "0",
+	"SAI_PORT_STAT_IF_IN_VLAN_DISCARDS": "0",
+	"SAI_PORT_STAT_IF_OUT_BROADCAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_OUT_DISCARDS": "0",
+	"SAI_PORT_STAT_IF_OUT_ERRORS": "0",
+	"SAI_PORT_STAT_IF_OUT_MULTICAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_OUT_NON_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IF_OUT_OCTETS": "0",
+	"SAI_PORT_STAT_IF_OUT_QLEN": "0",
+	"SAI_PORT_STAT_IF_OUT_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IPV6_IN_DISCARDS": "0",
+	"SAI_PORT_STAT_IPV6_IN_MCAST_PKTS": "0",
+	"SAI_PORT_STAT_IPV6_IN_NON_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IPV6_IN_OCTETS": "0",
+	"SAI_PORT_STAT_IPV6_IN_RECEIVES": "0",
+	"SAI_PORT_STAT_IPV6_IN_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IPV6_OUT_DISCARDS": "0",
+	"SAI_PORT_STAT_IPV6_OUT_MCAST_PKTS": "0",
+	"SAI_PORT_STAT_IPV6_OUT_NON_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IPV6_OUT_OCTETS": "0",
+	"SAI_PORT_STAT_IPV6_OUT_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IP_IN_DISCARDS": "0",
+	"SAI_PORT_STAT_IP_IN_NON_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IP_IN_OCTETS": "0",
+	"SAI_PORT_STAT_IP_IN_RECEIVES": "0",
+	"SAI_PORT_STAT_IP_IN_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IP_OUT_DISCARDS": "0",
+	"SAI_PORT_STAT_IP_OUT_NON_UCAST_PKTS": "0",
+	"SAI_PORT_STAT_IP_OUT_OCTETS": "0",
+	"SAI_PORT_STAT_IP_OUT_UCAST_PKTS": "0"
+}

testdata/Interfaces_Port_name_Ethernet68_1_PfcCounter.txt

@@ -0,0 +1,18 @@
+{
+	"SAI_PORT_STAT_PFC_0_RX_PKTS": "0",
+	"SAI_PORT_STAT_PFC_0_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_1_RX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_1_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_2_RX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_2_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_3_RX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_3_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_4_RX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_4_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_5_RX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_5_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_6_RX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_6_TX_PKTS": "0",	
+	"SAI_PORT_STAT_PFC_7_RX_PKTS": "2",	
+	"SAI_PORT_STAT_PFC_7_TX_PKTS": "0"
+}