Add burrow dump command

backed by new forensics package containing block explorer

Signed-off-by: Silas Davis <silas@monax.io>

Gopkg.lock

@@ -470,6 +470,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "2a58c6e6ec7d2792452873c5cb57032cc6d1a4750345647f562a8c9e8bc1e0b4"
+  inputs-digest = "53146785a49a1d6940ce537262f1ff17196fed4244188f33c2f14342fadac88a"
   solver-name = "gps-cdcl"
   solver-version = 1

cmd/burrow/commands/dump.go

+package commands
+
+import (
+	"encoding/json"
+
+	"github.com/hyperledger/burrow/forensics"
+	"github.com/hyperledger/burrow/txs"
+	cli "github.com/jawher/mow.cli"
+	"github.com/tendermint/tmlibs/db"
+)
+
+func Dump(output Output) func(cmd *cli.Cmd) {
+	return func(dump *cli.Cmd) {
+		configOpt := dump.StringOpt("c config", "", "Use the a specified burrow config file")
+
+		var explorer *forensics.BlockExplorer
+
+		dump.Before = func() {
+			conf, err := obtainBurrowConfig(*configOpt, "")
+			if err != nil {
+				output.Fatalf("Could not obtain config: %v", err)
+			}
+			tmConf := conf.Tendermint.TendermintConfig()
+
+			explorer = forensics.NewBlockExplorer(db.DBBackendType(tmConf.DBBackend), tmConf.DBDir())
+		}
+
+		dump.Command("blocks", "dump blocks to stdout", func(cmd *cli.Cmd) {
+			rangeArg := cmd.StringArg("RANGE", "", "Range as START_HEIGHT:END_HEIGHT where omitting "+
+				"either endpoint implicitly describes the start/end and a negative index counts back from the last block")
+
+			cmd.Spec = "[RANGE]"
+
+			cmd.Action = func() {
+				start, end, err := parseRange(*rangeArg)
+
+				_, err = explorer.Blocks(start, end,
+					func(block *forensics.Block) (stop bool) {
+						bs, err := json.Marshal(block)
+						if err != nil {
+							output.Fatalf("Could not serialise block: %v", err)
+						}
+						output.Printf(string(bs))
+						return false
+					})
+				if err != nil {
+					output.Fatalf("Error iterating over blocks: %v", err)
+				}
+			}
+		})
+
+		dump.Command("txs", "dump transactions to stdout", func(cmd *cli.Cmd) {
+			rangeArg := cmd.StringArg("RANGE", "", "Range as START_HEIGHT:END_HEIGHT where omitting "+
+				"either endpoint implicitly describes the start/end and a negative index counts back from the last block")
+
+			cmd.Spec = "[RANGE]"
+
+			cmd.Action = func() {
+				start, end, err := parseRange(*rangeArg)
+
+				_, err = explorer.Blocks(start, end,
+					func(block *forensics.Block) (stop bool) {
+						stopped, err := block.Transactions(func(tx txs.Tx) (stop bool) {
+							bs, err := json.Marshal(tx)
+							if err != nil {
+								output.Fatalf("Could not deserialise transaction: %v", err)
+							}
+							output.Printf(string(bs))
+							return false
+						})
+						if err != nil {
+							output.Fatalf("Error iterating over transactions: %v", err)
+						}
+						// If we stopped transactions stop everything
+						return stopped
+					})
+				if err != nil {
+					output.Fatalf("Error iterating over blocks: %v", err)
+				}
+			}
+		})
+	}
+}

cmd/burrow/commands/helpers.go

@@ -3,9 +3,13 @@ package commands
 import (
 	"fmt"
 
+	"strconv"
+	"strings"
+
 	"github.com/hyperledger/burrow/config"
 	"github.com/hyperledger/burrow/config/source"
 	"github.com/hyperledger/burrow/genesis"
+	logging_config "github.com/hyperledger/burrow/logging/config"
 )
 
 type Output interface {
@@ -14,6 +18,29 @@ type Output interface {
 	Fatalf(format string, args ...interface{})
 }
 
+func obtainBurrowConfig(configFile, genesisDocFile string) (*config.BurrowConfig, error) {
+	// We need to reflect on whether this obscures where values are coming from
+	conf := config.DefaultBurrowConfig()
+	// We treat logging a little differently in that if anything is set for logging we will not
+	// set default outputs
+	conf.Logging = nil
+	err := source.EachOf(
+		burrowConfigProvider(configFile),
+		source.FirstOf(
+			genesisDocProvider(genesisDocFile, false),
+			// Try working directory
+			genesisDocProvider(config.DefaultGenesisDocJSONFileName, true)),
+	).Apply(conf)
+	if err != nil {
+		return nil, err
+	}
+	// If no logging config was provided use the default
+	if conf.Logging == nil {
+		conf.Logging = logging_config.DefaultNodeLoggingConfig()
+	}
+	return conf, nil
+}
+
 func burrowConfigProvider(configFile string) source.ConfigProvider {
 	return source.FirstOf(
 		// Will fail if file doesn't exist, but still skipped it configFile == ""
@@ -45,3 +72,32 @@ func genesisDocProvider(genesisFile string, skipNonExistent bool) source.ConfigP
 			return nil
 		})
 }
+
+func parseRange(rangeString string) (start int64, end int64, err error) {
+	start = 0
+	end = -1
+
+	if rangeString == "" {
+		return
+	}
+
+	bounds := strings.Split(rangeString, ":")
+	if len(bounds) == 1 {
+		startString := bounds[0]
+		start, err = strconv.ParseInt(startString, 10, 64)
+		return
+	}
+	if len(bounds) == 2 {
+		if bounds[0] != "" {
+			start, err = strconv.ParseInt(bounds[0], 10, 64)
+			if err != nil {
+				return
+			}
+		}
+		if bounds[1] != "" {
+			end, err = strconv.ParseInt(bounds[1], 10, 64)
+		}
+		return
+	}
+	return 0, 0, fmt.Errorf("could not parse range from %s", rangeString)
+}

cmd/burrow/commands/helpers_test.go

+package commands
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestParseRange(t *testing.T) {
+	start, end, err := parseRange("")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), start)
+	assert.Equal(t, int64(-1), end)
+
+	start, end, err = parseRange(":")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), start)
+	assert.Equal(t, int64(-1), end)
+
+	start, end, err = parseRange("0:")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), start)
+	assert.Equal(t, int64(-1), end)
+
+	start, end, err = parseRange(":-1")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), start)
+	assert.Equal(t, int64(-1), end)
+
+	start, end, err = parseRange("0:-1")
+	require.NoError(t, err)
+	assert.Equal(t, int64(0), start)
+	assert.Equal(t, int64(-1), end)
+
+	start, end, err = parseRange("123123:-123")
+	require.NoError(t, err)
+	assert.Equal(t, int64(123123), start)
+	assert.Equal(t, int64(-123), end)
+}

cmd/burrow/commands/start.go

@@ -3,10 +3,7 @@ package commands
 import (
 	"context"
 
-	"github.com/hyperledger/burrow/config"
-	"github.com/hyperledger/burrow/config/source"
 	"github.com/hyperledger/burrow/crypto"
-	logging_config "github.com/hyperledger/burrow/logging/config"
 	"github.com/jawher/mow.cli"
 )
 
@@ -47,24 +44,7 @@ func Start(output Output) func(cmd *cli.Cmd) {
 			"[--genesis=<genesis json file>]"
 
 		cmd.Action = func() {
-
-			// We need to reflect on whether this obscures where values are coming from
-			conf := config.DefaultBurrowConfig()
-			// We treat logging a little differently in that if anything is set for logging we will not
-			// set default outputs
-			conf.Logging = nil
-			err := source.EachOf(
-				burrowConfigProvider(*configOpt),
-				source.FirstOf(
-					genesisDocProvider(*genesisOpt, false),
-					// Try working directory
-					genesisDocProvider(config.DefaultGenesisDocJSONFileName, true)),
-			).Apply(conf)
-
-			// If no logging config was provided use the default
-			if conf.Logging == nil {
-				conf.Logging = logging_config.DefaultNodeLoggingConfig()
-			}
+			conf, err := obtainBurrowConfig(*configOpt, *genesisOpt)
 			if err != nil {
 				output.Fatalf("could not obtain config: %v", err)
 			}

cmd/burrow/main.go

@@ -39,6 +39,9 @@ func burrow(output commands.Output) *cli.Cli {
 	app.Command("keys", "A tool for doing a bunch of cool stuff with keys",
 		commands.Keys(output))
 
+	app.Command("dump", "Dump objects from an offline Burrow .burrow directory",
+		commands.Dump(output))
+
 	return app
 }
 

consensus/tendermint/tendermint.go

@@ -33,9 +33,13 @@ type Node struct {
 	}
 }
 
+func DBProvider(ID string, backendType dbm.DBBackendType, dbDir string) dbm.DB {
+	return dbm.NewDB(ID, backendType, dbDir)
+}
+
 // Since Tendermint doesn't close its DB connections
 func (n *Node) DBProvider(ctx *node.DBContext) (dbm.DB, error) {
-	db := dbm.NewDB(ctx.ID, dbm.DBBackendType(ctx.Config.DBBackend), ctx.Config.DBDir())
+	db := DBProvider(ctx.ID, dbm.DBBackendType(ctx.Config.DBBackend), ctx.Config.DBDir())
 	n.closers = append(n.closers, db)
 	return db, nil
 }

forensics/block.go

+package forensics
+
+import (
+	"github.com/hyperledger/burrow/txs"
+	"github.com/tendermint/tendermint/types"
+)
+
+type Block struct {
+	txDecoder txs.Decoder
+	*types.Block
+}
+
+func NewBlock(txDecoder txs.Decoder, block *types.Block) *Block {
+	return &Block{
+		txDecoder: txDecoder,
+		Block:     block,
+	}
+}
+
+func (b *Block) Transactions(iter func(txs.Tx) (stop bool)) (stopped bool, err error) {
+	for i := 0; i < len(b.Txs); i++ {
+		tx, err := b.txDecoder.DecodeTx(b.Txs[i])
+		if err != nil {
+			return false, err
+		}
+		if iter(tx) {
+			return true, nil
+		}
+	}
+	return false, nil
+}

forensics/block_explorer.go

+package forensics
+
+import (
+	"fmt"
+
+	"github.com/hyperledger/burrow/consensus/tendermint"
+	"github.com/hyperledger/burrow/txs"
+	"github.com/tendermint/tendermint/blockchain"
+	"github.com/tendermint/tmlibs/db"
+)
+
+type BlockExplorer struct {
+	txDecoder txs.Decoder
+	*blockchain.BlockStore
+}
+
+func NewBlockExplorer(dbBackendType db.DBBackendType, dbDir string) *BlockExplorer {
+	return &BlockExplorer{
+		txDecoder:  txs.NewGoWireCodec(),
+		BlockStore: blockchain.NewBlockStore(tendermint.DBProvider("blockstore", dbBackendType, dbDir)),
+	}
+}
+
+func (be *BlockExplorer) Block(height int64) (block *Block, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			err = fmt.Errorf("could not get block at height %v: %v", height, r)
+		}
+	}()
+
+	tmBlock := be.LoadBlock(height)
+	if tmBlock == nil {
+		return nil, fmt.Errorf("could not pull block at height: %v", height)
+	}
+	return NewBlock(be.txDecoder, tmBlock), nil
+}
+
+// Iterate over blocks between start (inclusive) and end (exclusive)
+func (be *BlockExplorer) Blocks(start, end int64, iter func(*Block) (stop bool)) (stopped bool, err error) {
+	if end > 0 && start >= end {
+		return false, fmt.Errorf("end height must be strictly greater than start height")
+	}
+	if start <= 0 {
+		// From first block
+		start = 1
+	}
+	if end < 0 {
+		// -1 means include the very last block so + 1 for offset
+		end = be.Height() + end + 1
+	}
+
+	for height := start; height <= end; height++ {
+		block, err := be.Block(height)
+		if err != nil {
+			return false, err
+		}
+		if iter(block) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}