From ecd5130a8dddb1c31e6245f109410704892388a3 Mon Sep 17 00:00:00 2001 From: Silas Davis <silas@monax.io> Date: Fri, 4 May 2018 20:02:21 +0100 Subject: [PATCH] Fix test issues, and reorganise CLI UX. Read validator address on startup. Signed-off-by: Silas Davis <silas@monax.io> --- Makefile | 10 +- README.md | 36 +++++-- client/rpc/client_test.go | 14 +-- cmd/burrow/main.go | 187 ++++++++++++++++++++++------------- cmd/burrow/main_test.go | 12 +++ config/config.go | 19 ++-- config/source/source.go | 13 ++- keys/mock/key_client_mock.go | 5 +- scripts/deps/bos.sh | 2 +- 9 files changed, 196 insertions(+), 102 deletions(-) create mode 100644 cmd/burrow/main_test.go diff --git a/Makefile b/Makefile index a9575ca5..87ea9c00 100644 --- a/Makefile +++ b/Makefile @@ -105,8 +105,9 @@ build_race: check build_race_db build_race_client # build burrow .PHONY: build_db build_db: commit_hash - go build -ldflags "-extldflags '-static' -X github.com/hyperledger/burrow/project.commit=$(shell cat commit_hash.txt)"\ - -o ${REPO}/bin/burrow ./cmd/burrow + go build -ldflags "-extldflags '-static' \ + -X github.com/hyperledger/burrow/project.commit=$(shell cat commit_hash.txt)" \ + -o ${REPO}/bin/burrow ./cmd/burrow .PHONY: install_db install_db: build_db @@ -115,8 +116,9 @@ install_db: build_db # build burrow-client .PHONY: build_client build_client: commit_hash - go build -ldflags "-extldflags '-static' -X github.com/hyperledger/burrow/project.commit=$(shell cat commit_hash.txt)"\ - -o ${REPO}/bin/burrow-client ./client/cmd/burrow-client + go build -ldflags "-extldflags '-static' \ + -X github.com/hyperledger/burrow/project.commit=$(shell cat commit_hash.txt)" \ + -o ${REPO}/bin/burrow-client ./client/cmd/burrow-client # build burrow with checks for race conditions .PHONY: build_race_db diff --git a/README.md b/README.md index cd9db0bd..79bd9aa6 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,11 @@ Hyperledger Burrow is a permissioned blockchain node that executes smart contrac - **Application Binary Interface (ABI):** transactions need to be formulated in a binary format that can be processed by the blockchain node. Current tooling provides functionality to compile, deploy and link solidity smart contracts and formulate transactions to call smart contracts on the chain. - **API Gateway:** Burrow exposes REST and JSON-RPC endpoints to interact with the blockchain network and the application state through broadcasting transactions, or querying the current state of the application. Websockets allow to subscribe to events, which is particularly valuable as the consensus engine and smart contract application can give unambiguously finalised results to transactions within one blocktime of about one second. -## Installation +## Project documentation and Roadmap + +Project information generally updated on a quarterly basis can be found on the [Hyperledger Burrow Wiki](https://wiki.hyperledger.org/projects/burrow). +## Installation - [Install go](https://golang.org/doc/install) and have `$GOPATH` set - Ensure you have `gmp` installed (`sudo apt-get install libgmp3-dev || brew install gmp`) @@ -32,35 +35,54 @@ make build This will build the `burrow` and `burrow-client` binaries and put them in the `bin/` directory. They can be executed from there or put wherever is convenient. +You can also install `burrow` into `$GOPATH/bin` with `make install_db`, + ## Usage The end result will be a `burrow.toml` that will be read in from your current working directory when starting `burrow`. ### Configuration -Note: here we'll need `monax-keys` to start the keys server (TODO, where?) +#### Install monax-keys +Monax-keys is our key-signing daemon. In a future release this will be merged with Burrow and will support a GPG backend +in addition to the development mode that monax-keys currently supplies. + +We need to run monax-keys so that `burrow configure` can generate keys for us in the following step. +```shell +# Install monax-keys +go get -u github.com/monax/bosmarmot/keys/cmd/monax-keys +# run monax-keys server in background +monax-keys server & +``` +#### Configure Burrow The quick-and-dirty one-liner looks like: -``` -burrow spec -p1 -f1 | burrow configure -s- -v0 +```shell +# Read spec on stdin +burrow spec -p1 -f1 | burrow configure -s- > burrow.toml ``` which translates into: -``` +```shell +# This is a place we can store config files and burrow's working directory '.burrow' +mkdir chain_dir && cd chain_dir burrow spec --participant-accounts=1 --full-accounts=1 > genesis-spec.json burrow configure --genesis-spec=genesis-spec.json --validator-index=0 > burrow.toml ``` - +#### Run Burrow Once the `burrow.toml` has been created, we run: ``` -burrow +burrow serve ``` and the logs will start streaming through. +If you would like to reset your node you can just delete its working directory with `rm -rf .burrow`. In the context of a +multi-node chain it will resync with peers, otherwise it will restart from height 0. + ### Logging Logging is highly configurable through the `burrow.toml` `[logging]` section. Each log line is a list of key-value pairs that flows from the root sink through possible child sinks. Each sink can have an output, a transform, and sinks that it outputs to. Below is a more involved example of than the one appearing in the default generated config of what you can configure: diff --git a/client/rpc/client_test.go b/client/rpc/client_test.go index d67acb36..d64ab91d 100644 --- a/client/rpc/client_test.go +++ b/client/rpc/client_test.go @@ -55,7 +55,7 @@ func testSend(t *testing.T, nodeClient *mockclient.MockNodeClient, keyClient *mockkeys.MockKeyClient) { // generate an ED25519 key and ripemd160 address - addressString := keyClient.NewKey().String() + addressString := keyClient.NewKey("").String() // Public key can be queried from mockKeyClient.PublicKey(address) // but here we let the transaction factory retrieve the public key // which will then also overwrite the address we provide the function. @@ -63,7 +63,7 @@ func testSend(t *testing.T, // to address in generated transation. publicKeyString := "" // generate an additional address to send amount to - toAddressString := keyClient.NewKey().String() + toAddressString := keyClient.NewKey("").String() // set an amount to transfer amountString := "1000" // unset sequence so that we retrieve sequence from account @@ -80,7 +80,7 @@ func testCall(t *testing.T, nodeClient *mockclient.MockNodeClient, keyClient *mockkeys.MockKeyClient) { // generate an ED25519 key and ripemd160 address - addressString := keyClient.NewKey().String() + addressString := keyClient.NewKey("").String() // Public key can be queried from mockKeyClient.PublicKey(address) // but here we let the transaction factory retrieve the public key // which will then also overwrite the address we provide the function. @@ -88,7 +88,7 @@ func testCall(t *testing.T, // to address in generated transation. publicKeyString := "" // generate an additional address to send amount to - toAddressString := keyClient.NewKey().String() + toAddressString := keyClient.NewKey("").String() // set an amount to transfer amountString := "1000" // unset sequence so that we retrieve sequence from account @@ -113,7 +113,7 @@ func testName(t *testing.T, nodeClient *mockclient.MockNodeClient, keyClient *mockkeys.MockKeyClient) { // generate an ED25519 key and ripemd160 address - addressString := keyClient.NewKey().String() + addressString := keyClient.NewKey("").String() // Public key can be queried from mockKeyClient.PublicKey(address) // but here we let the transaction factory retrieve the public key // which will then also overwrite the address we provide the function. @@ -144,7 +144,7 @@ func testPermissions(t *testing.T, nodeClient *mockclient.MockNodeClient, keyClient *mockkeys.MockKeyClient) { // generate an ED25519 key and ripemd160 address - addressString := keyClient.NewKey().String() + addressString := keyClient.NewKey("").String() // Public key can be queried from mockKeyClient.PublicKey(address) // but here we let the transaction factory retrieve the public key // which will then also overwrite the address we provide the function. @@ -152,7 +152,7 @@ func testPermissions(t *testing.T, // to address in generated transation. publicKeyString := "" // generate an additional address to set permissions for - permAddressString := keyClient.NewKey().String() + permAddressString := keyClient.NewKey("").String() // unset sequence so that we retrieve sequence from account sequenceString := "" diff --git a/cmd/burrow/main.go b/cmd/burrow/main.go index d1fa13f4..f7d812b4 100644 --- a/cmd/burrow/main.go +++ b/cmd/burrow/main.go @@ -3,11 +3,11 @@ package main import ( "context" "fmt" + "io/ioutil" "os" "strings" - "io/ioutil" - + acm "github.com/hyperledger/burrow/account" "github.com/hyperledger/burrow/config" "github.com/hyperledger/burrow/config/source" "github.com/hyperledger/burrow/execution" @@ -23,58 +23,116 @@ import ( ) func main() { - burrow := cli.App("burrow", "The EVM smart contract machine with Tendermint consensus") - - genesisOpt := burrow.StringOpt("g genesis", "", - "Use the specified genesis JSON file rather than a key in the main config, use - to read from STDIN") - - configOpt := burrow.StringOpt("c config", "", - "Use the a specified burrow config TOML file") + burrow().Run(os.Args) +} - versionOpt := burrow.BoolOpt("v version", false, "Print the Burrow version") +func burrow() *cli.Cli { + app := cli.App("burrow", "The EVM smart contract machine with Tendermint consensus") - burrow.Spec = "[--config=<config file>] [--genesis=<genesis json file>] [--version]" + versionOpt := app.BoolOpt("v version", false, "Print the Burrow version") + app.Spec = "[--version]" - burrow.Action = func() { + app.Action = func() { if *versionOpt { fmt.Println(project.FullVersion()) os.Exit(0) } - // We need to reflect on whether this obscures where values are coming from - conf := config.DefaultBurrowConfig() - // We treat logging a little differently in that if anything is set for logging we will not - // set default outputs - conf.Logging = nil - err := source.EachOf( - burrowConfigProvider(*configOpt), - source.FirstOf( - genesisDocProvider(*genesisOpt, false), - // Try working directory - genesisDocProvider(config.DefaultGenesisDocJSONFileName, true)), - ).Apply(conf) - // If no logging config was provided use the default - if conf.Logging == nil { - conf.Logging = logging_config.DefaultNodeLoggingConfig() - } - if err != nil { - fatalf("could not obtain config: %v", err) - } + } - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - kern, err := conf.Kernel(ctx) - if err != nil { - fatalf("could not create Burrow kernel: %v", err) - } + app.Command("serve", "", + func(cmd *cli.Cmd) { + genesisOpt := cmd.StringOpt("g genesis", "", + "Use the specified genesis JSON file rather than a key in the main config, use - to read from STDIN") + + configOpt := cmd.StringOpt("c config", "", "Use the a specified burrow config file") + + validatorIndexOpt := cmd.Int(cli.IntOpt{ + Name: "v validator-index", + Desc: "Validator index (in validators list - GenesisSpec or GenesisDoc) from which to set ValidatorAddress", + Value: -1, + EnvVar: "BURROW_VALIDATOR_INDEX", + }) + + validatorAddressOpt := cmd.String(cli.StringOpt{ + Name: "a validator-address", + Desc: "The address of the the signing key of this validator", + EnvVar: "BURROW_VALIDATOR_ADDRESS", + }) + + validatorPassphraseOpt := cmd.String(cli.StringOpt{ + Name: "p validator-passphrase", + Desc: "The passphrase of the signing key of this validator (currently unimplemented but planned for future version of our KeyClient interface)", + EnvVar: "BURROW_VALIDATOR_PASSPHRASE", + }) + + cmd.Spec = "[--config=<config file>] " + + "[--validator-index=<index of validator in GenesisDoc> | --validator-address=<address of validator signing key>] " + + "[--genesis=<genesis json file>]" - err = kern.Boot() - if err != nil { - fatalf("could not boot Burrow kernel: %v", err) - } - kern.WaitForShutdown() - } + cmd.Action = func() { - burrow.Command("spec", + // We need to reflect on whether this obscures where values are coming from + conf := config.DefaultBurrowConfig() + // We treat logging a little differently in that if anything is set for logging we will not + // set default outputs + conf.Logging = nil + err := source.EachOf( + burrowConfigProvider(*configOpt), + source.FirstOf( + genesisDocProvider(*genesisOpt, false), + // Try working directory + genesisDocProvider(config.DefaultGenesisDocJSONFileName, true)), + ).Apply(conf) + + // If no logging config was provided use the default + if conf.Logging == nil { + conf.Logging = logging_config.DefaultNodeLoggingConfig() + } + if err != nil { + fatalf("could not obtain config: %v", err) + } + + // Which validator am I? + if *validatorAddressOpt != "" { + address, err := acm.AddressFromHexString(*validatorAddressOpt) + if err != nil { + fatalf("could not read address for validator in '%s'", *validatorAddressOpt) + } + conf.ValidatorAddress = &address + } else if *validatorIndexOpt > -1 { + if conf.GenesisDoc == nil { + fatalf("Unable to set ValidatorAddress from provided validator-index since no " + + "GenesisDoc/GenesisSpec provided.") + } + if *validatorIndexOpt >= len(conf.GenesisDoc.Validators) { + fatalf("validator-index of %v given but only %v validators specified in GenesisDoc", + *validatorIndexOpt, len(conf.GenesisDoc.Validators)) + } + conf.ValidatorAddress = &conf.GenesisDoc.Validators[*validatorIndexOpt].Address + printf("Using validator index %v (address: %s)", *validatorIndexOpt, *conf.ValidatorAddress) + } + + if *validatorPassphraseOpt != "" { + conf.ValidatorPassphrase = validatorPassphraseOpt + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + kern, err := conf.Kernel(ctx) + if err != nil { + fatalf("could not create Burrow kernel: %v", err) + } + + err = kern.Boot() + if err != nil { + fatalf("could not boot Burrow kernel: %v", err) + } + kern.WaitForShutdown() + } + }) + + app.Command("spec", "Build a GenesisSpec that acts as a template for a GenesisDoc and the configure command", func(cmd *cli.Cmd) { tomlOpt := cmd.BoolOpt("t toml", false, "Emit GenesisSpec as TOML rather than the "+ @@ -131,7 +189,7 @@ func main() { } }) - burrow.Command("configure", + app.Command("configure", "Create Burrow configuration by consuming a GenesisDoc or GenesisSpec, creating keys, and emitting the config", func(cmd *cli.Cmd) { genesisSpecOpt := cmd.StringOpt("s genesis-spec", "", @@ -143,6 +201,8 @@ func main() { keysUrlOpt := cmd.StringOpt("k keys-url", "", fmt.Sprintf("Provide keys URL, default: %s", keys.DefaultKeysConfig().URL)) + configOpt := cmd.StringOpt("c base-config", "", "Use the a specified burrow config file as a base") + genesisDocOpt := cmd.StringOpt("g genesis-doc", "", "GenesisDoc in JSON or TOML to embed in config") generateKeysOpt := cmd.StringOpt("x generate-keys", "", @@ -154,10 +214,7 @@ func main() { "file specified with --generate-keys. Default:\n%s", mock.LeftTemplateDelim, mock.RightTemplateDelim, mock.DefaultDumpKeysFormat)) - separateGenesisDoc := cmd.StringOpt("s separate-genesis-doc", "", "Emit a separate genesis doc as JSON or TOML") - - validatorIndexOpt := cmd.IntOpt("v validator-index", -1, - "Validator index (in validators list - GenesisSpec or GenesisDoc) from which to set ValidatorAddress") + separateGenesisDoc := cmd.StringOpt("w separate-genesis-doc", "", "Emit a separate genesis doc as JSON or TOML") loggingOpt := cmd.StringOpt("l logging", "", "Comma separated list of logging instructions which form a 'program' which is a depth-first "+ @@ -174,7 +231,7 @@ func main() { cmd.Spec = "[--keys-url=<keys URL> | (--generate-keys=<secret keys files> [--keys-template=<text template for each key>])] " + "[--genesis-spec=<GenesisSpec file> | --genesis-doc=<GenesisDoc file>] " + - "[--separate-genesis-doc=<genesis JSON file>] [--validator-index=<index>] [--chain-name] [--json-out] " + + "[--separate-genesis-doc=<genesis JSON file>] [--chain-name] [--json-out] " + "[--logging=<logging program>] [--describe-logging] [--debug]" cmd.Action = func() { @@ -208,11 +265,14 @@ func main() { genesisSpec := new(spec.GenesisSpec) err := source.FromFile(*genesisSpecOpt, genesisSpec) if err != nil { - fatalf("could not read GenesisSpec: %v", err) + fatalf("Could not read GenesisSpec: %v", err) } if *generateKeysOpt != "" { keyClient := mock.NewMockKeyClient() conf.GenesisDoc, err = genesisSpec.GenesisDoc(keyClient) + if err != nil { + fatalf("Could not generate GenesisDoc from GenesisSpec using MockKeyClient: %v", err) + } secretKeysString, err := keyClient.DumpKeys(*keysTemplateOpt) if err != nil { @@ -237,22 +297,6 @@ func main() { conf.GenesisDoc = genesisDoc } - // Which validator am I? - if *validatorIndexOpt > -1 { - if conf.GenesisDoc == nil { - fatalf("Unable to set ValidatorAddress from provided validator-index since no " + - "GenesisDoc/GenesisSpec provided.") - } - if len(conf.GenesisDoc.Validators) < *validatorIndexOpt { - fatalf("validator-index of %v given but only %v validators specified in GenesisDoc", - *validatorIndexOpt, len(conf.GenesisDoc.Validators)) - } - conf.ValidatorAddress = &conf.GenesisDoc.Validators[*validatorIndexOpt].Address - } else if conf.GenesisDoc != nil && len(conf.GenesisDoc.Validators) > 0 { - // Pick first validator otherwise - might want to change this when we support non-validating node - conf.ValidatorAddress = &conf.GenesisDoc.Validators[0].Address - } - // Logging if *loggingOpt != "" { ops := strings.Split(*loggingOpt, ",") @@ -301,7 +345,7 @@ func main() { } }) - burrow.Command("help", + app.Command("help", "Get more detailed or exhaustive options of selected commands or flags.", func(cmd *cli.Cmd) { @@ -310,7 +354,8 @@ func main() { cmd.Action = func() { } }) - burrow.Run(os.Args) + + return app } // Print informational output to Stderr @@ -342,8 +387,8 @@ func genesisDocProvider(genesisFile string, skipNonExistent bool) source.ConfigP return fmt.Errorf("config passed was not BurrowConfig") } if conf.GenesisDoc != nil { - return fmt.Errorf("sourcing GenesisDoc from file, but GenesisDoc was defined earlier " + - "in config cascade, only specify GenesisDoc in one place") + return fmt.Errorf("sourcing GenesisDoc from file %v, but GenesisDoc was defined in earlier "+ + "config source, only specify GenesisDoc in one place", genesisFile) } genesisDoc := new(genesis.GenesisDoc) err := source.FromFile(genesisFile, genesisDoc) diff --git a/cmd/burrow/main_test.go b/cmd/burrow/main_test.go new file mode 100644 index 00000000..c2690fd2 --- /dev/null +++ b/cmd/burrow/main_test.go @@ -0,0 +1,12 @@ +package main + +import "testing" + +func TestBurrow(t *testing.T) { + app := burrow() + // Basic smoke test for cli config + app.Run([]string{"--version"}) + app.Run([]string{"spec"}) + app.Run([]string{"configure"}) + app.Run([]string{"serve"}) +} diff --git a/config/config.go b/config/config.go index fa39dba4..fa0e3227 100644 --- a/config/config.go +++ b/config/config.go @@ -23,13 +23,16 @@ const DefaultBurrowConfigJSONEnvironmentVariable = "BURROW_CONFIG_JSON" const DefaultGenesisDocJSONFileName = "genesis.json" type BurrowConfig struct { - ValidatorAddress *acm.Address `json:",omitempty" toml:",omitempty"` - GenesisDoc *genesis.GenesisDoc `json:",omitempty" toml:",omitempty"` - Tendermint *tendermint.BurrowTendermintConfig `json:",omitempty" toml:",omitempty"` - Execution *execution.ExecutionConfig `json:",omitempty" toml:",omitempty"` - Keys *keys.KeysConfig `json:",omitempty" toml:",omitempty"` - RPC *rpc.RPCConfig `json:",omitempty" toml:",omitempty"` - Logging *logging_config.LoggingConfig `json:",omitempty" toml:",omitempty"` + // Set on startup + ValidatorAddress *acm.Address `json:",omitempty" toml:",omitempty"` + ValidatorPassphrase *string `json:",omitempty" toml:",omitempty"` + // From config file + GenesisDoc *genesis.GenesisDoc `json:",omitempty" toml:",omitempty"` + Tendermint *tendermint.BurrowTendermintConfig `json:",omitempty" toml:",omitempty"` + Execution *execution.ExecutionConfig `json:",omitempty" toml:",omitempty"` + Keys *keys.KeysConfig `json:",omitempty" toml:",omitempty"` + RPC *rpc.RPCConfig `json:",omitempty" toml:",omitempty"` + Logging *logging_config.LoggingConfig `json:",omitempty" toml:",omitempty"` } func DefaultBurrowConfig() *BurrowConfig { @@ -46,7 +49,7 @@ func (conf *BurrowConfig) Kernel(ctx context.Context) (*core.Kernel, error) { return nil, fmt.Errorf("no GenesisDoc defined in config, cannot make Kernel") } if conf.ValidatorAddress == nil { - return nil, fmt.Errorf("no validator address in config, cannot make Kernel") + return nil, fmt.Errorf("no validator address provided, cannot make Kernel") } logger, err := lifecycle.NewLoggerFromLoggingConfig(conf.Logging) if err != nil { diff --git a/config/source/source.go b/config/source/source.go index 16a1ad78..4b35b901 100644 --- a/config/source/source.go +++ b/config/source/source.go @@ -10,10 +10,11 @@ import ( "reflect" "strings" + "regexp" + "github.com/BurntSushi/toml" "github.com/cep21/xdgbasedir" "github.com/imdario/mergo" - "regexp" ) // If passed this identifier try to read config from STDIN @@ -40,7 +41,7 @@ const ( Unknown Format = "" ) -var jsonRegex = regexp.MustCompile(`\s*{`) +var jsonRegex = regexp.MustCompile(`^\s*{`) type configSource struct { from string @@ -130,9 +131,15 @@ func EachOf(providers ...ConfigProvider) *configSource { // Try to source config from provided file detecting the file format, is skipNonExistent is true then the provider will // fall-through (skip) when the file doesn't exist, rather than returning an error func File(configFile string, skipNonExistent bool) *configSource { + var from string + if configFile == STDINFileIdentifier { + from = "Config from STDIN" + } else { + from = fmt.Sprintf("Config file at '%s'", configFile) + } return &configSource{ skip: ShouldSkipFile(configFile, skipNonExistent), - from: fmt.Sprintf("JSON config file at '%s'", configFile), + from: from, apply: func(baseConfig interface{}) error { return FromFile(configFile, baseConfig) }, diff --git a/keys/mock/key_client_mock.go b/keys/mock/key_client_mock.go index 1faa65e2..d2c78231 100644 --- a/keys/mock/key_client_mock.go +++ b/keys/mock/key_client_mock.go @@ -25,11 +25,11 @@ import ( acm "github.com/hyperledger/burrow/account" . "github.com/hyperledger/burrow/keys" + "github.com/pkg/errors" "github.com/tendermint/ed25519" crypto "github.com/tendermint/go-crypto" "github.com/tmthrgd/go-hex" "golang.org/x/crypto/ripemd160" - "github.com/pkg/errors" ) //--------------------------------------------------------------------- @@ -84,6 +84,9 @@ func newMockKey(name string) (*MockKey, error) { if err != nil { return nil, err } + if key.Name == "" { + key.Name = key.Address.String() + } return key, nil } diff --git a/scripts/deps/bos.sh b/scripts/deps/bos.sh index 550f98b0..ba42f37f 100755 --- a/scripts/deps/bos.sh +++ b/scripts/deps/bos.sh @@ -1,4 +1,4 @@ #!/usr/bin/env bash # The git revision of Bosmarmot/bos we will build and install into ./bin/ for integration tests -echo "b6f5208284f54006b22955b8fcd5c86df1675c7f" +echo "520a381e112f108e0ce71f4e6bc1bb8e3fde236f" \ No newline at end of file -- GitLab