Skip to content

Commit 24b9e84

Browse files
fedekunzealexanderbez
fedekunze
authored and
alexanderbez
committed
Merge PR #4784: JSON representation of event stats
1 parent 0ba74bb commit 24b9e84

File tree

13 files changed

+164
-149
lines changed

13 files changed

+164
-149
lines changed

‎.clog.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,4 @@ tags:
1111
- rest
1212
- cli
1313
- modules
14+
- simulation

‎.pending/improvements/simulation/_4670-JSON-sim-stats

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
#4670 Update simulation statistics to JSON format
2+
- Support exporting the simulation stats to a given JSON file

‎simapp/sim_test.go

+9-8
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ func init() {
3636
flag.StringVar(&exportParamsPath, "ExportParamsPath", "", "custom file path to save the exported params JSON")
3737
flag.IntVar(&exportParamsHeight, "ExportParamsHeight", 0, "height to which export the randomly generated params")
3838
flag.StringVar(&exportStatePath, "ExportStatePath", "", "custom file path to save the exported app state JSON")
39+
flag.StringVar(&exportStatsPath, "ExportStatsPath", "", "custom file path to save the exported simulation statistics JSON")
3940
flag.Int64Var(&seed, "Seed", 42, "simulation random seed")
4041
flag.IntVar(&numBlocks, "NumBlocks", 500, "number of blocks")
4142
flag.IntVar(&blockSize, "BlockSize", 200, "operations per block")
@@ -52,15 +53,15 @@ func init() {
5253
// helper function for populating input for SimulateFromSeed
5354
func getSimulateFromSeedInput(tb testing.TB, w io.Writer, app *SimApp) (
5455
testing.TB, io.Writer, *baseapp.BaseApp, simulation.AppStateFn, int64,
55-
simulation.WeightedOperations, sdk.Invariants, int, int, int,
56+
simulation.WeightedOperations, sdk.Invariants, int, int, int, string,
5657
bool, bool, bool, bool, bool, map[string]bool) {
5758

5859
exportParams := exportParamsPath != ""
5960

6061
return tb, w, app.BaseApp, appStateFn, seed,
6162
testAndRunTxs(app), invariants(app),
6263
numBlocks, exportParamsHeight, blockSize,
63-
exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
64+
exportStatsPath, exportParams, commit, lean, onOperation, allInvariants, app.ModuleAccountAddrs()
6465
}
6566

6667
func appStateFn(
@@ -412,7 +413,7 @@ func BenchmarkFullAppSimulation(b *testing.B) {
412413
}
413414

414415
if commit {
415-
fmt.Println("GoLevelDB Stats")
416+
fmt.Println("\nGoLevelDB Stats")
416417
fmt.Println(db.Stats()["leveldb.stats"])
417418
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
418419
}
@@ -471,7 +472,7 @@ func TestFullAppSimulation(t *testing.T) {
471472
if commit {
472473
// for memdb:
473474
// fmt.Println("Database Size", db.Stats()["database.size"])
474-
fmt.Println("GoLevelDB Stats")
475+
fmt.Println("\nGoLevelDB Stats")
475476
fmt.Println(db.Stats()["leveldb.stats"])
476477
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
477478
}
@@ -528,7 +529,7 @@ func TestAppImportExport(t *testing.T) {
528529
if commit {
529530
// for memdb:
530531
// fmt.Println("Database Size", db.Stats()["database.size"])
531-
fmt.Println("GoLevelDB Stats")
532+
fmt.Println("\nGoLevelDB Stats")
532533
fmt.Println(db.Stats()["leveldb.stats"])
533534
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
534535
}
@@ -644,7 +645,7 @@ func TestAppSimulationAfterImport(t *testing.T) {
644645
if commit {
645646
// for memdb:
646647
// fmt.Println("Database Size", db.Stats()["database.size"])
647-
fmt.Println("GoLevelDB Stats")
648+
fmt.Println("\nGoLevelDB Stats")
648649
fmt.Println(db.Stats()["leveldb.stats"])
649650
fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
650651
}
@@ -705,7 +706,7 @@ func TestAppStateDeterminism(t *testing.T) {
705706
simulation.SimulateFromSeed(
706707
t, os.Stdout, app.BaseApp, appStateFn, seed,
707708
testAndRunTxs(app), []sdk.Invariant{},
708-
50, 100, 0,
709+
50, 100, 0, "",
709710
false, true, false, false, false, app.ModuleAccountAddrs(),
710711
)
711712
appHash := app.LastCommitID().Hash
@@ -734,7 +735,7 @@ func BenchmarkInvariants(b *testing.B) {
734735
_, params, simErr := simulation.SimulateFromSeed(
735736
b, ioutil.Discard, app.BaseApp, appStateFn, seed, testAndRunTxs(app),
736737
[]sdk.Invariant{}, numBlocks, exportParamsHeight, blockSize,
737-
exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
738+
exportStatsPath, exportParams, commit, lean, onOperation, false, app.ModuleAccountAddrs(),
738739
)
739740

740741
// export state and params before the simulation error is checked

‎simapp/utils.go

+1
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ var (
4343
exportParamsPath string
4444
exportParamsHeight int
4545
exportStatePath string
46+
exportStatsPath string
4647
seed int64
4748
numBlocks int
4849
blockSize int

‎x/distribution/simulation/msgs.go

+6-6
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
"github.com/cosmos/cosmos-sdk/x/simulation"
1414
)
1515

16-
// SimulateMsgSetWithdrawAddress
16+
// SimulateMsgSetWithdrawAddress generates a MsgSetWithdrawAddress with random values.
1717
func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
1818
handler := distribution.NewHandler(k)
1919
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
@@ -24,7 +24,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper)
2424
msg := distribution.NewMsgSetWithdrawAddress(accountOrigin.Address, accountDestination.Address)
2525

2626
if msg.ValidateBasic() != nil {
27-
return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
27+
return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
2828
}
2929

3030
ctx, write := ctx.CacheContext()
@@ -38,7 +38,7 @@ func SimulateMsgSetWithdrawAddress(m auth.AccountKeeper, k distribution.Keeper)
3838
}
3939
}
4040

41-
// SimulateMsgWithdrawDelegatorReward
41+
// SimulateMsgWithdrawDelegatorReward generates a MsgWithdrawDelegatorReward with random values.
4242
func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
4343
handler := distribution.NewHandler(k)
4444
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
@@ -49,7 +49,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee
4949
msg := distribution.NewMsgWithdrawDelegatorReward(delegatorAccount.Address, sdk.ValAddress(validatorAccount.Address))
5050

5151
if msg.ValidateBasic() != nil {
52-
return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
52+
return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
5353
}
5454

5555
ctx, write := ctx.CacheContext()
@@ -63,7 +63,7 @@ func SimulateMsgWithdrawDelegatorReward(m auth.AccountKeeper, k distribution.Kee
6363
}
6464
}
6565

66-
// SimulateMsgWithdrawValidatorCommission
66+
// SimulateMsgWithdrawValidatorCommission generates a MsgWithdrawValidatorCommission with random values.
6767
func SimulateMsgWithdrawValidatorCommission(m auth.AccountKeeper, k distribution.Keeper) simulation.Operation {
6868
handler := distribution.NewHandler(k)
6969
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context,
@@ -73,7 +73,7 @@ func SimulateMsgWithdrawValidatorCommission(m auth.AccountKeeper, k distribution
7373
msg := distribution.NewMsgWithdrawValidatorCommission(sdk.ValAddress(account.Address))
7474

7575
if msg.ValidateBasic() != nil {
76-
return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
76+
return simulation.NoOpMsg(distribution.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
7777
}
7878

7979
ctx, write := ctx.CacheContext()

‎x/genutil/types/genesis_state_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@ package types
33
import (
44
"testing"
55

6+
"github.com/stretchr/testify/require"
7+
"github.com/tendermint/tendermint/crypto/ed25519"
8+
69
sdk "github.com/cosmos/cosmos-sdk/types"
710
authtypes "github.com/cosmos/cosmos-sdk/x/auth/types"
811
stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
9-
"github.com/stretchr/testify/require"
10-
"github.com/tendermint/tendermint/crypto/ed25519"
1112
)
1213

1314
var (

‎x/gov/simulation/msgs.go

+8-9
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, contentSim Con
5454
content := contentSim(r, app, ctx, accs)
5555
msg, err := simulationCreateMsgSubmitProposal(r, content, sender)
5656
if err != nil {
57-
return simulation.NoOpMsg(), nil, err
57+
return simulation.NoOpMsg(gov.ModuleName), nil, err
5858
}
5959

6060
ok := simulateHandleMsgSubmitProposal(msg, handler, ctx)
@@ -66,7 +66,7 @@ func SimulateSubmittingVotingAndSlashingForProposal(k gov.Keeper, contentSim Con
6666

6767
proposalID, err := k.GetProposalID(ctx)
6868
if err != nil {
69-
return simulation.NoOpMsg(), nil, err
69+
return simulation.NoOpMsg(gov.ModuleName), nil, err
7070
}
7171

7272
proposalID = uint64(math.Max(float64(proposalID)-1, 0))
@@ -122,20 +122,20 @@ func simulationCreateMsgSubmitProposal(r *rand.Rand, c gov.Content, s simulation
122122
return
123123
}
124124

125-
// SimulateMsgDeposit
125+
// SimulateMsgDeposit generates a MsgDeposit with random values.
126126
func SimulateMsgDeposit(k gov.Keeper) simulation.Operation {
127127
return func(r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, accs []simulation.Account) (
128128
opMsg simulation.OperationMsg, fOps []simulation.FutureOperation, err error) {
129129

130130
acc := simulation.RandomAcc(r, accs)
131131
proposalID, ok := randomProposalID(r, k, ctx)
132132
if !ok {
133-
return simulation.NoOpMsg(), nil, nil
133+
return simulation.NoOpMsg(gov.ModuleName), nil, nil
134134
}
135135
deposit := randomDeposit(r)
136136
msg := gov.NewMsgDeposit(acc.Address, proposalID, deposit)
137137
if msg.ValidateBasic() != nil {
138-
return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
138+
return simulation.NoOpMsg(gov.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
139139
}
140140
ctx, write := ctx.CacheContext()
141141
ok = gov.NewHandler(k)(ctx, msg).IsOK()
@@ -148,8 +148,7 @@ func SimulateMsgDeposit(k gov.Keeper) simulation.Operation {
148148
}
149149
}
150150

151-
// SimulateMsgVote
152-
// nolint: unparam
151+
// SimulateMsgVote generates a MsgVote with random values.
153152
func SimulateMsgVote(k gov.Keeper) simulation.Operation {
154153
return operationSimulateMsgVote(k, simulation.Account{}, 0)
155154
}
@@ -167,14 +166,14 @@ func operationSimulateMsgVote(k gov.Keeper, acc simulation.Account, proposalID u
167166
var ok bool
168167
proposalID, ok = randomProposalID(r, k, ctx)
169168
if !ok {
170-
return simulation.NoOpMsg(), nil, nil
169+
return simulation.NoOpMsg(gov.ModuleName), nil, nil
171170
}
172171
}
173172
option := randomVotingOption(r)
174173

175174
msg := gov.NewMsgVote(acc.Address, proposalID, option)
176175
if msg.ValidateBasic() != nil {
177-
return simulation.NoOpMsg(), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
176+
return simulation.NoOpMsg(gov.ModuleName), nil, fmt.Errorf("expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
178177
}
179178

180179
ctx, write := ctx.CacheContext()

‎x/simulation/event_stats.go

+38-16
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,55 @@
11
package simulation
22

33
import (
4+
"encoding/json"
45
"fmt"
56
"io"
6-
"sort"
7+
"io/ioutil"
78
)
89

9-
type eventStats map[string]uint
10+
// EventStats defines an object that keeps a tally of each event that has occurred
11+
// during a simulation.
12+
type EventStats map[string]map[string]map[string]int
1013

11-
func newEventStats() eventStats {
12-
events := make(map[string]uint)
13-
return events
14+
// NewEventStats creates a new empty EventStats object
15+
func NewEventStats() EventStats {
16+
return make(EventStats)
1417
}
1518

16-
func (es eventStats) tally(eventDesc string) {
17-
es[eventDesc]++
19+
// Tally increases the count of a simulation event.
20+
func (es EventStats) Tally(route, op, evResult string) {
21+
_, ok := es[route]
22+
if !ok {
23+
es[route] = make(map[string]map[string]int)
24+
}
25+
26+
_, ok = es[route][op]
27+
if !ok {
28+
es[route][op] = make(map[string]int)
29+
}
30+
31+
es[route][op][evResult]++
1832
}
1933

20-
// Pretty-print events as a table
21-
func (es eventStats) Print(w io.Writer) {
22-
var keys []string
23-
for key := range es {
24-
keys = append(keys, key)
34+
// Print the event stats in JSON format.
35+
func (es EventStats) Print(w io.Writer) {
36+
obj, err := json.MarshalIndent(es, "", " ")
37+
if err != nil {
38+
panic(err)
2539
}
2640

27-
sort.Strings(keys)
28-
fmt.Fprintf(w, "Event statistics: \n")
41+
fmt.Fprintf(w, string(obj))
42+
}
43+
44+
// ExportJSON saves the event stats as a JSON file on a given path
45+
func (es EventStats) ExportJSON(path string) {
46+
bz, err := json.MarshalIndent(es, "", " ")
47+
if err != nil {
48+
panic(err)
49+
}
2950

30-
for _, key := range keys {
31-
fmt.Fprintf(w, " % 60s => %d\n", key, es[key])
51+
err = ioutil.WriteFile(path, bz, 0644)
52+
if err != nil {
53+
panic(err)
3254
}
3355
}

‎x/simulation/mock_tendermint.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ func (vals mockValidators) randomProposer(r *rand.Rand) cmn.HexBytes {
7979
// nolint: unparam
8080
func updateValidators(tb testing.TB, r *rand.Rand, params Params,
8181
current map[string]mockValidator, updates []abci.ValidatorUpdate,
82-
event func(string)) map[string]mockValidator {
82+
event func(route, op, evResult string)) map[string]mockValidator {
8383

8484
for _, update := range updates {
8585
str := fmt.Sprintf("%v", update.PubKey)
@@ -88,21 +88,21 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params,
8888
if _, ok := current[str]; !ok {
8989
tb.Fatalf("tried to delete a nonexistent validator")
9090
}
91-
event("endblock/validatorupdates/kicked")
91+
event("end_block", "validator_updates", "kicked")
9292
delete(current, str)
9393

9494
} else if mVal, ok := current[str]; ok {
9595
// validator already exists
9696
mVal.val = update
97-
event("endblock/validatorupdates/updated")
97+
event("end_block", "validator_updates", "updated")
9898

9999
} else {
100100
// Set this new validator
101101
current[str] = mockValidator{
102102
update,
103103
GetMemberOfInitialState(r, params.InitialLivenessWeightings),
104104
}
105-
event("endblock/validatorupdates/added")
105+
event("end_block", "validator_updates", "added")
106106
}
107107
}
108108

@@ -114,7 +114,7 @@ func updateValidators(tb testing.TB, r *rand.Rand, params Params,
114114
func RandomRequestBeginBlock(r *rand.Rand, params Params,
115115
validators mockValidators, pastTimes []time.Time,
116116
pastVoteInfos [][]abci.VoteInfo,
117-
event func(string), header abci.Header) abci.RequestBeginBlock {
117+
event func(route, op, evResult string), header abci.Header) abci.RequestBeginBlock {
118118

119119
if len(validators) == 0 {
120120
return abci.RequestBeginBlock{
@@ -139,9 +139,9 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params,
139139
}
140140

141141
if signed {
142-
event("beginblock/signing/signed")
142+
event("begin_block", "signing", "signed")
143143
} else {
144-
event("beginblock/signing/missed")
144+
event("begin_block", "signing", "missed")
145145
}
146146

147147
pubkey, err := tmtypes.PB2TM.PubKey(mVal.val.PubKey)
@@ -197,7 +197,7 @@ func RandomRequestBeginBlock(r *rand.Rand, params Params,
197197
TotalVotingPower: totalVotingPower,
198198
},
199199
)
200-
event("beginblock/evidence")
200+
event("begin_block", "evidence", "ok")
201201
}
202202

203203
return abci.RequestBeginBlock{

0 commit comments

Comments
 (0)