add reverse iteration to pagination (cosmos#8875)

* WIP

* add tests

* add tests

* fix lint

* fix pagination

* add proto message doc

* fix filtered_pagination

* fix test

* cleanup

* add reverse flag to pagination

* changelog

* Update client/flags/flags.go

* Update CHANGELOG.md

Co-authored-by: Alessio Treglia <[email protected]>
Co-authored-by: Federico Kunze <[email protected]>

(cherry picked from commit a78f777)

Author: aleem1314

client/flags/flags.go

@@ -70,6 +70,7 @@ const (
 	FlagCountTotal       = "count-total"
 	FlagTimeoutHeight    = "timeout-height"
 	FlagKeyAlgorithm     = "algo"
+	FlagReverse          = "reverse"
 
 	// Tendermint logging flags
 	FlagLogLevel  = "log_level"
@@ -124,6 +125,7 @@ func AddPaginationFlagsToCmd(cmd *cobra.Command, query string) {
 	cmd.Flags().Uint64(FlagOffset, 0, fmt.Sprintf("pagination offset of %s to query for", query))
 	cmd.Flags().Uint64(FlagLimit, 100, fmt.Sprintf("pagination limit of %s to query for", query))
 	cmd.Flags().Bool(FlagCountTotal, false, fmt.Sprintf("count total number of records in %s to query for", query))
+	cmd.Flags().Bool(FlagReverse, false, "results are sorted in descending order")
 }
 
 // GasSetting encapsulates the possible values passed through the --gas flag.

client/utils.go

@@ -52,6 +52,7 @@ func ReadPageRequest(flagSet *pflag.FlagSet) (*query.PageRequest, error) {
 	limit, _ := flagSet.GetUint64(flags.FlagLimit)
 	countTotal, _ := flagSet.GetBool(flags.FlagCountTotal)
 	page, _ := flagSet.GetUint64(flags.FlagPage)
+	reverse, _ := flagSet.GetBool(flags.FlagReverse)
 
 	if page > 1 && offset > 0 {
 		return nil, sdkerrors.Wrap(sdkerrors.ErrInvalidRequest, "page and offset cannot be used together")
@@ -66,6 +67,7 @@ func ReadPageRequest(flagSet *pflag.FlagSet) (*query.PageRequest, error) {
 		Offset:     offset,
 		Limit:      limit,
 		CountTotal: countTotal,
+		Reverse:    reverse,
 	}, nil
 }
 

proto/cosmos/base/query/v1beta1/pagination.proto

@@ -30,6 +30,9 @@ message PageRequest {
   // count_total is only respected when offset is used. It is ignored when key
   // is set.
   bool count_total = 4;
+
+  // reverse is set to true indicates that, results to be returned in the descending order.
+  bool reverse = 5;
 }
 
 // PageResponse is to be embedded in gRPC response messages where the

types/query/filtered_pagination.go

@@ -29,6 +29,7 @@ func FilteredPaginate(
 	key := pageRequest.Key
 	limit := pageRequest.Limit
 	countTotal := pageRequest.CountTotal
+	reverse := pageRequest.Reverse
 
 	if offset > 0 && key != nil {
 		return nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
@@ -42,7 +43,7 @@ func FilteredPaginate(
 	}
 
 	if len(key) != 0 {
-		iterator := prefixStore.Iterator(key, nil)
+		iterator := getIterator(prefixStore, key, reverse)
 		defer iterator.Close()
 
 		var numHits uint64
@@ -73,7 +74,7 @@ func FilteredPaginate(
 		}, nil
 	}
 
-	iterator := prefixStore.Iterator(nil, nil)
+	iterator := getIterator(prefixStore, nil, reverse)
 	defer iterator.Close()
 
 	end := offset + limit

types/query/filtered_pagination_test.go

@@ -2,7 +2,6 @@ package query_test
 
 import (
 	"fmt"
-
 	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/store/prefix"
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -87,6 +86,87 @@ func (s *paginationTestSuite) TestFilteredPaginations() {
 	s.Require().LessOrEqual(len(balances), 2)
 }
 
+func (s *paginationTestSuite) TestReverseFilteredPaginations() {
+	app, ctx, appCodec := setupTest()
+
+	var balances sdk.Coins
+	for i := 0; i < numBalances; i++ {
+		denom := fmt.Sprintf("foo%ddenom", i)
+		balances = append(balances, sdk.NewInt64Coin(denom, 100))
+	}
+
+	for i := 0; i < 10; i++ {
+		denom := fmt.Sprintf("test%ddenom", i)
+		balances = append(balances, sdk.NewInt64Coin(denom, 250))
+	}
+
+	balances = balances.Sort()
+	addr1 := sdk.AccAddress([]byte("addr1"))
+	acc1 := app.AccountKeeper.NewAccountWithAddress(ctx, addr1)
+	app.AccountKeeper.SetAccount(ctx, acc1)
+	s.Require().NoError(app.BankKeeper.SetBalances(ctx, addr1, balances))
+	store := ctx.KVStore(app.GetKey(authtypes.StoreKey))
+
+	// verify pagination with limit > total values
+	pageReq := &query.PageRequest{Key: nil, Limit: 5, CountTotal: true, Reverse: true}
+	balns, res, err := execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(5, len(balns))
+
+	s.T().Log("verify empty request")
+	balns, res, err = execFilterPaginate(store, nil, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(10, len(balns))
+	s.Require().Equal(uint64(10), res.Total)
+	s.Require().Nil(res.NextKey)
+
+	s.T().Log("verify default limit")
+	pageReq = &query.PageRequest{Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(10, len(balns))
+	s.Require().Equal(uint64(10), res.Total)
+
+	s.T().Log("verify nextKey is returned if there are more results")
+	pageReq = &query.PageRequest{Limit: 2, CountTotal: true, Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(2, len(balns))
+	s.Require().NotNil(res.NextKey)
+	s.Require().Equal(string(res.NextKey), fmt.Sprintf("test7denom"))
+	s.Require().Equal(uint64(10), res.Total)
+
+	s.T().Log("verify both key and offset can't be given")
+	pageReq = &query.PageRequest{Key: res.NextKey, Limit: 1, Offset: 2, Reverse: true}
+	_, _, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().Error(err)
+
+	s.T().Log("use nextKey for query and reverse true")
+	pageReq = &query.PageRequest{Key: res.NextKey, Limit: 2, Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(2, len(balns))
+	s.Require().NotNil(res.NextKey)
+	s.Require().Equal(string(res.NextKey), fmt.Sprintf("test5denom"))
+
+	s.T().Log("verify last page records, nextKey for query and reverse true")
+	pageReq = &query.PageRequest{Key: res.NextKey, Reverse: true}
+	balns, res, err = execFilterPaginate(store, pageReq, appCodec)
+	s.Require().NoError(err)
+	s.Require().NotNil(res)
+	s.Require().Equal(6, len(balns))
+	s.Require().Nil(res.NextKey)
+
+	s.T().Log("verify Reverse pagination returns valid result")
+	s.Require().Equal(balances[235:241].String(), balns.Sort().String())
+
+}
+
 func ExampleFilteredPaginate() {
 	app, ctx, appCodec := setupTest()
 

types/query/pagination.go

@@ -7,6 +7,7 @@ import (
 	"google.golang.org/grpc/status"
 
 	"github.com/cosmos/cosmos-sdk/store/types"
+	db "github.com/tendermint/tm-db"
 )
 
 // DefaultLimit is the default `limit` for queries
@@ -54,6 +55,7 @@ func Paginate(
 	key := pageRequest.Key
 	limit := pageRequest.Limit
 	countTotal := pageRequest.CountTotal
+	reverse := pageRequest.Reverse
 
 	if offset > 0 && key != nil {
 		return nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
@@ -67,13 +69,14 @@ func Paginate(
 	}
 
 	if len(key) != 0 {
-		iterator := prefixStore.Iterator(key, nil)
+		iterator := getIterator(prefixStore, key, reverse)
 		defer iterator.Close()
 
 		var count uint64
 		var nextKey []byte
 
 		for ; iterator.Valid(); iterator.Next() {
+
 			if count == limit {
 				nextKey = iterator.Key()
 				break
@@ -94,7 +97,7 @@ func Paginate(
 		}, nil
 	}
 
-	iterator := prefixStore.Iterator(nil, nil)
+	iterator := getIterator(prefixStore, nil, reverse)
 	defer iterator.Close()
 
 	end := offset + limit
@@ -132,3 +135,19 @@ func Paginate(
 
 	return res, nil
 }
+
+func getIterator(prefixStore types.KVStore, start []byte, reverse bool) db.Iterator {
+	if reverse {
+		var end []byte
+		if start != nil {
+			itr := prefixStore.Iterator(start, nil)
+			defer itr.Close()
+			if itr.Valid() {
+				itr.Next()
+				end = itr.Key()
+			}
+		}
+		return prefixStore.ReverseIterator(nil, end)
+	}
+	return prefixStore.Iterator(start, nil)
+}