basho_metrics re-entrant?

classic Classic list List threaded Threaded
3 messages Options
Reply | Threaded
Open this post in threaded view
|

basho_metrics re-entrant?

Joel Meyer
Hello,

I'd like to capture statistics on all calls to my riak_core service and the execution time of vnode commands. I looked at folsom_metrics, but for a service that does 10-30k QPS I'm concerned about the overhead introduced of doing ets calls. Given that, my current plan was to create several basho_metrics on startup and store the opaque refs using mochiglobal. Then my service will make module calls to update the metrics. Will using basho_metrics in that matter cause problems?

Here's what I'm thinking:

-module(freqserver_stats).

-export([ init/0,
            vnode_count/1,
            ... ]).

-define( VNODE_COUNT,     freqserver_stats_vnode_count     ).                                                                                                                       
...

-define( METRICS,
             [ { ?VNODE_COUNT,     histogram },
               ... ] ).

  -define( STAT(Name), mochiglobal:get(Name) ).

init() ->                                                                                                                                                                           
    F = fun( { S, Type } , _ ) ->
          Ref = case Type of
                  histogram -> basho_metrics_nifs:histogram_new();
                  meter     -> basho_metrics_nifs:meter_new()
                end,
          ok = mochiglobal:put( S, Ref )
        end,
    lists:foldl( F, [], ?METRICS ).

  vnode_count( Micros ) ->
    basho_metrics_nifs:histogram_update( ?STAT( ?VNODE_COUNT ), Micros ).

I'm also open to better ideas, if people have them. I'm concerned about back pressure and/or overflowing the message queue if I dedicate a process to handling these statistics. Let me know if I'm needlessly concerned about that.

Thanks,
Joel

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
Reply | Threaded
Open this post in threaded view
|

Re: basho_metrics re-entrant?

Bob Ippolito
I don't believe that you'll be able to store these NIF instances in mochiglobal. You can only store things that can be represented as BEAM code.

On Wed, Feb 29, 2012 at 2:46 PM, Joel Meyer <[hidden email]> wrote:
Hello,

I'd like to capture statistics on all calls to my riak_core service and the execution time of vnode commands. I looked at folsom_metrics, but for a service that does 10-30k QPS I'm concerned about the overhead introduced of doing ets calls. Given that, my current plan was to create several basho_metrics on startup and store the opaque refs using mochiglobal. Then my service will make module calls to update the metrics. Will using basho_metrics in that matter cause problems?

Here's what I'm thinking:

-module(freqserver_stats).

-export([ init/0,
            vnode_count/1,
            ... ]).

-define( VNODE_COUNT,     freqserver_stats_vnode_count     ).                                                                                                                       
...

-define( METRICS,
             [ { ?VNODE_COUNT,     histogram },
               ... ] ).

  -define( STAT(Name), mochiglobal:get(Name) ).

init() ->                                                                                                                                                                           
    F = fun( { S, Type } , _ ) ->
          Ref = case Type of
                  histogram -> basho_metrics_nifs:histogram_new();
                  meter     -> basho_metrics_nifs:meter_new()
                end,
          ok = mochiglobal:put( S, Ref )
        end,
    lists:foldl( F, [], ?METRICS ).

  vnode_count( Micros ) ->
    basho_metrics_nifs:histogram_update( ?STAT( ?VNODE_COUNT ), Micros ).

I'm also open to better ideas, if people have them. I'm concerned about back pressure and/or overflowing the message queue if I dedicate a process to handling these statistics. Let me know if I'm needlessly concerned about that.

Thanks,
Joel

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com



_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
Reply | Threaded
Open this post in threaded view
|

Re: basho_metrics re-entrant?

Joel Meyer


On Wed, Feb 29, 2012 at 2:58 PM, Bob Ippolito <[hidden email]> wrote:
I don't believe that you'll be able to store these NIF instances in mochiglobal. You can only store things that can be represented as BEAM code.

Ah, you're right. I tested putting and getting, but I didn't test calling the nif with the result of getting, which fails with a bad argument exception.

Eshell V5.8.4  (abort with ^G)
1> {ok, Stat} = basho_metrics_nifs:histogram_new().
{ok,<<>>}
2> mochiglobal:put(stat,Stat).
ok
3> MGStat = mochiglobal:get(stat).
<<>>
4> basho_metrics_nifs:histogram_stats(MGStat).
** exception error: bad argument
     in function  basho_metrics_nifs:histogram_stats/1
        called as basho_metrics_nifs:histogram_stats(<<>>)
5>

It looks like storing the refs in an ets table works, so I'll try that.

Thanks,
Joel
 

On Wed, Feb 29, 2012 at 2:46 PM, Joel Meyer <[hidden email]> wrote:
Hello,

I'd like to capture statistics on all calls to my riak_core service and the execution time of vnode commands. I looked at folsom_metrics, but for a service that does 10-30k QPS I'm concerned about the overhead introduced of doing ets calls. Given that, my current plan was to create several basho_metrics on startup and store the opaque refs using mochiglobal. Then my service will make module calls to update the metrics. Will using basho_metrics in that matter cause problems?

Here's what I'm thinking:

-module(freqserver_stats).

-export([ init/0,
            vnode_count/1,
            ... ]).

-define( VNODE_COUNT,     freqserver_stats_vnode_count     ).                                                                                                                       
...

-define( METRICS,
             [ { ?VNODE_COUNT,     histogram },
               ... ] ).

  -define( STAT(Name), mochiglobal:get(Name) ).

init() ->                                                                                                                                                                           
    F = fun( { S, Type } , _ ) ->
          Ref = case Type of
                  histogram -> basho_metrics_nifs:histogram_new();
                  meter     -> basho_metrics_nifs:meter_new()
                end,
          ok = mochiglobal:put( S, Ref )
        end,
    lists:foldl( F, [], ?METRICS ).

  vnode_count( Micros ) ->
    basho_metrics_nifs:histogram_update( ?STAT( ?VNODE_COUNT ), Micros ).

I'm also open to better ideas, if people have them. I'm concerned about back pressure and/or overflowing the message queue if I dedicate a process to handling these statistics. Let me know if I'm needlessly concerned about that.

Thanks,
Joel

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com




_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com