Yokozuna's inconsistent query problem

classic Classic list List threaded Threaded
4 messages Options
Reply | Threaded
Open this post in threaded view
|

Yokozuna's inconsistent query problem

Witeman Zheng
Hi,


I am having a 10 nodes of RiakKV 2.2.0, and turn on Riak Search(Yokozuna).  Having about 3million records in one bucket with index, every record has about 1k size.

Then when it is triggered a Yokozuna query for one specific id, sometimes return the record, sometimes return NOT FOUND, it is very weird.

FOUND case:
wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":57,
    "params":{
      "10.100.205.80:8093":"_yz_pn:230 OR _yz_pn:200 OR _yz_pn:170 OR _yz_pn:140 OR _yz_pn:110 OR _yz_pn:80 OR _yz_pn:50 OR _yz_pn:20",
      "10.100.205.79:8093":"_yz_pn:239 OR _yz_pn:209 OR _yz_pn:179 OR _yz_pn:149 OR _yz_pn:119 OR _yz_pn:89 OR _yz_pn:59 OR _yz_pn:29",
      "indent":"true",
      "10.100.205.73:8093":"_yz_pn:233 OR _yz_pn:203 OR _yz_pn:173 OR _yz_pn:143 OR _yz_pn:113 OR _yz_pn:83 OR _yz_pn:53 OR _yz_pn:23",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:236 OR _yz_pn:206 OR _yz_pn:176 OR _yz_pn:146 OR _yz_pn:116 OR _yz_pn:86 OR _yz_pn:56 OR _yz_pn:26",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:251 OR _yz_pn:221 OR _yz_pn:191 OR _yz_pn:161 OR _yz_pn:131 OR _yz_pn:101 OR _yz_pn:71 OR _yz_pn:41 OR _yz_pn:11",
      "10.100.205.74:8093":"_yz_pn:224 OR _yz_pn:194 OR _yz_pn:164 OR _yz_pn:134 OR _yz_pn:104 OR _yz_pn:74 OR _yz_pn:44 OR _yz_pn:14",
      "10.100.205.77:8093":"_yz_pn:227 OR _yz_pn:197 OR _yz_pn:167 OR _yz_pn:137 OR _yz_pn:107 OR _yz_pn:77 OR _yz_pn:47 OR _yz_pn:17",
      "10.100.205.78:8093":"_yz_pn:248 OR _yz_pn:218 OR _yz_pn:188 OR _yz_pn:158 OR _yz_pn:128 OR _yz_pn:98 OR _yz_pn:68 OR _yz_pn:38 OR _yz_pn:8",
      "10.100.205.75:8093":"_yz_pn:255 OR _yz_pn:245 OR _yz_pn:215 OR _yz_pn:185 OR _yz_pn:155 OR _yz_pn:125 OR _yz_pn:95 OR _yz_pn:65 OR _yz_pn:35 OR _yz_pn:5",
      "wt":"json",
      "10.100.205.72:8093":"(_yz_pn:252 AND (_yz_fpn:252)) OR _yz_pn:242 OR _yz_pn:212 OR _yz_pn:182 OR _yz_pn:152 OR _yz_pn:122 OR _yz_pn:92 OR _yz_pn:62 OR _yz_pn:32 OR _yz_pn:2"}},
  "response":{"numFound":1,"start":0,"docs":[
      {
        “some_field_1_l":0,
        "some_field_2_l":0,
        "some_field_3_s":"[]",
        "some_field_4_s":"[]",
        "some_field_5_l":0,
        "some_field_6_l":0,
        "some_field_7_s":"[]",
        "some_field_8_s":"[[1,0],[2,0],[3,0],[4,0],[5,0],[6,0],[7,0],[8,0],[9,0],[10,0],[11,0],[12,0],[13,0],[14,0],[15,0],[16,0],[17,0],[18,0],[19,0],[20,0]]",
        "collect_id_l":2765608,
        "some_field_9_i":1191,
        "some_field_10_i":1191,
        "some_field_11_i":1,
        "some_field_12_l":0,
        "some_field_13_l":2000,
        "some_field_14_l":764846,
        "some_field_15_l":766846,
        "some_field_16_i":57,
        "some_field_17_i":1,
        "some_field_18_s":"UTC_-4",
        "some_field_19_i":1487822270,
        "some_field_20_s":"61.221.181.7",
        "some_field_21_l":2869104,
        "some_field_22_s":"[1,4,10,4,5,3,5,6,8,4,3,2,2,1,10]",
        "some_field_23_i":20,
        "some_field_24_i":1,
        "some_field_25_i":1,
        "agent_uid_i":1191,
        "some_field_26_l":2772439,
        "some_field_27_i":100,
        "_yz_id":"1*default*data_201702*2772439*203",
        "_yz_rk":"2772439",
        "_yz_rt":"default",
        "_yz_rb":"data_201702"}]

  }}




NOTFOUND case:

wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":62,
    "params":{
      "10.100.205.80:8093":"_yz_pn:240 OR _yz_pn:210 OR _yz_pn:180 OR _yz_pn:150 OR _yz_pn:120 OR _yz_pn:90 OR _yz_pn:60 OR _yz_pn:30",
      "10.100.205.79:8093":"_yz_pn:249 OR _yz_pn:219 OR _yz_pn:189 OR _yz_pn:159 OR _yz_pn:129 OR _yz_pn:99 OR _yz_pn:69 OR _yz_pn:39 OR _yz_pn:9",
      "indent":"true",
      "10.100.205.73:8093":"(_yz_pn:253 AND (_yz_fpn:253)) OR _yz_pn:243 OR _yz_pn:213 OR _yz_pn:183 OR _yz_pn:153 OR _yz_pn:123 OR _yz_pn:93 OR _yz_pn:63 OR _yz_pn:33 OR _yz_pn:3",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:256 OR _yz_pn:246 OR _yz_pn:216 OR _yz_pn:186 OR _yz_pn:156 OR _yz_pn:126 OR _yz_pn:96 OR _yz_pn:66 OR _yz_pn:36 OR _yz_pn:6",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:231 OR _yz_pn:201 OR _yz_pn:171 OR _yz_pn:141 OR _yz_pn:111 OR _yz_pn:81 OR _yz_pn:51 OR _yz_pn:21",
      "10.100.205.74:8093":"_yz_pn:234 OR _yz_pn:204 OR _yz_pn:174 OR _yz_pn:144 OR _yz_pn:114 OR _yz_pn:84 OR _yz_pn:54 OR _yz_pn:24",
      "10.100.205.77:8093":"_yz_pn:237 OR _yz_pn:207 OR _yz_pn:177 OR _yz_pn:147 OR _yz_pn:117 OR _yz_pn:87 OR _yz_pn:57 OR _yz_pn:27",
      "10.100.205.78:8093":"_yz_pn:228 OR _yz_pn:198 OR _yz_pn:168 OR _yz_pn:138 OR _yz_pn:108 OR _yz_pn:78 OR _yz_pn:48 OR _yz_pn:18",
      "10.100.205.75:8093":"_yz_pn:225 OR _yz_pn:195 OR _yz_pn:165 OR _yz_pn:135 OR _yz_pn:105 OR _yz_pn:75 OR _yz_pn:45 OR _yz_pn:15",
      "wt":"json",
      "10.100.205.72:8093":"_yz_pn:252 OR _yz_pn:222 OR _yz_pn:192 OR _yz_pn:162 OR _yz_pn:132 OR _yz_pn:102 OR _yz_pn:72 OR _yz_pn:42 OR _yz_pn:12"}},
  "response":{"numFound":0,"start":0,"docs":[]
  }}


I can select this record from solr webapp from some of these 10 nodes.  So this record should be indexed by solr.  So I guessed the problem cause this is related with Yokozuna’s distributed shards, since one query would shard to every solr instance by Yokozuna, then Yokozuna would collect all the returns and reduce to a fix result, something like map-reduce mechanism.

So the problem here may underlay in map phase or reduce phase.

The default configuration of riak search applied in these 10 nodes.

Would anyone have some insight on how to fix this?  By modified the configuration of search?


Best regards,
Witeman


_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
Reply | Threaded
Open this post in threaded view
|

Re: Yokozuna's inconsistent query problem

Fred Dushin
Hello Witeman,

What you are seeing with your two queries is the result of two different coverage plans, querying different parts of the cluster.  Riak Search translates coverage plans to Solr sharded queries, and will periodically change the coverage plan, so as to more evenly distribute queries across the cluster.  So what you are seeing is effectively two different sharded queries, hitting different solr instances in your cluster.  You are seeing inconsistent search results, which suggests there is a discrepancy between what is stored in Solr and what is stored in Riak.  Generally speaking, AAE should detect these repairs and correct them over time.

Can you please send the output of the following two commands:

riak-admin aae-status
riak-admin search aae-status

That will tell you something about the behavior of the underlying KV and Yokozuna AAE subsystems.

Out of curiosity, can you you read the "2772439" key in the "data_201702" bucket in Riak with quorum equal to all?  If you do that read, does that affect the behavior of your query you posted?  (I am wondering if you trigger read repair that will repair the entry in Solr)

Thanks,
-Fred

On Feb 23, 2017, at 10:11 PM, Witeman Zheng <[hidden email]> wrote:

Hi,


I am having a 10 nodes of RiakKV 2.2.0, and turn on Riak Search(Yokozuna).  Having about 3million records in one bucket with index, every record has about 1k size.

Then when it is triggered a Yokozuna query for one specific id, sometimes return the record, sometimes return NOT FOUND, it is very weird.

FOUND case:
wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":57,
    "params":{
      "10.100.205.80:8093":"_yz_pn:230 OR _yz_pn:200 OR _yz_pn:170 OR _yz_pn:140 OR _yz_pn:110 OR _yz_pn:80 OR _yz_pn:50 OR _yz_pn:20",
      "10.100.205.79:8093":"_yz_pn:239 OR _yz_pn:209 OR _yz_pn:179 OR _yz_pn:149 OR _yz_pn:119 OR _yz_pn:89 OR _yz_pn:59 OR _yz_pn:29",
      "indent":"true",
      "10.100.205.73:8093":"_yz_pn:233 OR _yz_pn:203 OR _yz_pn:173 OR _yz_pn:143 OR _yz_pn:113 OR _yz_pn:83 OR _yz_pn:53 OR _yz_pn:23",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:236 OR _yz_pn:206 OR _yz_pn:176 OR _yz_pn:146 OR _yz_pn:116 OR _yz_pn:86 OR _yz_pn:56 OR _yz_pn:26",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:251 OR _yz_pn:221 OR _yz_pn:191 OR _yz_pn:161 OR _yz_pn:131 OR _yz_pn:101 OR _yz_pn:71 OR _yz_pn:41 OR _yz_pn:11",
      "10.100.205.74:8093":"_yz_pn:224 OR _yz_pn:194 OR _yz_pn:164 OR _yz_pn:134 OR _yz_pn:104 OR _yz_pn:74 OR _yz_pn:44 OR _yz_pn:14",
      "10.100.205.77:8093":"_yz_pn:227 OR _yz_pn:197 OR _yz_pn:167 OR _yz_pn:137 OR _yz_pn:107 OR _yz_pn:77 OR _yz_pn:47 OR _yz_pn:17",
      "10.100.205.78:8093":"_yz_pn:248 OR _yz_pn:218 OR _yz_pn:188 OR _yz_pn:158 OR _yz_pn:128 OR _yz_pn:98 OR _yz_pn:68 OR _yz_pn:38 OR _yz_pn:8",
      "10.100.205.75:8093":"_yz_pn:255 OR _yz_pn:245 OR _yz_pn:215 OR _yz_pn:185 OR _yz_pn:155 OR _yz_pn:125 OR _yz_pn:95 OR _yz_pn:65 OR _yz_pn:35 OR _yz_pn:5",
      "wt":"json",
      "10.100.205.72:8093":"(_yz_pn:252 AND (_yz_fpn:252)) OR _yz_pn:242 OR _yz_pn:212 OR _yz_pn:182 OR _yz_pn:152 OR _yz_pn:122 OR _yz_pn:92 OR _yz_pn:62 OR _yz_pn:32 OR _yz_pn:2"}},
  "response":{"numFound":1,"start":0,"docs":[
      {
        “some_field_1_l":0,
        "some_field_2_l":0,
        "some_field_3_s":"[]",
        "some_field_4_s":"[]",
        "some_field_5_l":0,
        "some_field_6_l":0,
        "some_field_7_s":"[]",
        "some_field_8_s":"[[1,0],[2,0],[3,0],[4,0],[5,0],[6,0],[7,0],[8,0],[9,0],[10,0],[11,0],[12,0],[13,0],[14,0],[15,0],[16,0],[17,0],[18,0],[19,0],[20,0]]",
        "collect_id_l":2765608,
        "some_field_9_i":1191,
        "some_field_10_i":1191,
        "some_field_11_i":1,
        "some_field_12_l":0,
        "some_field_13_l":2000,
        "some_field_14_l":764846,
        "some_field_15_l":766846,
        "some_field_16_i":57,
        "some_field_17_i":1,
        "some_field_18_s":"UTC_-4",
        "some_field_19_i":1487822270,
        "some_field_20_s":"61.221.181.7",
        "some_field_21_l":2869104,
        "some_field_22_s":"[1,4,10,4,5,3,5,6,8,4,3,2,2,1,10]",
        "some_field_23_i":20,
        "some_field_24_i":1,
        "some_field_25_i":1,
        "agent_uid_i":1191,
        "some_field_26_l":2772439,
        "some_field_27_i":100,
        "_yz_id":"1*default*data_201702*2772439*203",
        "_yz_rk":"2772439",
        "_yz_rt":"default",
        "_yz_rb":"data_201702"}]

  }}




NOTFOUND case:

wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":62,
    "params":{
      "10.100.205.80:8093":"_yz_pn:240 OR _yz_pn:210 OR _yz_pn:180 OR _yz_pn:150 OR _yz_pn:120 OR _yz_pn:90 OR _yz_pn:60 OR _yz_pn:30",
      "10.100.205.79:8093":"_yz_pn:249 OR _yz_pn:219 OR _yz_pn:189 OR _yz_pn:159 OR _yz_pn:129 OR _yz_pn:99 OR _yz_pn:69 OR _yz_pn:39 OR _yz_pn:9",
      "indent":"true",
      "10.100.205.73:8093":"(_yz_pn:253 AND (_yz_fpn:253)) OR _yz_pn:243 OR _yz_pn:213 OR _yz_pn:183 OR _yz_pn:153 OR _yz_pn:123 OR _yz_pn:93 OR _yz_pn:63 OR _yz_pn:33 OR _yz_pn:3",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:256 OR _yz_pn:246 OR _yz_pn:216 OR _yz_pn:186 OR _yz_pn:156 OR _yz_pn:126 OR _yz_pn:96 OR _yz_pn:66 OR _yz_pn:36 OR _yz_pn:6",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:231 OR _yz_pn:201 OR _yz_pn:171 OR _yz_pn:141 OR _yz_pn:111 OR _yz_pn:81 OR _yz_pn:51 OR _yz_pn:21",
      "10.100.205.74:8093":"_yz_pn:234 OR _yz_pn:204 OR _yz_pn:174 OR _yz_pn:144 OR _yz_pn:114 OR _yz_pn:84 OR _yz_pn:54 OR _yz_pn:24",
      "10.100.205.77:8093":"_yz_pn:237 OR _yz_pn:207 OR _yz_pn:177 OR _yz_pn:147 OR _yz_pn:117 OR _yz_pn:87 OR _yz_pn:57 OR _yz_pn:27",
      "10.100.205.78:8093":"_yz_pn:228 OR _yz_pn:198 OR _yz_pn:168 OR _yz_pn:138 OR _yz_pn:108 OR _yz_pn:78 OR _yz_pn:48 OR _yz_pn:18",
      "10.100.205.75:8093":"_yz_pn:225 OR _yz_pn:195 OR _yz_pn:165 OR _yz_pn:135 OR _yz_pn:105 OR _yz_pn:75 OR _yz_pn:45 OR _yz_pn:15",
      "wt":"json",
      "10.100.205.72:8093":"_yz_pn:252 OR _yz_pn:222 OR _yz_pn:192 OR _yz_pn:162 OR _yz_pn:132 OR _yz_pn:102 OR _yz_pn:72 OR _yz_pn:42 OR _yz_pn:12"}},
  "response":{"numFound":0,"start":0,"docs":[]
  }}


I can select this record from solr webapp from some of these 10 nodes.  So this record should be indexed by solr.  So I guessed the problem cause this is related with Yokozuna’s distributed shards, since one query would shard to every solr instance by Yokozuna, then Yokozuna would collect all the returns and reduce to a fix result, something like map-reduce mechanism.

So the problem here may underlay in map phase or reduce phase.

The default configuration of riak search applied in these 10 nodes.

Would anyone have some insight on how to fix this?  By modified the configuration of search?


Best regards,
Witeman

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com


_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
Reply | Threaded
Open this post in threaded view
|

Re: Yokozuna's inconsistent query problem

Witeman Zheng
Hello Fred,

riak-admin aae-status:
================================== Exchanges ==================================
Index                                              Last (ago)    All (ago)   
-------------------------------------------------------------------------------
0                                                  12.9 min      14.2 min    
57089907708238395242331438777979805455309864960    9.1 min       30.6 min    
114179815416476790484662877555959610910619729920   7.6 min       29.1 min    
171269723124715185726994316333939416365929594880   6.1 min       27.6 min    
228359630832953580969325755111919221821239459840   4.6 min       26.1 min    
285449538541191976211657193889899027276549324800   3.1 min       24.6 min    
342539446249430371453988632667878832731859189760   1.6 min       23.1 min    
399629353957668766696320071445858638187169054720   6.4 s         21.6 min    
456719261665907161938651510223838443642478919680   56.5 s        36.4 min    
513809169374145557180982949001818249097788784640   18.6 min      38.9 min    
570899077082383952423314387779798054553098649600   17.1 min      37.4 min    
627988984790622347665645826557777860008408514560   15.6 min      35.9 min    
685078892498860742907977265335757665463718379520   14.1 min      34.4 min    
742168800207099138150308704113737470919028244480   12.6 min      32.9 min    
799258707915337533392640142891717276374338109440   11.1 min      31.4 min    
856348615623575928634971581669697081829647974400   9.6 min       29.9 min    
913438523331814323877303020447676887284957839360   8.1 min       28.4 min    
970528431040052719119634459225656692740267704320   6.6 min       26.9 min    
1027618338748291114361965898003636498195577569280  5.1 min       25.4 min    
1084708246456529509604297336781616303650887434240  3.6 min       23.9 min    
1141798154164767904846628775559596109106197299200  2.1 min       22.4 min    
1198888061873006300088960214337575914561507164160  33.2 s        20.9 min    
1255977969581244695331291653115555720016817029120  15.1 min      36.6 min    
1313067877289483090573623091893535525472126894080  13.6 min      35.1 min    
1370157784997721485815954530671515330927436759040  12.1 min      33.6 min    
1427247692705959881058285969449495136382746624000  10.6 min      32.1 min    

================================ Entropy Trees ================================
Index                                              Built (ago)
-------------------------------------------------------------------------------
0                                                  6.8 d
57089907708238395242331438777979805455309864960    14.5 hr
114179815416476790484662877555959610910619729920   2.5 hr
171269723124715185726994316333939416365929594880   6.5 hr
228359630832953580969325755111919221821239459840   5.5 hr
285449538541191976211657193889899027276549324800   7.0 d
342539446249430371453988632667878832731859189760   1.4 d
399629353957668766696320071445858638187169054720   6.6 d
456719261665907161938651510223838443642478919680   1.5 hr
513809169374145557180982949001818249097788784640   4.5 hr
570899077082383952423314387779798054553098649600   1.4 d
627988984790622347665645826557777860008408514560   1.4 d
685078892498860742907977265335757665463718379520   6.7 d
742168800207099138150308704113737470919028244480   6.8 d
799258707915337533392640142891717276374338109440   6.9 d
856348615623575928634971581669697081829647974400   1.3 d
913438523331814323877303020447676887284957839360   3.5 hr
970528431040052719119634459225656692740267704320   6.9 d
1027618338748291114361965898003636498195577569280  7.5 hr
1084708246456529509604297336781616303650887434240  6.9 d
1141798154164767904846628775559596109106197299200  6.7 d
1198888061873006300088960214337575914561507164160  1.4 d
1255977969581244695331291653115555720016817029120  13.5 hr
1313067877289483090573623091893535525472126894080  30.0 min
1370157784997721485815954530671515330927436759040  1.3 d
1427247692705959881058285969449495136382746624000  1.2 d

================================ Keys Repaired ================================
Index                                                Last      Mean      Max   
-------------------------------------------------------------------------------
0                                                     0         0         0    
57089907708238395242331438777979805455309864960       0         0         0    
114179815416476790484662877555959610910619729920      0         0         0    
171269723124715185726994316333939416365929594880      0         0         0    
228359630832953580969325755111919221821239459840      0         0         0    
285449538541191976211657193889899027276549324800      0         0         0    
342539446249430371453988632667878832731859189760      0         0         1    
399629353957668766696320071445858638187169054720      0         0         0    
456719261665907161938651510223838443642478919680      0         0         1    
513809169374145557180982949001818249097788784640      0         0         0    
570899077082383952423314387779798054553098649600      0         0         0    
627988984790622347665645826557777860008408514560      0         0         0    
685078892498860742907977265335757665463718379520      0         0         0    
742168800207099138150308704113737470919028244480      0         0         0    
799258707915337533392640142891717276374338109440      0         0         0    
856348615623575928634971581669697081829647974400      0         0         0    
913438523331814323877303020447676887284957839360      0         0         0    
970528431040052719119634459225656692740267704320      0         0         0    
1027618338748291114361965898003636498195577569280     0         0         0    
1084708246456529509604297336781616303650887434240     0         0         0    
1141798154164767904846628775559596109106197299200     0         0         0    
1198888061873006300088960214337575914561507164160     0         0         0    
1255977969581244695331291653115555720016817029120     0         0         0    
1313067877289483090573623091893535525472126894080     0         0         0    
1370157784997721485815954530671515330927436759040     0         0         0    
1427247692705959881058285969449495136382746624000     0         0         0 

riak-admin search aae-status

================================== Exchanges ==================================
Index                                              Last (ago)    All (ago)   
-------------------------------------------------------------------------------
0                                                  5.1 min       5.6 min     
57089907708238395242331438777979805455309864960    4.4 min       4.9 min     
114179815416476790484662877555959610910619729920   3.6 min       4.1 min     
171269723124715185726994316333939416365929594880   2.9 min       3.4 min     
228359630832953580969325755111919221821239459840   2.1 min       2.6 min     
285449538541191976211657193889899027276549324800   1.4 min       1.9 min     
342539446249430371453988632667878832731859189760   36.5 s        1.1 min     
399629353957668766696320071445858638187169054720   6.5 s         19.4 min    
456719261665907161938651510223838443642478919680   18.6 min      19.1 min    
513809169374145557180982949001818249097788784640   17.9 min      18.4 min    
570899077082383952423314387779798054553098649600   17.1 min      17.6 min    
627988984790622347665645826557777860008408514560   16.4 min      16.9 min    
685078892498860742907977265335757665463718379520   15.6 min      16.1 min    
742168800207099138150308704113737470919028244480   14.9 min      15.4 min    
799258707915337533392640142891717276374338109440   14.1 min      14.6 min    
856348615623575928634971581669697081829647974400   13.4 min      13.9 min    
913438523331814323877303020447676887284957839360   12.6 min      13.1 min    
970528431040052719119634459225656692740267704320   11.9 min      12.4 min    
1027618338748291114361965898003636498195577569280  11.1 min      11.6 min    
1084708246456529509604297336781616303650887434240  10.4 min      10.9 min    
1141798154164767904846628775559596109106197299200  9.6 min       10.1 min    
1198888061873006300088960214337575914561507164160  8.9 min       9.4 min     
1255977969581244695331291653115555720016817029120  8.1 min       8.6 min     
1313067877289483090573623091893535525472126894080  7.4 min       7.9 min     
1370157784997721485815954530671515330927436759040  6.6 min       7.1 min     
1427247692705959881058285969449495136382746624000  5.9 min       6.4 min     

================================ Entropy Trees ================================
Index                                              Built (ago)
-------------------------------------------------------------------------------
0                                                  1.5 d
57089907708238395242331438777979805455309864960    1.2 d
114179815416476790484662877555959610910619729920   20.5 hr
171269723124715185726994316333939416365929594880   1.4 d
228359630832953580969325755111919221821239459840   22.5 hr
285449538541191976211657193889899027276549324800   19.5 hr
342539446249430371453988632667878832731859189760   1.6 d
399629353957668766696320071445858638187169054720   1.1 d
456719261665907161938651510223838443642478919680   1.2 d
513809169374145557180982949001818249097788784640   16.5 hr
570899077082383952423314387779798054553098649600   15.5 hr
627988984790622347665645826557777860008408514560   1.5 d
685078892498860742907977265335757665463718379520   1.4 d
742168800207099138150308704113737470919028244480   21.5 hr
799258707915337533392640142891717276374338109440   18.5 hr
856348615623575928634971581669697081829647974400   1.6 d
913438523331814323877303020447676887284957839360   1.3 d
970528431040052719119634459225656692740267704320   1.1 d
1027618338748291114361965898003636498195577569280  14.5 hr
1084708246456529509604297336781616303650887434240  1.3 d
1141798154164767904846628775559596109106197299200  1.4 d
1198888061873006300088960214337575914561507164160  13.5 hr
1255977969581244695331291653115555720016817029120  1.0 d
1313067877289483090573623091893535525472126894080  23.5 hr
1370157784997721485815954530671515330927436759040  1.1 d
1427247692705959881058285969449495136382746624000  17.5 hr

================================ Keys Repaired ================================
Index                                                Last      Mean      Max   
-------------------------------------------------------------------------------
0                                                     0         0         0    
57089907708238395242331438777979805455309864960       0         0         0    
114179815416476790484662877555959610910619729920      0         0         0    
171269723124715185726994316333939416365929594880      0         0         0    
228359630832953580969325755111919221821239459840      0         0         0    
285449538541191976211657193889899027276549324800      0         0         0    
342539446249430371453988632667878832731859189760      0         0         0    
399629353957668766696320071445858638187169054720      0         0         1    
456719261665907161938651510223838443642478919680      0         0         0    
513809169374145557180982949001818249097788784640      0         0         0    
570899077082383952423314387779798054553098649600      0         0         0    
627988984790622347665645826557777860008408514560      0         0         0    
685078892498860742907977265335757665463718379520      0         0         0    
742168800207099138150308704113737470919028244480      0         0         0    
799258707915337533392640142891717276374338109440      0         0         0    
856348615623575928634971581669697081829647974400      0         0         0    
913438523331814323877303020447676887284957839360      0         0         0    
970528431040052719119634459225656692740267704320      0         0         0    
1027618338748291114361965898003636498195577569280     0         0         0    
1084708246456529509604297336781616303650887434240     0         0         1    
1141798154164767904846628775559596109106197299200     1         0         1    
1198888061873006300088960214337575914561507164160     0         0         0    
1255977969581244695331291653115555720016817029120     0         0         0    
1313067877289483090573623091893535525472126894080     0         0         0    
1370157784997721485815954530671515330927436759040     0         0         0    
1427247692705959881058285969449495136382746624000     0         0         0


I have tried to read the data from RiakKV with r=3, it successfully retrieves the data of this entry(read repair maybe fail).

But to use Riak Search Query again, it still returns results in discrepancy.

I am curious that why just map to all nodes then once the data in the range got, then return and reduce.  I found that this entry in solr via solr-webapp, it is located in 2 nodes of 10. While some other entry located in 3 nodes of 10.

Best regards,
Witeman

On 24 Feb 2017, at 11:43 AM, Fred Dushin <[hidden email]> wrote:

Hello Witeman,

What you are seeing with your two queries is the result of two different coverage plans, querying different parts of the cluster.  Riak Search translates coverage plans to Solr sharded queries, and will periodically change the coverage plan, so as to more evenly distribute queries across the cluster.  So what you are seeing is effectively two different sharded queries, hitting different solr instances in your cluster.  You are seeing inconsistent search results, which suggests there is a discrepancy between what is stored in Solr and what is stored in Riak.  Generally speaking, AAE should detect these repairs and correct them over time.

Can you please send the output of the following two commands:

riak-admin aae-status
riak-admin search aae-status

That will tell you something about the behavior of the underlying KV and Yokozuna AAE subsystems.

Out of curiosity, can you you read the "2772439" key in the "data_201702" bucket in Riak with quorum equal to all?  If you do that read, does that affect the behavior of your query you posted?  (I am wondering if you trigger read repair that will repair the entry in Solr)

Thanks,
-Fred

On Feb 23, 2017, at 10:11 PM, Witeman Zheng <[hidden email]> wrote:

Hi,


I am having a 10 nodes of RiakKV 2.2.0, and turn on Riak Search(Yokozuna).  Having about 3million records in one bucket with index, every record has about 1k size.

Then when it is triggered a Yokozuna query for one specific id, sometimes return the record, sometimes return NOT FOUND, it is very weird.

FOUND case:
wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":57,
    "params":{
      "10.100.205.80:8093":"_yz_pn:230 OR _yz_pn:200 OR _yz_pn:170 OR _yz_pn:140 OR _yz_pn:110 OR _yz_pn:80 OR _yz_pn:50 OR _yz_pn:20",
      "10.100.205.79:8093":"_yz_pn:239 OR _yz_pn:209 OR _yz_pn:179 OR _yz_pn:149 OR _yz_pn:119 OR _yz_pn:89 OR _yz_pn:59 OR _yz_pn:29",
      "indent":"true",
      "10.100.205.73:8093":"_yz_pn:233 OR _yz_pn:203 OR _yz_pn:173 OR _yz_pn:143 OR _yz_pn:113 OR _yz_pn:83 OR _yz_pn:53 OR _yz_pn:23",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:236 OR _yz_pn:206 OR _yz_pn:176 OR _yz_pn:146 OR _yz_pn:116 OR _yz_pn:86 OR _yz_pn:56 OR _yz_pn:26",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:251 OR _yz_pn:221 OR _yz_pn:191 OR _yz_pn:161 OR _yz_pn:131 OR _yz_pn:101 OR _yz_pn:71 OR _yz_pn:41 OR _yz_pn:11",
      "10.100.205.74:8093":"_yz_pn:224 OR _yz_pn:194 OR _yz_pn:164 OR _yz_pn:134 OR _yz_pn:104 OR _yz_pn:74 OR _yz_pn:44 OR _yz_pn:14",
      "10.100.205.77:8093":"_yz_pn:227 OR _yz_pn:197 OR _yz_pn:167 OR _yz_pn:137 OR _yz_pn:107 OR _yz_pn:77 OR _yz_pn:47 OR _yz_pn:17",
      "10.100.205.78:8093":"_yz_pn:248 OR _yz_pn:218 OR _yz_pn:188 OR _yz_pn:158 OR _yz_pn:128 OR _yz_pn:98 OR _yz_pn:68 OR _yz_pn:38 OR _yz_pn:8",
      "10.100.205.75:8093":"_yz_pn:255 OR _yz_pn:245 OR _yz_pn:215 OR _yz_pn:185 OR _yz_pn:155 OR _yz_pn:125 OR _yz_pn:95 OR _yz_pn:65 OR _yz_pn:35 OR _yz_pn:5",
      "wt":"json",
      "10.100.205.72:8093":"(_yz_pn:252 AND (_yz_fpn:252)) OR _yz_pn:242 OR _yz_pn:212 OR _yz_pn:182 OR _yz_pn:152 OR _yz_pn:122 OR _yz_pn:92 OR _yz_pn:62 OR _yz_pn:32 OR _yz_pn:2"}},
  "response":{"numFound":1,"start":0,"docs":[
      {
        “some_field_1_l":0,
        "some_field_2_l":0,
        "some_field_3_s":"[]",
        "some_field_4_s":"[]",
        "some_field_5_l":0,
        "some_field_6_l":0,
        "some_field_7_s":"[]",
        "some_field_8_s":"[[1,0],[2,0],[3,0],[4,0],[5,0],[6,0],[7,0],[8,0],[9,0],[10,0],[11,0],[12,0],[13,0],[14,0],[15,0],[16,0],[17,0],[18,0],[19,0],[20,0]]",
        "collect_id_l":2765608,
        "some_field_9_i":1191,
        "some_field_10_i":1191,
        "some_field_11_i":1,
        "some_field_12_l":0,
        "some_field_13_l":2000,
        "some_field_14_l":764846,
        "some_field_15_l":766846,
        "some_field_16_i":57,
        "some_field_17_i":1,
        "some_field_18_s":"UTC_-4",
        "some_field_19_i":1487822270,
        "some_field_20_s":"61.221.181.7",
        "some_field_21_l":2869104,
        "some_field_22_s":"[1,4,10,4,5,3,5,6,8,4,3,2,2,1,10]",
        "some_field_23_i":20,
        "some_field_24_i":1,
        "some_field_25_i":1,
        "agent_uid_i":1191,
        "some_field_26_l":2772439,
        "some_field_27_i":100,
        "_yz_id":"1*default*data_201702*2772439*203",
        "_yz_rk":"2772439",
        "_yz_rt":"default",
        "_yz_rb":"data_201702"}]

  }}




NOTFOUND case:

wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":62,
    "params":{
      "10.100.205.80:8093":"_yz_pn:240 OR _yz_pn:210 OR _yz_pn:180 OR _yz_pn:150 OR _yz_pn:120 OR _yz_pn:90 OR _yz_pn:60 OR _yz_pn:30",
      "10.100.205.79:8093":"_yz_pn:249 OR _yz_pn:219 OR _yz_pn:189 OR _yz_pn:159 OR _yz_pn:129 OR _yz_pn:99 OR _yz_pn:69 OR _yz_pn:39 OR _yz_pn:9",
      "indent":"true",
      "10.100.205.73:8093":"(_yz_pn:253 AND (_yz_fpn:253)) OR _yz_pn:243 OR _yz_pn:213 OR _yz_pn:183 OR _yz_pn:153 OR _yz_pn:123 OR _yz_pn:93 OR _yz_pn:63 OR _yz_pn:33 OR _yz_pn:3",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:256 OR _yz_pn:246 OR _yz_pn:216 OR _yz_pn:186 OR _yz_pn:156 OR _yz_pn:126 OR _yz_pn:96 OR _yz_pn:66 OR _yz_pn:36 OR _yz_pn:6",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:231 OR _yz_pn:201 OR _yz_pn:171 OR _yz_pn:141 OR _yz_pn:111 OR _yz_pn:81 OR _yz_pn:51 OR _yz_pn:21",
      "10.100.205.74:8093":"_yz_pn:234 OR _yz_pn:204 OR _yz_pn:174 OR _yz_pn:144 OR _yz_pn:114 OR _yz_pn:84 OR _yz_pn:54 OR _yz_pn:24",
      "10.100.205.77:8093":"_yz_pn:237 OR _yz_pn:207 OR _yz_pn:177 OR _yz_pn:147 OR _yz_pn:117 OR _yz_pn:87 OR _yz_pn:57 OR _yz_pn:27",
      "10.100.205.78:8093":"_yz_pn:228 OR _yz_pn:198 OR _yz_pn:168 OR _yz_pn:138 OR _yz_pn:108 OR _yz_pn:78 OR _yz_pn:48 OR _yz_pn:18",
      "10.100.205.75:8093":"_yz_pn:225 OR _yz_pn:195 OR _yz_pn:165 OR _yz_pn:135 OR _yz_pn:105 OR _yz_pn:75 OR _yz_pn:45 OR _yz_pn:15",
      "wt":"json",
      "10.100.205.72:8093":"_yz_pn:252 OR _yz_pn:222 OR _yz_pn:192 OR _yz_pn:162 OR _yz_pn:132 OR _yz_pn:102 OR _yz_pn:72 OR _yz_pn:42 OR _yz_pn:12"}},
  "response":{"numFound":0,"start":0,"docs":[]
  }}


I can select this record from solr webapp from some of these 10 nodes.  So this record should be indexed by solr.  So I guessed the problem cause this is related with Yokozuna’s distributed shards, since one query would shard to every solr instance by Yokozuna, then Yokozuna would collect all the returns and reduce to a fix result, something like map-reduce mechanism.

So the problem here may underlay in map phase or reduce phase.

The default configuration of riak search applied in these 10 nodes.

Would anyone have some insight on how to fix this?  By modified the configuration of search?


Best regards,
Witeman

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com


_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com
Reply | Threaded
Open this post in threaded view
|

Re: Yokozuna's inconsistent query problem

Witeman Zheng
Curious enough, when I leaved 5 of 10 nodes, everything goes well again.

No index missed.

But when I had 10 nodes, the read repair cannot help to find back the index.

I found RiakTS is still in very beginning stage, I nearly try to shift from RiakKV + YZ to RiakTS.


Best regards,
Witeman
On 24 Feb 2017, at 3:26 PM, Witeman Zheng <[hidden email]> wrote:

Hello Fred,

riak-admin aae-status:
================================== Exchanges ==================================
Index                                              Last (ago)    All (ago)   
-------------------------------------------------------------------------------
0                                                  12.9 min      14.2 min    
57089907708238395242331438777979805455309864960    9.1 min       30.6 min    
114179815416476790484662877555959610910619729920   7.6 min       29.1 min    
171269723124715185726994316333939416365929594880   6.1 min       27.6 min    
228359630832953580969325755111919221821239459840   4.6 min       26.1 min    
285449538541191976211657193889899027276549324800   3.1 min       24.6 min    
342539446249430371453988632667878832731859189760   1.6 min       23.1 min    
399629353957668766696320071445858638187169054720   6.4 s         21.6 min    
456719261665907161938651510223838443642478919680   56.5 s        36.4 min    
513809169374145557180982949001818249097788784640   18.6 min      38.9 min    
570899077082383952423314387779798054553098649600   17.1 min      37.4 min    
627988984790622347665645826557777860008408514560   15.6 min      35.9 min    
685078892498860742907977265335757665463718379520   14.1 min      34.4 min    
742168800207099138150308704113737470919028244480   12.6 min      32.9 min    
799258707915337533392640142891717276374338109440   11.1 min      31.4 min    
856348615623575928634971581669697081829647974400   9.6 min       29.9 min    
913438523331814323877303020447676887284957839360   8.1 min       28.4 min    
970528431040052719119634459225656692740267704320   6.6 min       26.9 min    
1027618338748291114361965898003636498195577569280  5.1 min       25.4 min    
1084708246456529509604297336781616303650887434240  3.6 min       23.9 min    
1141798154164767904846628775559596109106197299200  2.1 min       22.4 min    
1198888061873006300088960214337575914561507164160  33.2 s        20.9 min    
1255977969581244695331291653115555720016817029120  15.1 min      36.6 min    
1313067877289483090573623091893535525472126894080  13.6 min      35.1 min    
1370157784997721485815954530671515330927436759040  12.1 min      33.6 min    
1427247692705959881058285969449495136382746624000  10.6 min      32.1 min    

================================ Entropy Trees ================================
Index                                              Built (ago)
-------------------------------------------------------------------------------
0                                                  6.8 d
57089907708238395242331438777979805455309864960    14.5 hr
114179815416476790484662877555959610910619729920   2.5 hr
171269723124715185726994316333939416365929594880   6.5 hr
228359630832953580969325755111919221821239459840   5.5 hr
285449538541191976211657193889899027276549324800   7.0 d
342539446249430371453988632667878832731859189760   1.4 d
399629353957668766696320071445858638187169054720   6.6 d
456719261665907161938651510223838443642478919680   1.5 hr
513809169374145557180982949001818249097788784640   4.5 hr
570899077082383952423314387779798054553098649600   1.4 d
627988984790622347665645826557777860008408514560   1.4 d
685078892498860742907977265335757665463718379520   6.7 d
742168800207099138150308704113737470919028244480   6.8 d
799258707915337533392640142891717276374338109440   6.9 d
856348615623575928634971581669697081829647974400   1.3 d
913438523331814323877303020447676887284957839360   3.5 hr
970528431040052719119634459225656692740267704320   6.9 d
1027618338748291114361965898003636498195577569280  7.5 hr
1084708246456529509604297336781616303650887434240  6.9 d
1141798154164767904846628775559596109106197299200  6.7 d
1198888061873006300088960214337575914561507164160  1.4 d
1255977969581244695331291653115555720016817029120  13.5 hr
1313067877289483090573623091893535525472126894080  30.0 min
1370157784997721485815954530671515330927436759040  1.3 d
1427247692705959881058285969449495136382746624000  1.2 d

================================ Keys Repaired ================================
Index                                                Last      Mean      Max   
-------------------------------------------------------------------------------
0                                                     0         0         0    
57089907708238395242331438777979805455309864960       0         0         0    
114179815416476790484662877555959610910619729920      0         0         0    
171269723124715185726994316333939416365929594880      0         0         0    
228359630832953580969325755111919221821239459840      0         0         0    
285449538541191976211657193889899027276549324800      0         0         0    
342539446249430371453988632667878832731859189760      0         0         1    
399629353957668766696320071445858638187169054720      0         0         0    
456719261665907161938651510223838443642478919680      0         0         1    
513809169374145557180982949001818249097788784640      0         0         0    
570899077082383952423314387779798054553098649600      0         0         0    
627988984790622347665645826557777860008408514560      0         0         0    
685078892498860742907977265335757665463718379520      0         0         0    
742168800207099138150308704113737470919028244480      0         0         0    
799258707915337533392640142891717276374338109440      0         0         0    
856348615623575928634971581669697081829647974400      0         0         0    
913438523331814323877303020447676887284957839360      0         0         0    
970528431040052719119634459225656692740267704320      0         0         0    
1027618338748291114361965898003636498195577569280     0         0         0    
1084708246456529509604297336781616303650887434240     0         0         0    
1141798154164767904846628775559596109106197299200     0         0         0    
1198888061873006300088960214337575914561507164160     0         0         0    
1255977969581244695331291653115555720016817029120     0         0         0    
1313067877289483090573623091893535525472126894080     0         0         0    
1370157784997721485815954530671515330927436759040     0         0         0    
1427247692705959881058285969449495136382746624000     0         0         0 

riak-admin search aae-status

================================== Exchanges ==================================
Index                                              Last (ago)    All (ago)   
-------------------------------------------------------------------------------
0                                                  5.1 min       5.6 min     
57089907708238395242331438777979805455309864960    4.4 min       4.9 min     
114179815416476790484662877555959610910619729920   3.6 min       4.1 min     
171269723124715185726994316333939416365929594880   2.9 min       3.4 min     
228359630832953580969325755111919221821239459840   2.1 min       2.6 min     
285449538541191976211657193889899027276549324800   1.4 min       1.9 min     
342539446249430371453988632667878832731859189760   36.5 s        1.1 min     
399629353957668766696320071445858638187169054720   6.5 s         19.4 min    
456719261665907161938651510223838443642478919680   18.6 min      19.1 min    
513809169374145557180982949001818249097788784640   17.9 min      18.4 min    
570899077082383952423314387779798054553098649600   17.1 min      17.6 min    
627988984790622347665645826557777860008408514560   16.4 min      16.9 min    
685078892498860742907977265335757665463718379520   15.6 min      16.1 min    
742168800207099138150308704113737470919028244480   14.9 min      15.4 min    
799258707915337533392640142891717276374338109440   14.1 min      14.6 min    
856348615623575928634971581669697081829647974400   13.4 min      13.9 min    
913438523331814323877303020447676887284957839360   12.6 min      13.1 min    
970528431040052719119634459225656692740267704320   11.9 min      12.4 min    
1027618338748291114361965898003636498195577569280  11.1 min      11.6 min    
1084708246456529509604297336781616303650887434240  10.4 min      10.9 min    
1141798154164767904846628775559596109106197299200  9.6 min       10.1 min    
1198888061873006300088960214337575914561507164160  8.9 min       9.4 min     
1255977969581244695331291653115555720016817029120  8.1 min       8.6 min     
1313067877289483090573623091893535525472126894080  7.4 min       7.9 min     
1370157784997721485815954530671515330927436759040  6.6 min       7.1 min     
1427247692705959881058285969449495136382746624000  5.9 min       6.4 min     

================================ Entropy Trees ================================
Index                                              Built (ago)
-------------------------------------------------------------------------------
0                                                  1.5 d
57089907708238395242331438777979805455309864960    1.2 d
114179815416476790484662877555959610910619729920   20.5 hr
171269723124715185726994316333939416365929594880   1.4 d
228359630832953580969325755111919221821239459840   22.5 hr
285449538541191976211657193889899027276549324800   19.5 hr
342539446249430371453988632667878832731859189760   1.6 d
399629353957668766696320071445858638187169054720   1.1 d
456719261665907161938651510223838443642478919680   1.2 d
513809169374145557180982949001818249097788784640   16.5 hr
570899077082383952423314387779798054553098649600   15.5 hr
627988984790622347665645826557777860008408514560   1.5 d
685078892498860742907977265335757665463718379520   1.4 d
742168800207099138150308704113737470919028244480   21.5 hr
799258707915337533392640142891717276374338109440   18.5 hr
856348615623575928634971581669697081829647974400   1.6 d
913438523331814323877303020447676887284957839360   1.3 d
970528431040052719119634459225656692740267704320   1.1 d
1027618338748291114361965898003636498195577569280  14.5 hr
1084708246456529509604297336781616303650887434240  1.3 d
1141798154164767904846628775559596109106197299200  1.4 d
1198888061873006300088960214337575914561507164160  13.5 hr
1255977969581244695331291653115555720016817029120  1.0 d
1313067877289483090573623091893535525472126894080  23.5 hr
1370157784997721485815954530671515330927436759040  1.1 d
1427247692705959881058285969449495136382746624000  17.5 hr

================================ Keys Repaired ================================
Index                                                Last      Mean      Max   
-------------------------------------------------------------------------------
0                                                     0         0         0    
57089907708238395242331438777979805455309864960       0         0         0    
114179815416476790484662877555959610910619729920      0         0         0    
171269723124715185726994316333939416365929594880      0         0         0    
228359630832953580969325755111919221821239459840      0         0         0    
285449538541191976211657193889899027276549324800      0         0         0    
342539446249430371453988632667878832731859189760      0         0         0    
399629353957668766696320071445858638187169054720      0         0         1    
456719261665907161938651510223838443642478919680      0         0         0    
513809169374145557180982949001818249097788784640      0         0         0    
570899077082383952423314387779798054553098649600      0         0         0    
627988984790622347665645826557777860008408514560      0         0         0    
685078892498860742907977265335757665463718379520      0         0         0    
742168800207099138150308704113737470919028244480      0         0         0    
799258707915337533392640142891717276374338109440      0         0         0    
856348615623575928634971581669697081829647974400      0         0         0    
913438523331814323877303020447676887284957839360      0         0         0    
970528431040052719119634459225656692740267704320      0         0         0    
1027618338748291114361965898003636498195577569280     0         0         0    
1084708246456529509604297336781616303650887434240     0         0         1    
1141798154164767904846628775559596109106197299200     1         0         1    
1198888061873006300088960214337575914561507164160     0         0         0    
1255977969581244695331291653115555720016817029120     0         0         0    
1313067877289483090573623091893535525472126894080     0         0         0    
1370157784997721485815954530671515330927436759040     0         0         0    
1427247692705959881058285969449495136382746624000     0         0         0


I have tried to read the data from RiakKV with r=3, it successfully retrieves the data of this entry(read repair maybe fail).

But to use Riak Search Query again, it still returns results in discrepancy.

I am curious that why just map to all nodes then once the data in the range got, then return and reduce.  I found that this entry in solr via solr-webapp, it is located in 2 nodes of 10. While some other entry located in 3 nodes of 10.

Best regards,
Witeman

On 24 Feb 2017, at 11:43 AM, Fred Dushin <[hidden email]> wrote:

Hello Witeman,

What you are seeing with your two queries is the result of two different coverage plans, querying different parts of the cluster.  Riak Search translates coverage plans to Solr sharded queries, and will periodically change the coverage plan, so as to more evenly distribute queries across the cluster.  So what you are seeing is effectively two different sharded queries, hitting different solr instances in your cluster.  You are seeing inconsistent search results, which suggests there is a discrepancy between what is stored in Solr and what is stored in Riak.  Generally speaking, AAE should detect these repairs and correct them over time.

Can you please send the output of the following two commands:

riak-admin aae-status
riak-admin search aae-status

That will tell you something about the behavior of the underlying KV and Yokozuna AAE subsystems.

Out of curiosity, can you you read the "2772439" key in the "data_201702" bucket in Riak with quorum equal to all?  If you do that read, does that affect the behavior of your query you posted?  (I am wondering if you trigger read repair that will repair the entry in Solr)

Thanks,
-Fred

On Feb 23, 2017, at 10:11 PM, Witeman Zheng <[hidden email]> wrote:

Hi,


I am having a 10 nodes of RiakKV 2.2.0, and turn on Riak Search(Yokozuna).  Having about 3million records in one bucket with index, every record has about 1k size.

Then when it is triggered a Yokozuna query for one specific id, sometimes return the record, sometimes return NOT FOUND, it is very weird.

FOUND case:
wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":57,
    "params":{
      "10.100.205.80:8093":"_yz_pn:230 OR _yz_pn:200 OR _yz_pn:170 OR _yz_pn:140 OR _yz_pn:110 OR _yz_pn:80 OR _yz_pn:50 OR _yz_pn:20",
      "10.100.205.79:8093":"_yz_pn:239 OR _yz_pn:209 OR _yz_pn:179 OR _yz_pn:149 OR _yz_pn:119 OR _yz_pn:89 OR _yz_pn:59 OR _yz_pn:29",
      "indent":"true",
      "10.100.205.73:8093":"_yz_pn:233 OR _yz_pn:203 OR _yz_pn:173 OR _yz_pn:143 OR _yz_pn:113 OR _yz_pn:83 OR _yz_pn:53 OR _yz_pn:23",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:236 OR _yz_pn:206 OR _yz_pn:176 OR _yz_pn:146 OR _yz_pn:116 OR _yz_pn:86 OR _yz_pn:56 OR _yz_pn:26",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:251 OR _yz_pn:221 OR _yz_pn:191 OR _yz_pn:161 OR _yz_pn:131 OR _yz_pn:101 OR _yz_pn:71 OR _yz_pn:41 OR _yz_pn:11",
      "10.100.205.74:8093":"_yz_pn:224 OR _yz_pn:194 OR _yz_pn:164 OR _yz_pn:134 OR _yz_pn:104 OR _yz_pn:74 OR _yz_pn:44 OR _yz_pn:14",
      "10.100.205.77:8093":"_yz_pn:227 OR _yz_pn:197 OR _yz_pn:167 OR _yz_pn:137 OR _yz_pn:107 OR _yz_pn:77 OR _yz_pn:47 OR _yz_pn:17",
      "10.100.205.78:8093":"_yz_pn:248 OR _yz_pn:218 OR _yz_pn:188 OR _yz_pn:158 OR _yz_pn:128 OR _yz_pn:98 OR _yz_pn:68 OR _yz_pn:38 OR _yz_pn:8",
      "10.100.205.75:8093":"_yz_pn:255 OR _yz_pn:245 OR _yz_pn:215 OR _yz_pn:185 OR _yz_pn:155 OR _yz_pn:125 OR _yz_pn:95 OR _yz_pn:65 OR _yz_pn:35 OR _yz_pn:5",
      "wt":"json",
      "10.100.205.72:8093":"(_yz_pn:252 AND (_yz_fpn:252)) OR _yz_pn:242 OR _yz_pn:212 OR _yz_pn:182 OR _yz_pn:152 OR _yz_pn:122 OR _yz_pn:92 OR _yz_pn:62 OR _yz_pn:32 OR _yz_pn:2"}},
  "response":{"numFound":1,"start":0,"docs":[
      {
        “some_field_1_l":0,
        "some_field_2_l":0,
        "some_field_3_s":"[]",
        "some_field_4_s":"[]",
        "some_field_5_l":0,
        "some_field_6_l":0,
        "some_field_7_s":"[]",
        "some_field_8_s":"[[1,0],[2,0],[3,0],[4,0],[5,0],[6,0],[7,0],[8,0],[9,0],[10,0],[11,0],[12,0],[13,0],[14,0],[15,0],[16,0],[17,0],[18,0],[19,0],[20,0]]",
        "collect_id_l":2765608,
        "some_field_9_i":1191,
        "some_field_10_i":1191,
        "some_field_11_i":1,
        "some_field_12_l":0,
        "some_field_13_l":2000,
        "some_field_14_l":764846,
        "some_field_15_l":766846,
        "some_field_16_i":57,
        "some_field_17_i":1,
        "some_field_18_s":"UTC_-4",
        "some_field_19_i":1487822270,
        "some_field_20_s":"61.221.181.7",
        "some_field_21_l":2869104,
        "some_field_22_s":"[1,4,10,4,5,3,5,6,8,4,3,2,2,1,10]",
        "some_field_23_i":20,
        "some_field_24_i":1,
        "some_field_25_i":1,
        "agent_uid_i":1191,
        "some_field_26_l":2772439,
        "some_field_27_i":100,
        "_yz_id":"1*default*data_201702*2772439*203",
        "_yz_rk":"2772439",
        "_yz_rt":"default",
        "_yz_rb":"data_201702"}]

  }}




NOTFOUND case:

wt=json&q=*:*&rows=1000&start=0&sort=collect_id_l%20asc&fq=agent_uid_i:1191&fq=id_l:2772439&indent=true"
{
  "responseHeader":{
    "status":0,
    "QTime":62,
    "params":{
      "10.100.205.80:8093":"_yz_pn:240 OR _yz_pn:210 OR _yz_pn:180 OR _yz_pn:150 OR _yz_pn:120 OR _yz_pn:90 OR _yz_pn:60 OR _yz_pn:30",
      "10.100.205.79:8093":"_yz_pn:249 OR _yz_pn:219 OR _yz_pn:189 OR _yz_pn:159 OR _yz_pn:129 OR _yz_pn:99 OR _yz_pn:69 OR _yz_pn:39 OR _yz_pn:9",
      "indent":"true",
      "10.100.205.73:8093":"(_yz_pn:253 AND (_yz_fpn:253)) OR _yz_pn:243 OR _yz_pn:213 OR _yz_pn:183 OR _yz_pn:153 OR _yz_pn:123 OR _yz_pn:93 OR _yz_pn:63 OR _yz_pn:33 OR _yz_pn:3",
      "start":"0",
      "sort":"collect_id_l asc",
      "fq":["agent_uid_i:1191",
        "id_l:2772439"],
      "rows":"1000",
      "10.100.205.76:8093":"_yz_pn:256 OR _yz_pn:246 OR _yz_pn:216 OR _yz_pn:186 OR _yz_pn:156 OR _yz_pn:126 OR _yz_pn:96 OR _yz_pn:66 OR _yz_pn:36 OR _yz_pn:6",
      "q":"*:*",
      "shards":"10.100.205.71:8093/internal_solr/game_data_records_index,10.100.205.72:8093/internal_solr/game_data_records_index,10.100.205.73:8093/internal_solr/game_data_records_index,10.100.205.74:8093/internal_solr/game_data_records_index,10.100.205.75:8093/internal_solr/game_data_records_index,10.100.205.76:8093/internal_solr/game_data_records_index,10.100.205.77:8093/internal_solr/game_data_records_index,10.100.205.78:8093/internal_solr/game_data_records_index,10.100.205.79:8093/internal_solr/game_data_records_index,10.100.205.80:8093/internal_solr/game_data_records_index",
      "10.100.205.71:8093":"_yz_pn:231 OR _yz_pn:201 OR _yz_pn:171 OR _yz_pn:141 OR _yz_pn:111 OR _yz_pn:81 OR _yz_pn:51 OR _yz_pn:21",
      "10.100.205.74:8093":"_yz_pn:234 OR _yz_pn:204 OR _yz_pn:174 OR _yz_pn:144 OR _yz_pn:114 OR _yz_pn:84 OR _yz_pn:54 OR _yz_pn:24",
      "10.100.205.77:8093":"_yz_pn:237 OR _yz_pn:207 OR _yz_pn:177 OR _yz_pn:147 OR _yz_pn:117 OR _yz_pn:87 OR _yz_pn:57 OR _yz_pn:27",
      "10.100.205.78:8093":"_yz_pn:228 OR _yz_pn:198 OR _yz_pn:168 OR _yz_pn:138 OR _yz_pn:108 OR _yz_pn:78 OR _yz_pn:48 OR _yz_pn:18",
      "10.100.205.75:8093":"_yz_pn:225 OR _yz_pn:195 OR _yz_pn:165 OR _yz_pn:135 OR _yz_pn:105 OR _yz_pn:75 OR _yz_pn:45 OR _yz_pn:15",
      "wt":"json",
      "10.100.205.72:8093":"_yz_pn:252 OR _yz_pn:222 OR _yz_pn:192 OR _yz_pn:162 OR _yz_pn:132 OR _yz_pn:102 OR _yz_pn:72 OR _yz_pn:42 OR _yz_pn:12"}},
  "response":{"numFound":0,"start":0,"docs":[]
  }}


I can select this record from solr webapp from some of these 10 nodes.  So this record should be indexed by solr.  So I guessed the problem cause this is related with Yokozuna’s distributed shards, since one query would shard to every solr instance by Yokozuna, then Yokozuna would collect all the returns and reduce to a fix result, something like map-reduce mechanism.

So the problem here may underlay in map phase or reduce phase.

The default configuration of riak search applied in these 10 nodes.

Would anyone have some insight on how to fix this?  By modified the configuration of search?


Best regards,
Witeman

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com

_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com



_______________________________________________
riak-users mailing list
[hidden email]
http://lists.basho.com/mailman/listinfo/riak-users_lists.basho.com