前言:
此话题说来话长,老大让写多个查询接口,尽量考虑索引使用,因为数据非常多,不使用索引查询很慢,于是乎,我写了一个通用查询接口,指定索引生成query对象。根据mongo权威指南的说法 建立复合索引 A_1_B_1_C_1 此时可用索引为 A-B-C |A-B| A 三种, 跟DBA说了讨论了这个问题(PS:我也不想讨论但是已经写完了真的懒得改了),老大又说不能使用通用的DB层查询接口,索引也要为业务服务,于是乎我们3个人开始验证索引顺序对查询的影响,半小时被打N次脸,难道mongo权威指南是盗版书? DBA 测试 A-B 和 B-A 都是用复合索引 A_1_B_1_C_1 且实际扫描文档数 和 返回文档树一致 2者相同。这就有点不科学了。但事实就摆在那里。。。一定有什么误会于是回家做实验了。
结论(PS 正文太长放结论先):
假设存在索引 A_1_B_1_C_1
AB与BA 都可以正常使用索引效率相同
AC 可使用索引效率较低
BC 不可使用索引 进入 COLLSCAN stage
AXBC 可使用索引与ABCX效率相同
A:{$exists:true}BC 可使用索引 KEEP_MUTATIONS stage 需要复验后返回对性能有影响(很小)
SORT 情况
sort 情况顺序不可变,不可出现 AC 此时无法使用索引排序进入 SORT stage 是、排序效率降低。
即A存在在查询方面可无视对象排序,只排序时关注索引定义。prefix原则是只要存在index首字段即可并不需要考虑位置。
正文:
以公司的测试服务器的数据做实验 ,执行db.collection.getIndexes() 方法,此文档有大约27万数据。我们使用第8个复合索引作为测试索引
{
"v" : 1,
"key" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"name" : "state_1_user_1_stock_1",
"ns" : "mango_order.od_reservations",
"background" : true
}
我们来看最佳 explain 即ABC db.od_reservations.find({state:1,user:32432424,stock:3243242}).explain(“executionStats”)
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"state" : {
"$eq" : 1
}
},
{
"stock" : {
"$eq" : 3243242
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[1.0, 1.0]"
],
"user" : [
"[MinKey, MaxKey]"
],
"stock" : [
"[3243242.0, 3243242.0]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 157,
"totalKeysExamined" : 29806,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 160,
"works" : 29806,
"advanced" : 0,
"needTime" : 29805,
"needYield" : 0,
"saveState" : 232,
"restoreState" : 232,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 160,
"works" : 29806,
"advanced" : 0,
"needTime" : 29805,
"needYield" : 0,
"saveState" : 232,
"restoreState" : 232,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[1.0, 1.0]"
],
"user" : [
"[MinKey, MaxKey]"
],
"stock" : [
"[3243242.0, 3243242.0]"
]
},
"keysExamined" : 29806,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
接下来尝试 A_C 检索 db.od_reservations.find({state:1,stock:3243242}).explain(“executionStats”) 证明 A_C 使用该复合索引
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"state" : {
"$eq" : 1
}
},
{
"stock" : {
"$eq" : 3243242
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[1.0, 1.0]"
],
"user" : [
"[MinKey, MaxKey]"
],
"stock" : [
"[3243242.0, 3243242.0]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 157,
"totalKeysExamined" : 29806,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 160,
"works" : 29806,
"advanced" : 0,
"needTime" : 29805,
"needYield" : 0,
"saveState" : 232,
"restoreState" : 232,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 160,
"works" : 29806,
"advanced" : 0,
"needTime" : 29805,
"needYield" : 0,
"saveState" : 232,
"restoreState" : 232,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[1.0, 1.0]"
],
"user" : [
"[MinKey, MaxKey]"
],
"stock" : [
"[3243242.0, 3243242.0]"
]
},
"keysExamined" : 29806,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
接下来验证 _BC db.od_reservations.find({user:32432424,stock:3243242}).explain(“executionStats”) 证明 _BC 不使用索引
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"stock" : {
"$eq" : 3243242
}
},
{
"user" : {
"$eq" : 32432424
}
}
]
},
"winningPlan" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"stock" : {
"$eq" : 3243242
}
},
{
"user" : {
"$eq" : 32432424
}
}
]
},
"direction" : "forward"
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 92,
"totalKeysExamined" : 0,
"totalDocsExamined" : 277389,
"executionStages" : {
"stage" : "COLLSCAN",
"filter" : {
"$and" : [
{
"stock" : {
"$eq" : 3243242
}
},
{
"user" : {
"$eq" : 32432424
}
}
]
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 80,
"works" : 277401,
"advanced" : 0,
"needTime" : 277390,
"needYield" : 10,
"saveState" : 2171,
"restoreState" : 2171,
"isEOF" : 1,
"invalidates" : 0,
"direction" : "forward",
"docsExamined" : 277389
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
接下来验证 BA 是否使用索引 db.od_reservations.find({user:32432424,state:1}).explain(“executionStats”) 证明DBA正确,此时使用了复合索引 我们接下来看看和AB是否有性能差距
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"state" : {
"$eq" : 1
}
},
{
"user" : {
"$eq" : 32432424
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[1.0, 1.0]"
],
"user" : [
"[32432424.0, 32432424.0]"
],
"stock" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[1.0, 1.0]"
],
"user" : [
"[32432424.0, 32432424.0]"
],
"stock" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
验证 AB db.od_reservations.find({state:1,user:32432424}).explain(“executionStats”) 证明 BA与AB索引效率相同,AB可高效使用索引而AC则不行。
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"state" : {
"$eq" : "1703282329098"
}
},
{
"user" : {
"$eq" : 32432424
}
}
]
},
"winningPlan" : {
"stage" : "FETCH",
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[\"1703282329098\", \"1703282329098\"]"
],
"user" : [
"[32432424.0, 32432424.0]"
],
"stock" : [
"[MinKey, MaxKey]"
]
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 0,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "FETCH",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 1,
"advanced" : 0,
"needTime" : 0,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[\"1703282329098\", \"1703282329098\"]"
],
"user" : [
"[32432424.0, 32432424.0]"
],
"stock" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 0,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
验证 $exists A:$exists—B db.od_reservations.find({state:{$exists: true},user:32432424}).explain(“executionStats”) 结论 $exites 返回多结果对性能有影响(
KEEP_UTATIONS stage 表示检索到结果后,在检索过程中的删除或者修改,会触发一个复验的过程,当复验成功文档将被返回
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"user" : {
"$eq" : 32432424
}
},
{
"state" : {
"$exists" : true
}
}
]
},
"winningPlan" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"state" : {
"$exists" : true
}
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[MinKey, MaxKey]"
],
"user" : [
"[32432424.0, 32432424.0]"
],
"stock" : [
"[MinKey, MaxKey]"
]
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : true,
"nReturned" : 0,
"executionTimeMillis" : 0,
"totalKeysExamined" : 3,
"totalDocsExamined" : 0,
"executionStages" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"state" : {
"$exists" : true
}
},
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"docsExamined" : 0,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 0,
"executionTimeMillisEstimate" : 0,
"works" : 4,
"advanced" : 0,
"needTime" : 3,
"needYield" : 0,
"saveState" : 0,
"restoreState" : 0,
"isEOF" : 1,
"invalidates" : 0,
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[MinKey, MaxKey]"
],
"user" : [
"[32432424.0, 32432424.0]"
],
"stock" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 3,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
sort情况 db.od_reservations.find({state:{$exists:true},user:{$exists:true},stock:{$exists:true}}).sort({ state:1, stock:1}).explain(“executionStats”)
进行多种测试证明 sort排序时不能使用 AC BC 等索引 属于 SORT stage
{
"queryPlanner" : {
"plannerVersion" : 1,
"namespace" : "mango_order.od_reservations",
"indexFilterSet" : false,
"parsedQuery" : {
"$and" : [
{
"state" : {
"$exists" : true
}
},
{
"stock" : {
"$exists" : true
}
},
{
"user" : {
"$exists" : true
}
}
]
},
"winningPlan" : {
"stage" : "SORT",
"sortPattern" : {
"state" : 1,
"stock" : 1
},
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"state" : {
"$exists" : true
}
},
{
"user" : {
"$exists" : true
}
},
{
"stock" : {
"$exists" : true
}
}
]
},
"inputStage" : {
"stage" : "IXSCAN",
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[MinKey, MaxKey]"
],
"user" : [
"[MinKey, MaxKey]"
],
"stock" : [
"[MinKey, MaxKey]"
]
}
}
}
}
}
},
"rejectedPlans" : [ ]
},
"executionStats" : {
"executionSuccess" : false,
"errorMessage" : "Exec error: OperationFailed: Sort operation used more than the maximum 33554432 bytes of RAM. Add an index, or specify a smaller limit., state: FAILURE",
"errorCode" : 96,
"nReturned" : 0,
"executionTimeMillis" : 313,
"totalKeysExamined" : 101140,
"totalDocsExamined" : 101140,
"executionStages" : {
"stage" : "SORT",
"nReturned" : 0,
"executionTimeMillisEstimate" : 310,
"works" : 101142,
"advanced" : 0,
"needTime" : 101141,
"needYield" : 0,
"saveState" : 790,
"restoreState" : 790,
"isEOF" : 0,
"invalidates" : 0,
"sortPattern" : {
"state" : 1,
"stock" : 1
},
"memUsage" : 33554524,
"memLimit" : 33554432,
"inputStage" : {
"stage" : "KEEP_MUTATIONS",
"nReturned" : 101140,
"executionTimeMillisEstimate" : 260,
"works" : 101141,
"advanced" : 101140,
"needTime" : 1,
"needYield" : 0,
"saveState" : 790,
"restoreState" : 790,
"isEOF" : 0,
"invalidates" : 0,
"inputStage" : {
"stage" : "SORT_KEY_GENERATOR",
"nReturned" : 0,
"executionTimeMillisEstimate" : 260,
"works" : 101141,
"advanced" : 0,
"needTime" : 1,
"needYield" : 0,
"saveState" : 790,
"restoreState" : 790,
"isEOF" : 0,
"invalidates" : 0,
"inputStage" : {
"stage" : "FETCH",
"filter" : {
"$and" : [
{
"state" : {
"$exists" : true
}
},
{
"user" : {
"$exists" : true
}
},
{
"stock" : {
"$exists" : true
}
}
]
},
"nReturned" : 101140,
"executionTimeMillisEstimate" : 200,
"works" : 101140,
"advanced" : 101140,
"needTime" : 0,
"needYield" : 0,
"saveState" : 790,
"restoreState" : 790,
"isEOF" : 0,
"invalidates" : 0,
"docsExamined" : 101140,
"alreadyHasObj" : 0,
"inputStage" : {
"stage" : "IXSCAN",
"nReturned" : 101140,
"executionTimeMillisEstimate" : 40,
"works" : 101140,
"advanced" : 101140,
"needTime" : 0,
"needYield" : 0,
"saveState" : 790,
"restoreState" : 790,
"isEOF" : 0,
"invalidates" : 0,
"keyPattern" : {
"state" : 1,
"user" : 1,
"stock" : 1
},
"indexName" : "state_1_user_1_stock_1",
"isMultiKey" : false,
"isUnique" : false,
"isSparse" : false,
"isPartial" : false,
"indexVersion" : 1,
"direction" : "forward",
"indexBounds" : {
"state" : [
"[MinKey, MaxKey]"
],
"user" : [
"[MinKey, MaxKey]"
],
"stock" : [
"[MinKey, MaxKey]"
]
},
"keysExamined" : 101140,
"dupsTested" : 0,
"dupsDropped" : 0,
"seenInvalidated" : 0
}
}
}
}
}
},
"serverInfo" : {
"host" : "mango",
"port" : 27017,
"version" : "3.2.8",
"gitVersion" : "ed70e33130c977bda0024c125b56d159573dbaf0"
},
"ok" : 1
}
