如何将 Couchbase 中的数据与 map reduce 合并？

Question

我的应用程序将多种文档类型存储在同一个存储桶中。我知道这不是好的做法，但我对可以在服务器上创建的存储桶数量有限制，目前没有办法解决。这些文档以它们的类型为前缀，所以当我得到一个文档时，我只需要连接前缀和 id 来获取密钥，然后我就可以进行密钥查找。

我需要创建一个报告，从多种文档类型中提取信息。

我的地图是这样的：

function(doc, meta) {

  var getStep = function(stepName, exit, mapper) {
    if (meta.id.indexOf(stepName) !== -1) {
      var hotelId = parseInt(meta.id.replace(stepName + '_', ''));
      if (hotelId > 0) {
        var result = {
          hotelId: hotelId,
          exit: exit
        };
        if (mapper !== undefined) {
          mapper(result);
        }
        return result;
      }
    }
    return null;
  };

  var photos = getStep('PHOTOS', 7);
  if (photos != null) {
    emit(photos.hotelId, photos);
  }
  var pricing = getStep('PICR', 5);
  if (pricing != null) {
    emit(pricing.hotelId, pricing);
  }
  var owner = getStep('OWNER', 1);
  if (owner != null) {
    emit(owner.hotelId, owner);
  }
  var amenity = getStep('AM', 4);
  if (amenity != null) {
    emit(amenity.hotelId, amenity);
  }
  var description = getStep('HDESC', 3, function(result) {
    result.description = doc.description;
    result.hotelRoomTypeId = doc.hotelRoomTypeId;
    result.starRating = doc.starRating;
  });
  if (description != null) {
    emit(description.hotelId, description);
  }
  var contact = getStep('DC', 3, function(result) {
    result.email = doc.emailAddress;
    result.contact = doc.mainContactName;
  });
  if (contact != null) {
    emit(contact.hotelId, contact);
  }
  var location = getStep('LOC', 2, function(result) {
    result.city = doc.cityName;
    result.zip = doc.postalCode;
    result.country = doc.countryName;
    result.street = doc.stateName + ', ' + doc.streetName;
  });
  if (location != null) {
    emit(location.hotelId, location);
  }
  var property = getStep('PRP', 1, function(result) {
    result.paymentMethodId = doc.paymentMethodId
  });
  if (property != null) {
    emit(property.hotelId, property);
  }
}

它生成此输出：

"total_rows":...,"rows":[
{"id":"DC_1","key":1,"value":{"hotelId":1,"exit":3,"email":"test@example.com","contact":"Jeno"}},
{"id":"HDESC_1","key":1,"value":{"hotelId":1,"exit":3,"description":".","hotelRoomTypeId":0,"starRating":5}},
{"id":"LOC_1","key":1,"value":{"hotelId":1,"exit":2,"city":"Barcelona","zip":"1222","country":"Spain","street":"Catalonia, someplacenice"}},
{"id":"PRP_1","key":1,"value":{"hotelId":1,"exit":1}},
{"id":"PRP_2","key":2,"value":{"hotelId":2,"exit":1}},
{"id":"AM_3","key":3,"value":{"hotelId":3,"exit":4}},
{"id":"AM_4","key":4,"value":{"hotelId":4,"exit":4}},
{"id":"PHOTOS_4","key":4,"value":{"hotelId":4,"exit":7}},
{"id":"PRP_4","key":4,"value":{"hotelId":4"exit":1}},
{"id":"AM_4","key":4,"value":{"hotelId":4,"exit":4}},
{"id":"PRP_4","key":4,"value":{"hotelId":4,"exit":1}},
{"id":"PHOTOS_5","key":5,"value":{"hotelId":5,"exit":7}}
...

]

我正在尝试按新键 hotelId 对日期进行分组，并使用自定义缩减器将字段合并到一个文档中。根据错误类型，我收到不同的错误，但所有错误似乎都表明减速器可以 return 的日期有限制。如果我将 return 类型从一个对象更改为一个关联数组，它的工作方式几乎相同，我会得到一个更好的错误。

function(key, values, rereduce) { 
  if (rereduce) {
    return values;
  } else {
    var results = {}; // Object!
    for (var i = 0; i < values.length; i++) {
      var row = values[i];
      if (!results[row.hotelId]) {
        results[row.hotelId] = {
          phone: '',
          exit: 1
        };
      }
      var result = results[row.hotelId];
      for (var name in row) {
        result[name] = row[name];
      }
      if (row.exit > row.exit) {
        result.exit = row.exit;
      }
    };

    return results;
  }
}

给我 RangeError: Maximum call stack size exceeded

function(key, values, rereduce) { 
  if (rereduce) {
    return values;
  } else {
    var results = []; // Array!
    for (var i = 0; i < values.length; i++) {
      var row = values[i];
      if (!results[row.hotelId]) {
        results[row.hotelId] = {
          phone: '',
          exit: 1
        };
      }
      var result = results[row.hotelId];
      for (var name in row) {
        result[name] = row[name];
      }
      if (row.exit > row.exit) {
        result.exit = row.exit;
      }
    };

    return results;
  }
}

给我 reduction too large error

function(key, values, rereduce) { 
  if (rereduce) {
    return values;
  } else {    
    return values;
  }
}

给我 RangeError: Maximum call stack size exceeded

如果我运行：

function(key, values, rereduce) { 
  if (rereduce) {
    return values;
  } else {        
    return values.length;
  }
}

我回来了：

[ 68, 72, 65, 66, 68, 68, 70, 114 ]

JavaScript 引擎应该能够减少最大 114 大小的数组，输出数据应该更小。显然，reduce 可以 return max_kv_size_per_doc 限制多少数据，即 1Mb，也有 10 秒的执行限制，但在我的例子中是另外一回事。有没有办法通过改变算法来绕过这些限制，returning 数组或数组之类的？我可以在地图上做些什么，或者我可以在 rereduce 中使用一些技巧吗？

Answer 1

我明白了。如果我使用复合键和 group_level，它会起作用。

因此，如果我将地图更改为 return 一个数组作为酒店 ID 的键，并且我设置 group_level = 1 那么这些值将按照我最初的预期为我分组：

function(doc, meta) {

  var getStep = function(stepName, exit, mapper) {
    if (meta.id.indexOf(stepName) !== -1) {
      var hotelId = parseInt(meta.id.replace(stepName + '_', ''));
      if (hotelId > 0) {
        var result = {
          hotelId: hotelId,
          exit: exit
        };
        if (mapper !== undefined) {
          mapper(result);
        }
        return result;
      }
    }
    return null;
  };

  var photos = getStep('PHOTOS', 7);
  if (photos != null) {
    emit([photos.hotelId], photos); // array as key
  }
  var pricing = getStep('PICR', 5); // array as key
  if (pricing != null) {
    emit([pricing.hotelId], pricing);
  }
  var owner = getStep('OWNER', 1); // array as key
  if (owner != null) {
    emit([owner.hotelId], owner);
  }
  var amenity = getStep('AM', 4); // array as key
  if (amenity != null) {
    emit([amenity.hotelId], amenity);
  }
  var description = getStep('HDESC', 3, function(result) {
    result.description = doc.description;
    result.hotelRoomTypeId = doc.hotelRoomTypeId;
    result.starRating = doc.starRating;
  });
  if (description != null) {
    emit([description.hotelId], description); // array as key
  }
  var contact = getStep('DC', 3, function(result) {
    result.email = doc.emailAddress;
    result.contact = doc.mainContactName;
  });
  if (contact != null) {
    emit([contact.hotelId], contact); // array as key
  }
  var location = getStep('LOC', 2, function(result) {
    result.city = doc.cityName;
    result.zip = doc.postalCode;
    result.country = doc.countryName;
    result.street = doc.stateName + ', ' + doc.streetName;
  });
  if (location != null) {
    emit([location.hotelId], location); // array as key
  }
  var property = getStep('PRP', 1, function(result) {
    result.paymentMethodId = doc.paymentMethodId
  });
  if (property != null) {
    emit([property.hotelId], property); // array as key
  } 
}

然后我需要设置group_level=1和reduce=true。您可以在视图编辑器或查询字符串中执行此操作。

最后一位是reduce:

function(key, values, rereduce) { 
  if (rereduce) {
    return values;
  } else {           
    var result = {};
    values.forEach(function(item){
        for(var name in item){
            result[name] = item[name];
        }
    });

    return result;
  }
}

结果将按预期由 hotelId 合并 :)

如何将 Couchbase 中的数据与 map reduce 合并？

How to merge data in Couchbase with map reduce?

javascript

mapreduce

couchbase