如何提取现有码本对应的BOW描述符？

Question

我已经使用 A-KAZE 特征描述符训练了 BOW 密码本（词汇），并尝试使用 BFMatcher 和 knnMatch 将新提取的特征与密码本进行比较。

相反，我收到以下错误，

OpenCV Error: Assertion failed (_queryDescriptors.type() == trainDescType) in knnMatchImpl, file /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp, line 722 terminate called after throwing an instance of 'cv::Exception'   what():  /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp:722: error: (-215) _queryDescriptors.type() == trainDescType in function knnMatchImpl

我使用了以下示例

BoW in OpenCV using precomputed features
How to train and predict using bag of words?

我的直觉是我错误地将密码本添加到匹配器，但我找不到任何支持其他方法的文档或示例。如何将我的密码本与新示例一起使用。

MCVE

/* BOWTest.cpp*/
#include <opencv2/imgcodecs.hpp>
#include <opencv2/videoio.hpp>
#include <opencv2/highgui.hpp>
#include <opencv2/features2d.hpp>
#include <opencv2/opencv.hpp>

#include <iostream>
#include <string>
#include <stdio.h>
#include <dirent.h>

using namespace cv;
using namespace std;

std::string outputFile = "test_codebook.png";
std::string trainingDir = "dataset/";
std::string outputPrefix = "output/res_custom_";

void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze);
void test(Mat codebook, Ptr<Feature2D> akaze);

int main(int ac, char** av) {

    Ptr<Feature2D> feature = AKAZE::create();
    Mat codebook;
    int codebook_n = 100;
    //train(codebook, codebook_n, feature);
    test(codebook, feature);
}

//I included the train method to show how the codebook is trained, but it is not actually called in this example
void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze){
    //defining terms for bowkmeans trainer
    TermCriteria tc(TermCriteria::MAX_ITER + TermCriteria::EPS, 10, 0.001);
    int retries = 1;
    int flags = KMEANS_PP_CENTERS;
    BOWKMeansTrainer bowTrainer(codebook_n, tc, retries, flags);

    int i = 0;
    unsigned long numPoints = 0;
    DIR           *d;
    struct dirent *dir;
    d = opendir(trainingDir.c_str());
    if (d)  {
        while ((dir = readdir(d)) != NULL){

            try {
            Mat img;
            std::string imgName = trainingDir + dir->d_name;
            i = i + 1;

            printf("%d, %lu: %s ...", i,numPoints, imgName.c_str());
            img = imread(imgName, CV_LOAD_IMAGE_COLOR);
            if(img.empty()){ //not image
                printf("bad.\n");
                continue;
            }

            printf("loaded.\n");
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, Mat(), keypoints, features);
            features.convertTo(features, CV_32F);
            bowTrainer.add(features);

            Mat res;
            drawKeypoints(img, keypoints, res);
            std::string output_img =  outputPrefix + dir->d_name;
            imwrite(output_img, res);

            numPoints += features.rows;

            }catch(int e){
                cout << "An exception occurred. Nr. " << e << '\n';
            }
        }

        printf("Read images!");
        closedir(d);

        codebook = bowTrainer.cluster();
        imwrite(outputFile, codebook);
    }
}

void test(Mat codebook, Ptr<Feature2D> akaze){
    codebook = imread(outputFile);
    int codebook_n = codebook.rows;

    BFMatcher matcher(NORM_L2);
    matcher.add(std::vector<cv::Mat>(1, codebook));

    Mat res(Size(codebook_n * 10, 3*10), CV_8UC3, Scalar(0));
    vector<int> res_idx(codebook_n, 0);

    try {
        Mat img;
        String imgName = trainingDir + "dog1.jpeg";
        img = imread(imgName, CV_LOAD_IMAGE_COLOR);
        if(img.empty()){ //not image
            printf("bad.\n");
        }else{
            printf("loaded.\n");
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, noArray(), keypoints, features);
            features.convertTo(features, CV_32F);

            vector< vector< DMatch > > nn_matches;
            matcher.knnMatch(features, nn_matches, 1);

            printf("%d matched keypoints", nn_matches.size());
        }

    }catch(int e){
        cout << "An exception occurred. Nr. " << e << '\n';
    }
}

test_codebook.png

dog1.jpeg

输出

loaded.
OpenCV Error: Assertion failed (_queryDescriptors.type() == trainDescType) in knnMatchImpl, file /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp, line 722
terminate called after throwing an instance of 'cv::Exception'
  what():  /home/cecilia/opencv-3.0.0/modules/features2d/src/matchers.cpp:722: error: (-215) _queryDescriptors.type() == trainDescType in function knnMatchImpl

Answer 1

您不应该将 codebook 另存为图像。 imwrite最终会缩放和转换码本的值。 imread 使用默认参数会将其加载为 3 通道图像 CV_8UC3。要存储不是严格图像的矩阵，您应该使用 FileStorage.

保存：

FileStorage fs(outputFile, FileStorage::WRITE);
// Store codebook
fs << "codebook" << codebook;

加载：

FileStorage fs(outputFile, FileStorage::READ);
fs["codebook"] >> codebook;

您应该使用 BOWImgDescriptorExtractor 从您的特征开始计算 BoW 图像描述符，在这种情况下为 AKAZE：

Ptr<DescriptorMatcher> matcher = BFMatcher::create("BruteForce");
BOWImgDescriptorExtractor bow(akaze, matcher);
bow.setVocabulary(codebook);

// Mat img = ...

// AKAZE features
Mat features;
vector<KeyPoint> keypoints;
akaze->detectAndCompute(img, noArray(), keypoints, features);
features.convertTo(features, CV_32F);

// BoW descriptor
Mat bowFeatures;
vector<vector<int>> pointsIdxsOfCluster;
bow.compute(features, bowFeatures, &pointsIdxsOfCluster);

您可以使用内置 glob 从文件夹中读取图像，避免 dirent。

vector<String> fileNames;
glob(trainingDir, fileNames);

for (int i=0; i<fileNames.size(); ++i)
{
    Mat img = imread(fileNames[i]);
    ...

您可以将 iostream 与 cout 一起使用，而不是 printf。

代码如下所示：

#include <opencv2/opencv.hpp>
#include <iostream>
using namespace cv;
using namespace std;

std::string outputFile = "test_codebook.yml";
std::string trainingDir = "path_to_train_folder/";
std::string outputPrefix = "path_to_output_folder/";

void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze);
void test(Mat codebook, Ptr<Feature2D> akaze);

int main(int ac, char** av) {

    Ptr<Feature2D> feature = AKAZE::create();
    Mat codebook;
    int codebook_n = 100;

    train(codebook, codebook_n, feature);
    test(codebook, feature);
}

//I included the train method to show how the codebook is trained, but it is not actually called in this example
void train(Mat codebook, int codebook_n, Ptr<Feature2D> akaze){
    //defining terms for bowkmeans trainer
    TermCriteria tc(TermCriteria::MAX_ITER + TermCriteria::EPS, 10, 0.001);
    int retries = 1;
    int flags = KMEANS_PP_CENTERS;
    BOWKMeansTrainer bowTrainer(codebook_n, tc, retries, flags);

    int i = 0;
    unsigned long numPoints = 0;

    vector<String> fileNames;
    glob(trainingDir, fileNames);

    for (int i=0; i<fileNames.size(); ++i)
    {
        try {
            Mat img;
            std::string imgName = fileNames[i];
            std::string filename = imgName.substr(trainingDir.length());

            cout << i << ", " << numPoints << " : " << imgName;
            img = imread(imgName, CV_LOAD_IMAGE_COLOR);
            if (img.empty()){ //not image
                cout << " bad" << endl;
                continue;
            }

            cout << " loaded" << endl;
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, Mat(), keypoints, features);
            features.convertTo(features, CV_32F);
            bowTrainer.add(features);

            Mat res;
            drawKeypoints(img, keypoints, res);
            std::string output_img = outputPrefix + filename;
            imwrite(output_img, res);

            numPoints += features.rows;

        }
        catch (int e){
            cout << "An exception occurred. Nr. " << e << '\n';
        }
    }

    cout << "Read images!" << endl;

    codebook = bowTrainer.cluster();

    {
        FileStorage fs(outputFile, FileStorage::WRITE);

        // Store codebook
        fs << "codebook" << codebook;

        // You can also store additional info, like the list of images

        //// Store train images filenames
        //fs << "train" << "[";
        //for (int i = 0; i < fileNames.size(); ++i)
        //{
        //  fs << fileNames[i];
        //}
        //fs << "]";
    }
}

void test(Mat codebook, Ptr<Feature2D> akaze)
{
    vector<String> trainFileNames;
    {
        FileStorage fs(outputFile, FileStorage::READ);
        fs["codebook"] >> codebook;

        /*FileNode trainingImages = fs["train"];
        FileNodeIterator it = trainingImages.begin(), it_end = trainingImages.end();
        int idx = 0;
        for (; it != it_end; ++it, idx++)
        {
            trainFileNames.push_back(*it);
        }*/
    }

    int codebook_n = codebook.rows;

    Ptr<DescriptorMatcher> matcher = BFMatcher::create("BruteForce");
    BOWImgDescriptorExtractor bow(akaze, matcher);
    bow.setVocabulary(codebook);

    try {
        Mat img;
        String imgName = "path_to_test_image";
        img = imread(imgName, CV_LOAD_IMAGE_COLOR);
        if (img.empty()){ //not image
            cout << "bad" << endl;
        }
        else{
            cout << "loaded" << endl;
            resize(img, img, Size(200, 200));

            Mat features;
            vector<KeyPoint> keypoints;
            akaze->detectAndCompute(img, noArray(), keypoints, features);
            features.convertTo(features, CV_32F);

            Mat bowFeatures;
            vector<vector<int>> pointsIdxsOfCluster;
            bow.compute(features, bowFeatures, &pointsIdxsOfCluster);

            // bowFeatures is the descriptor you're looking for
            // pointsIdxsOfCluster contains the indices of keypoints that belong to the cluster.
        }
    }
    catch (int e){
        cout << "An exception occurred. Nr. " << e << endl;
    }
}

如何提取现有码本对应的BOW描述符？

How to extract BOW descriptors corresponding to existing codebook?

c++

opencv3.0

MCVE

test_codebook.png

dog1.jpeg

输出