使用 Boost.Spirit.Lex 和流迭代器

我想使用 Boost.Spirit.Lex 来 lex 二进制文件；为此，我编写了以下程序（这是摘录）：

#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/support_multi_pass.hpp>
#include <boost/bind.hpp>
#include <boost/ref.hpp>
#include <fstream>
#include <iterator>
#include <string>

namespace spirit = boost::spirit;
namespace lex = spirit::lex;

#define X 1
#define Y 2
#define Z 3

template<typename L>
class word_count_tokens : public lex::lexer<L>
{
    public:
        word_count_tokens () {
            this->self.add
                ("[^ \t\n]+", X)
                ("\n", Y)
                (".", Z);
        }
};

class counter
{
    public:
        typedef bool result_type;

        template<typename T>
        bool operator () (const T &t, size_t &c, size_t &w, size_t &l) const {
            switch (t.id ()) {
               case X:
                   ++w; c += t.value ().size ();
                    break;
               case Y:
                   ++l; ++c;
                    break;
                case Z:
                    ++c;
                    break;
            }

            return true;
        }
};

int main (int argc, char **argv)
{
    std::ifstream ifs (argv[1], std::ios::in | std::ios::binary);
    auto first = spirit::make_default_multi_pass (std::istream_iterator<char> (ifs));
    auto last = spirit::make_default_multi_pass (std::istream_iterator<char> ());
    size_t w, c, l;
    word_count_tokens<lex::lexertl::lexer<>> word_count_functor;

    w = c = l = 0;

    bool r = lex::tokenize (first, last, word_count_functor, boost::bind (counter (), _1, boost::ref (c), boost::ref (w), boost::ref (l)));

    ifs.close ();

    if (r) {
        std::cout << l << ", " << w << ", " << c << std::endl;
    }

    return 0;
}

构建返回以下错误：

lexer.hpp:390:46: error: non-const lvalue reference to type 'const char *' cannot bind to a value of unrelated type

现在，错误是由于定义造成的concrete lexer, lex::lexer<>;事实上它的第一个参数默认为const char *。如果我使用，我也会得到同样的错误spirit::istream_iterator or spirit::make_default_multi_pass (.....).
但是如果我指定正确的模板参数lex::lexer<>我收到很多错误！

解决方案？

Update

我已经把所有源文件都放了；这是 word_counter 网站的示例。

好的，既然问题改变了，这里有一个新的答案，用完整的代码示例解决了一些问题。

1. 首先，您需要使用自定义令牌类型。 IE。

   word_count_tokens<lex::lexertl::lexer<lex::lexertl::token<boost::spirit::istream_iterator>>> word_count_functor;
   // instead of:
   // word_count_tokens<lex::lexertl::lexer<>> word_count_functor;
   

   显然，习惯上是 typedeflex::lexertl::token<boost::spirit::istream_iterator>

2. 你需要使用min_token_id而不是令牌 ID 1、2、3。另外，将其设为枚举以便于维护：

   enum token_ids {
       X = lex::min_token_id + 1,
       Y,
       Z,
   };
   

3. 你不能再仅仅使用.size()在默认令牌上value()因为迭代器范围不再是 RandomAccessRange。相反，雇用boost::distance()这是专门用于iterator_range:

           ++w; c += boost::distance(t.value()); // t.value ().size ();
   

结合这些修复：住在科里鲁 http://coliru.stacked-crooked.com/a/e2228051e17f6fbc

#include <boost/spirit/include/lex_lexertl.hpp>
#include <boost/spirit/include/support_istream_iterator.hpp>
#include <boost/bind.hpp>
#include <fstream>

namespace spirit = boost::spirit;
namespace lex    = spirit::lex;

enum token_ids {
    X = lex::min_token_id + 1,
    Y,
    Z,
};

template<typename L>
class word_count_tokens : public lex::lexer<L>
{
    public:
        word_count_tokens () {
            this->self.add
                ("[^ \t\n]+", X)
                ("\n"       , Y)
                ("."        , Z);
        }
};

struct counter
{
    typedef bool result_type;

    template<typename T>
    bool operator () (const T &t, size_t &c, size_t &w, size_t &l) const {
        switch (t.id ()) {
            case X:
                ++w; c += boost::distance(t.value()); // t.value ().size ();
                break;
            case Y:
                ++l; ++c;
                break;
            case Z:
                ++c;
                break;
        }

        return true;
    }
};

int main (int argc, char **argv)
{
    std::ifstream ifs (argv[1], std::ios::in | std::ios::binary);
    ifs >> std::noskipws;
    boost::spirit::istream_iterator first(ifs), last;
    word_count_tokens<lex::lexertl::lexer<lex::lexertl::token<boost::spirit::istream_iterator>>> word_count_functor;

    size_t w = 0, c = 0, l = 0;
    bool r = lex::tokenize (first, last, word_count_functor, 
            boost::bind (counter (), _1, boost::ref (c), boost::ref (w), boost::ref (l)));

    ifs.close ();

    if (r) {
        std::cout << l << ", " << w << ", " << c << std::endl;
    }
}

当自身运行时，打印

65, 183, 1665