MkDocs全文搜索中的中文支持

在前一篇MkDocs全文搜索中使用模糊匹配中已经实现了模糊匹配，这次通过lunr-languages来实现lunr对中文搜索的支持

###MkDocs版本 v0.16.3、lunr.js版本2.0.3、lunr-languages版本1.0.0

一 、下载lunr.stemmer.support.js和tinyseg.js（tinyseg.js原是用于lunr.ja.js，因其原理不清楚，此处照搬）并放到lunr.min.js同一目录下

cd /usr/lib/python2.7/site-packages/mkdocs/assets/search/mkdocs/js
wget https://raw.githubusercontent.com/MihaiValentin/lunr-languages/master/lunr.stemmer.support.js
wget https://raw.githubusercontent.com/MihaiValentin/lunr-languages/master/tinyseg.js

二 、在此目录下继续增加lunr.cn.js（根据lunr.ja.js做的修改，lunr.cn.wordCharacters不知其用途，沿用了ja的值未做修改；lunr.cn.stopWordFilter根据Chinese stopwords提供的数据做了修改）文件，内容如下：

/*!
 * Lunr languages, `Chinese` language
 * https://github.com/MihaiValentin/lunr-languages
 *
 * Copyright 2017, Xiang Li
 * http://www.mozilla.org/MPL/
 */
/*!
 * based on
 * Snowball JavaScript Library v0.3
 * http://code.google.com/p/urim/
 * http://snowball.tartarus.org/
 *
 * Copyright 2010, Oleg Mazko
 * http://www.mozilla.org/MPL/
 */

/**
 * export the module via AMD, CommonJS or as a browser global
 * Export code from https://github.com/umdjs/umd/blob/master/returnExports.js
 */
;
(function(root, factory) {
    if (typeof define === 'function' && define.amd) {
        // AMD. Register as an anonymous module.
        define(factory)
    } else if (typeof exports === 'object') {
        /**
         * Node. Does not work with strict CommonJS, but
         * only CommonJS-like environments that support module.exports,
         * like Node.
         */
        module.exports = factory()
    } else {
        // Browser globals (root is window)
        factory()(root.lunr);
    }
}(this, function() {
    /**
     * Just return a value to define the module export.
     * This example returns an object, but the module
     * can return a function as the exported value.
     */
    return function(lunr) {
        /* throw error if lunr is not yet included */
        if ('undefined' === typeof lunr) {
            throw new Error('Lunr is not present. Please include / require Lunr before this script.');
        }

        /* throw error if lunr stemmer support is not yet included */
        if ('undefined' === typeof lunr.stemmerSupport) {
            throw new Error('Lunr stemmer support is not present. Please include / require Lunr stemmer support before this script.');
        }

        /*
        Chinese tokenization is trickier, since it does not
        take into account spaces.
        Since the tokenization function is represented different
        internally for each of the Lunr versions, this had to be done
        in order to try to try to pick the best way of doing this based
        on the Lunr version
         */
        var isLunr2 = lunr.version[0] == "2";

        /* register specific locale function */
        lunr.cn = function() {
            this.pipeline.reset();
            this.pipeline.add(
                lunr.cn.stopWordFilter,
                lunr.cn.stemmer
            );

            // change the tokenizer for chinese one
            if (isLunr2) { // for lunr version 2.0.0
                this.tokenizer = lunr.cn.tokenizer;
            } else {
                if (lunr.tokenizer) { // for lunr version 0.6.0
                    lunr.tokenizer = lunr.cn.tokenizer;
                }
                if (this.tokenizerFn) { // for lunr version 0.7.0 -> 1.0.0
                    this.tokenizerFn = lunr.cn.tokenizer;
                }
            }
        };
        var segmenter = new lunr.TinySegmenter();

        lunr.cn.tokenizer = function (obj) {
            if (!arguments.length || obj == null || obj == undefined) return []
            if (Array.isArray(obj)) return obj.map(function (t) { return isLunr2 ? new lunr.Token(t.toLowerCase()) : t.toLowerCase() })

            var str = obj.toString().toLowerCase().replace(/^\s+/, '')

            for (var i = str.length - 1; i >= 0; i--) {
                if (/\S/.test(str.charAt(i))) {
                    str = str.substring(0, i + 1)
                    break
                }
            }

            var segs = segmenter.segment(str); 
            return segs.filter(function (token) {
                    return !!token
                })
                .map(function (token) {
                    return isLunr2 ? new lunr.Token(token) : token
                })
        }

        /* lunr stemmer function */
        lunr.cn.stemmer = (function() {

            /* TODO chinese stemmer  */
            return function(word) {
                return word;
            }
        })();

        lunr.Pipeline.registerFunction(lunr.cn.stemmer, 'stemmer-cn');
        lunr.cn.wordCharacters = "一二三四五六七八九十百千万億兆一-龠々〆ヵヶぁ-んァ-ヴーｱ-ﾝﾞa-zA-Zａ-ｚＡ-Ｚ0-9０-９";

        /* stop word filter function */
        lunr.cn.stopWordFilter = function(token) {
            if (lunr.cn.stopWordFilter.stopWords.indexOf(isLunr2 ? token.toString() : token) === -1) {
                return token;
            }
        };

        // stopword for chinese is from http://www.ranks.nl/stopwords/chinese-stopwords
        lunr.cn.stopWordFilter = lunr.generateStopWordFilter(
            '的 一 不 在 人 有 是 为 以 于 上 他 而 后 之 来 及 了 因 下 可 到 由 这 与 也 此 但 并 个 其 已 无 小 我 们 起 最 再 今 去 好 只 又 或 很 亦 某 把 那 你 乃 它 吧 被 比 别 趁 当 从 到 得 打 凡 儿 尔 该 各 给 跟 和 何 还 即 几 既 看 据 距 靠 啦 了 另 么 每 们 嘛 拿 哪 那 您 凭 且 却 让 仍 啥 如 若 使 谁 虽 随 同 所 她 哇 嗡 往 哪 些 向 沿 哟 用 于 咱 则 怎 曾 至 致 着 诸 自'.split(' '));

        lunr.Pipeline.registerFunction(lunr.cn.stopWordFilter, 'stopWordFilter-cn');
    };
}))

三 、修改search.js文件

require([
    base_url + '/mkdocs/js/mustache.min.js',
    base_url + '/mkdocs/js/lunr.min.js',
    base_url + '/mkdocs/js/lunr.stemmer.support.js',
    base_url + '/mkdocs/js/tinyseg.js',
    base_url + '/mkdocs/js/lunr.cn.js',
    'text!search-results-template.mustache',
    'text!../search_index.json',
], function (Mustache, lunr,stemmerSupport, TinySegmenter,  ja, results_template,    data) {
   "use strict";


    stemmerSupport(lunr); // adds lunr.stemmerSupport
    TinySegmenter(lunr);
    ja(lunr); // adds lunr.de key


    function getSearchTerm()
    {
        var sPageURL = window.location.search.substring(1);
        var sURLVariables = sPageURL.split('&');
        for (var i = 0; i < sURLVariables.length; i++)
        {
            var sParameterName = sURLVariables[i].split('=');
            if (sParameterName[0] == 'q')
            {
                return decodeURIComponent(sParameterName[1].replace(/\+/g, '%20'));
            }
        }
    }

    var documents = {};
    var doc;
    var index = lunr(function () {
        this.use(lunr.cn);
        this.field('title', {BOOST: 10});
        this.field('text');
        this.ref('location');

        data = JSON.parse(data);
        //var documents = {};

        for (var i=0; i < data.docs.length; i++){
            doc = data.docs[i];
            doc.location = base_url + doc.location;
            this.add(doc);
            documents[doc.location] = doc;
        }   

    });

    /*data = JSON.parse(data);
    var documents = {};

    for (var i=0; i < data.docs.length; i++){
        var doc = data.docs[i];
        doc.location = base_url + doc.location;
        index.add(doc);
        documents[doc.location] = doc;
    }*/

    var search = function(){

        var query = document.getElementById('mkdocs-search-query').value;
        var search_results = document.getElementById("mkdocs-search-results");
        while (search_results.firstChild) {
            search_results.removeChild(search_results.firstChild);
        }

        if(query === ''){
            return;
        }

        var results = index.search(query);

        if (results.length > 0){
            for (var i=0; i < results.length; i++){
                var result = results[i];
                doc = documents[result.ref];
                doc.base_url = base_url;
                doc.summary = doc.text.substring(0, 200);
                var html = Mustache.to_html(results_template, doc);
                search_results.insertAdjacentHTML('beforeend', html);
            }
        } else {
            search_results.insertAdjacentHTML('beforeend', "<p>No results found</p>");
        }

        if(jQuery){
            /*
             * We currently only automatically hide bootstrap models. This
             * requires jQuery to work.
             */
            jQuery('#mkdocs_search_modal a').click(function(){
                jQuery('#mkdocs_search_modal').modal('hide');
            });
        }

    };

    var search_input = document.getElementById('mkdocs-search-query');

    var term = getSearchTerm();
    if (term){
        search_input.value = term;
        search();
    }

    search_input.addEventListener("keyup", search);

});

大功告成，因为是根据lunr.ja.js做的修改，部分数据未做修改，可能会在搜索时出现未知的问题，目前测试运行良好