solr应用中文，庖丁分词

solr使用中文，庖丁分词
1、分析器改造 net.paoding.analysis.analyzer.PaodingTokenizer.java



extends Tokenizer 原来是 extends TokenStream



2、net.paoding.analysis.analyzer.solr.ChineseTokenizerFactory.java



package net.paoding.analysis.analyzer.solr;

import java.io.Reader;

import java.util.Map;



import net.paoding.analysis.analyzer.PaodingTokenizer;

import net.paoding.analysis.analyzer.TokenCollector;

import net.paoding.analysis.analyzer.impl.MaxWordLengthTokenCollector;

import net.paoding.analysis.analyzer.impl.MostWordsTokenCollector;

import net.paoding.analysis.knife.PaodingMaker;



import org.apache.lucene.analysis.TokenStream;

import org.apache.lucene.analysis.Tokenizer;

import org.apache.solr.analysis.BaseTokenizerFactory;



/**

* Created by IntelliJ IDEA. User: ronghao Date: 2007-11-3 Time: 14:40:59 中文切词

* 对庖丁切词的封装

*/

public class ChineseTokenizerFactory extends BaseTokenizerFactory {

/**

* 最多切分 默认模式

*/

public static final String MOST_WORDS_MODE = "most-words";

/**

* 按最大切分

*/

public static final String MAX_WORD_LENGTH_MODE = "max-word-length";

private String mode = null;



public void setMode(String mode) {

   if (mode == null || MOST_WORDS_MODE.equalsIgnoreCase(mode)

     || "default".equalsIgnoreCase(mode)) {

    this.mode = MOST_WORDS_MODE;

   } else if (MAX_WORD_LENGTH_MODE.equalsIgnoreCase(mode)) {

    this.mode = MAX_WORD_LENGTH_MODE;

   } else {

    throw new IllegalArgumentException(

      "不合法的分析器Mode                                                参数设置:"

        + mode);

   }

}



@Override

public void init(Map<String,String> args) {

   super.init(args);

   setMode(args.get("mode"));

}



public Tokenizer create(Reader input) {

   return new PaodingTokenizer(input, PaodingMaker.make(),

     createTokenCollector());

}



private TokenCollector createTokenCollector() {

   if (MOST_WORDS_MODE.equals(mode))

    return new MostWordsTokenCollector();

   if (MAX_WORD_LENGTH_MODE.equals(mode))

    return new MaxWordLengthTokenCollector();

   throw new Error("never happened");

}

}



3、下载庖丁 将paoding-analysis.jar改造成paoding-analysis-solr.jar（将1,2覆盖进去这个包）



4、定义庖丁词典 paoding-dic-home.properties



paoding.dic.home=D:/WORK/DOCSET/solr/paoding-analysis-2.0.4-beta/dic/



5、替换原有的分词类 D:\WORK\DOCSET\solr\j-solr1\solr\conf\schema.xml



<fieldtype name="text" positionIncrementGap="100">       

        <analyzer type="index">         

         <tokenizer mode="most-words"/>         

         <filter ignoreCase="true" words="stopwords.txt"/>      

         <filter generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0"/>         

         <filter mode="most-words"/>           

         <filter synonyms="synonyms.txt" ignoreCase="true" expand="true"/>        

         <filter ignoreCase="true" words="stopwords.txt"/>     

         <filter generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0"/>        

         <filter class="solr.LowerCaseFilterFactory"/>         

         <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>    

        </analyzer>

</fieldtype>