Lucene Highlighter

[zeah]

Hi All,
I was trying to implement highlighter in my search engine. I was able to implement the highlighter but the problem is it does not always work. Sometimes it highlight and sometimes it doesnt though lucene returns the file where the term was found. I can manually see the term in the document and also check the token stream. The word were there in both but still the highlighter doesnt work. Is it something to do with the size of the file?? Because thats what im thinkin...Please give me a clue to work on it.

public static void main(String[] args) throws Exception{
       // TODO code application logic here
       String q = "search word";
       IndexSearcher is = new IndexSearcher("luceneIndex");
       Analyzer analyzer = new StandardAnalyzer();
       QueryParser parser = new QueryParser("content", analyzer);
       Query query = parser.parse(q);
       Hits hits = is.search(query);

       SimpleHTMLFormatter formatter = new SimpleHTMLFormatter("<b><i>","</i></b>");
       
       for (int i=0; i<hits.length(); i++) {
           Document doc = hits.doc(i);
          
           int id = hits.id(i);
           
           String text=doc.get("content");
          
           TokenStream tokenStream = analyzer.tokenStream(null,new StringReader(text));
           CachingTokenFilter filter = new CachingTokenFilter(tokenStream);
           
           QueryScorer scorer = new QueryScorer(query);//, null,filter);
           
           Fragmenter fragmenter=new NullFragmenter();
           Highlighter highlighter=new Highlighter(formatter,scorer);
           highlighter.setTextFragmenter(fragmenter);
           String fragments = highlighter.getBestFragment(filter, text);
           filter.reset();

          
           displayTokenStream(filter);
           
               System.out.println("frags:"+fragments);
           
           //System.out.println(doc.get("content"));
               //System.out.println(text);
               
              
           
       }
       is.close();

   }
   public static void displayTokenStream(TokenStream tokenStream) throws IOException {

TermAttribute termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
TypeAttribute typeAtt = (TypeAttribute) tokenStream.getAttribute(TypeAttribute.class);

while (tokenStream.incrementToken()) {
System.out.println(termAtt.term());
//System.out.println(typeAtt.type());
}
}