29
29
import org .apache .lucene .analysis .WordlistLoader ;
30
30
import org .apache .lucene .analysis .ar .ArabicNormalizationFilter ;
31
31
import org .apache .lucene .analysis .core .DecimalDigitFilter ;
32
- import org .apache .lucene .analysis .miscellaneous .SetKeywordMarkerFilter ;
33
32
import org .apache .lucene .analysis .standard .StandardTokenizer ;
34
33
import org .apache .lucene .util .IOUtils ;
35
34
@@ -87,8 +86,6 @@ private static class DefaultSetHolder {
87
86
}
88
87
}
89
88
90
- private final CharArraySet stemExclusionSet ;
91
-
92
89
/** Builds an analyzer with the default stop words: {@link #DEFAULT_STOPWORD_FILE}. */
93
90
public PersianAnalyzer () {
94
91
this (DefaultSetHolder .DEFAULT_STOP_SET );
@@ -100,19 +97,7 @@ public PersianAnalyzer() {
100
97
* @param stopwords a stopword set
101
98
*/
102
99
public PersianAnalyzer (CharArraySet stopwords ) {
103
- this (stopwords , CharArraySet .EMPTY_SET );
104
- }
105
-
106
- /**
107
- * Builds an analyzer with the given stop word. If a none-empty stem exclusion set is provided
108
- * this analyzer will add a {@link SetKeywordMarkerFilter} before {@link PersianStemFilter}.
109
- *
110
- * @param stopwords a stopword set
111
- * @param stemExclusionSet a set of terms not to be stemmed
112
- */
113
- public PersianAnalyzer (CharArraySet stopwords , CharArraySet stemExclusionSet ) {
114
100
super (stopwords );
115
- this .stemExclusionSet = CharArraySet .unmodifiableSet (CharArraySet .copy (stemExclusionSet ));
116
101
}
117
102
118
103
/**
@@ -136,11 +121,7 @@ protected TokenStreamComponents createComponents(String fieldName) {
136
121
* the order here is important: the stopword list is normalized with the
137
122
* above!
138
123
*/
139
- result = new StopFilter (result , stopwords );
140
- if (!stemExclusionSet .isEmpty ()) {
141
- result = new SetKeywordMarkerFilter (result , stemExclusionSet );
142
- }
143
- return new TokenStreamComponents (source , new PersianStemFilter (result ));
124
+ return new TokenStreamComponents (source , new StopFilter (result , stopwords ));
144
125
}
145
126
146
127
@ Override
0 commit comments