From 6f1ecbbec09e73acacbfb8b0f498d8a3fa95573e Mon Sep 17 00:00:00 2001
From: kysin8 <kysin8@gmail.com>
Date: Tue, 7 Jan 2014 18:23:09 +0800
Subject: [PATCH 1/4] Changes for Chinese handling

---
 solr-home/ac/conf/schema.xml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/solr-home/ac/conf/schema.xml b/solr-home/ac/conf/schema.xml
index 36017d6..4dd6202 100755
--- a/solr-home/ac/conf/schema.xml
+++ b/solr-home/ac/conf/schema.xml
@@ -25,7 +25,7 @@
 		<fieldType name="text_suggest" class="solr.TextField" positionIncrementGap="100">
 			<analyzer type="index">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
-				<tokenizer class="solr.StandardTokenizerFactory"/>
+				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
 				<filter class="solr.WordDelimiterFilterFactory" 
 					    generateWordParts="1" 
 					    generateNumberParts="1"
@@ -37,11 +37,11 @@
 					    preserveOriginal="1"
 				/>
 				<filter class="solr.LowerCaseFilterFactory"/>
-				<filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement=" " replace="all"/>
+				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement=" " replace="all"/-->
 			</analyzer>
 			<analyzer type="query">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
-				<tokenizer class="solr.StandardTokenizerFactory"/>
+				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
 				<filter class="solr.WordDelimiterFilterFactory" 
 					    generateWordParts="0" 
 					    generateNumberParts="0"
@@ -52,7 +52,7 @@
 					    splitOnNumerics="0"
 				/>
 				<filter class="solr.LowerCaseFilterFactory"/>
-				<filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement=" " replace="all"/>
+				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement=" " replace="all"/-->
 			</analyzer>
 		</fieldType>
 		<!-- text_phonetic_do : Matches a phonetic version of the suggest text, for a fuzzy effect 
@@ -74,14 +74,14 @@
 				<filter class="solr.LowerCaseFilterFactory"/>
 				<filter class="solr.PatternReplaceFilterFactory" pattern="([\.,;:-_])" replacement=" " replace="all"/>
 				<filter class="solr.EdgeNGramFilterFactory" maxGramSize="30" minGramSize="1"/>
-				<filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/>
+				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 			</analyzer>
 			<analyzer type="query">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
 				<tokenizer class="solr.KeywordTokenizerFactory"/>
 				<filter class="solr.LowerCaseFilterFactory"/>
 				<filter class="solr.PatternReplaceFilterFactory" pattern="([\.,;:-_])" replacement=" " replace="all"/>
-				<filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/>
+				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 				<filter class="solr.PatternReplaceFilterFactory" pattern="^(.{30})(.*)?" replacement="$1" replace="all"/>
 			</analyzer>
 		</fieldType>
@@ -93,18 +93,18 @@
 		<fieldType name="autocomplete_ngram" class="solr.TextField">
 			<analyzer type="index">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
-				<tokenizer class="solr.StandardTokenizerFactory"/>
-				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
+				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
+				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="1" splitOnCaseChange="1"/>
 				<filter class="solr.LowerCaseFilterFactory"/>
 				<filter class="solr.EdgeNGramFilterFactory" maxGramSize="20" minGramSize="1"/>
-				<filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/>
+				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 			</analyzer>
 			<analyzer type="query">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
-				<tokenizer class="solr.StandardTokenizerFactory"/>
-				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="0"/>
+				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
+				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="0" catenateNumbers="0" catenateAll="1" splitOnCaseChange="0"/>
 				<filter class="solr.LowerCaseFilterFactory"/>
-				<filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/>
+				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 				<filter class="solr.PatternReplaceFilterFactory" pattern="^(.{20})(.*)?" replacement="$1" replace="all"/>
 			</analyzer>
 		</fieldType>

From fbb15a28c56b7c702a7580da6b399dea4f630810 Mon Sep 17 00:00:00 2001
From: kysin8 <kysin8@gmail.com>
Date: Fri, 17 Jan 2014 11:55:09 +0800
Subject: [PATCH 2/4] Change for phrase ordered match

Add ShingleFilter to ensure ordered phrase matching
---
 solr-home/ac/conf/schema.xml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/solr-home/ac/conf/schema.xml b/solr-home/ac/conf/schema.xml
index 4dd6202..7e4c92b 100755
--- a/solr-home/ac/conf/schema.xml
+++ b/solr-home/ac/conf/schema.xml
@@ -94,15 +94,16 @@
 			<analyzer type="index">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
 				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
+				<filter class="solr.SynonymFilterFactory" synonyms="syn.txt" ignoreCase="true" expand="true"/>
 				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="1" splitOnCaseChange="1"/>
 				<filter class="solr.LowerCaseFilterFactory"/>
+				<filter class="solr.ShingleFilterFactory" maxShingleSize="3" outputUnigrams="false"/>
 				<filter class="solr.EdgeNGramFilterFactory" maxGramSize="20" minGramSize="1"/>
 				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 			</analyzer>
 			<analyzer type="query">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
-				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
-				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="0" catenateNumbers="0" catenateAll="1" splitOnCaseChange="0"/>
+				<tokenizer class="solr.KeywordTokenizerFactory"/>				
 				<filter class="solr.LowerCaseFilterFactory"/>
 				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 				<filter class="solr.PatternReplaceFilterFactory" pattern="^(.{20})(.*)?" replacement="$1" replace="all"/>

From 7c30a70086fe59f4d679788c5e99d0504c3012d5 Mon Sep 17 00:00:00 2001
From: kysin8 <kysin8@gmail.com>
Date: Fri, 17 Jan 2014 12:28:32 +0800
Subject: [PATCH 3/4] Change maxShingleSize

Change maxShingleSize to 10
---
 solr-home/ac/conf/schema.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/solr-home/ac/conf/schema.xml b/solr-home/ac/conf/schema.xml
index 7e4c92b..154aed4 100755
--- a/solr-home/ac/conf/schema.xml
+++ b/solr-home/ac/conf/schema.xml
@@ -97,7 +97,7 @@
 				<filter class="solr.SynonymFilterFactory" synonyms="syn.txt" ignoreCase="true" expand="true"/>
 				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="1" splitOnCaseChange="1"/>
 				<filter class="solr.LowerCaseFilterFactory"/>
-				<filter class="solr.ShingleFilterFactory" maxShingleSize="3" outputUnigrams="false"/>
+				<filter class="solr.ShingleFilterFactory" maxShingleSize="10" outputUnigrams="false"/>
 				<filter class="solr.EdgeNGramFilterFactory" maxGramSize="20" minGramSize="1"/>
 				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 			</analyzer>

From aa0c26188231938808ee8e59335a32ed8792d0a3 Mon Sep 17 00:00:00 2001
From: kysin8 <kysin8@gmail.com>
Date: Mon, 20 Jan 2014 16:09:19 +0800
Subject: [PATCH 4/4] Change ShingleFilter to output unigram

If unigram is not output, the last term will not be processed by the
ngramFilter.
---
 solr-home/ac/conf/schema.xml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/solr-home/ac/conf/schema.xml b/solr-home/ac/conf/schema.xml
index 154aed4..b34ee9d 100755
--- a/solr-home/ac/conf/schema.xml
+++ b/solr-home/ac/conf/schema.xml
@@ -90,20 +90,20 @@
 		     This means that the field "A brown fox" will be matched by query "bro".
 		     We use this to get partial matches, but these whould be boosted lower than exact and left-anchored
 		-->
-		<fieldType name="autocomplete_ngram" class="solr.TextField">
+		<fieldType name="autocomplete_ngram" class="solr.TextField" autoGeneratePhraseQueries="true">
 			<analyzer type="index">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
 				<tokenizer class="solr.WhitespaceTokenizerFactory"/>
 				<filter class="solr.SynonymFilterFactory" synonyms="syn.txt" ignoreCase="true" expand="true"/>
 				<filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="1" splitOnCaseChange="1"/>
 				<filter class="solr.LowerCaseFilterFactory"/>
-				<filter class="solr.ShingleFilterFactory" maxShingleSize="10" outputUnigrams="false"/>
+				<filter class="solr.ShingleFilterFactory" maxShingleSize="10" outputUnigrams="true"/>
 				<filter class="solr.EdgeNGramFilterFactory" maxGramSize="20" minGramSize="1"/>
 				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 			</analyzer>
 			<analyzer type="query">
 				<charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
-				<tokenizer class="solr.KeywordTokenizerFactory"/>				
+				<tokenizer class="solr.WhitespaceTokenizerFactory"/>				
 				<filter class="solr.LowerCaseFilterFactory"/>
 				<!--filter class="solr.PatternReplaceFilterFactory" pattern="([^\w\d\*æøåÆØÅ ])" replacement="" replace="all"/-->
 				<filter class="solr.PatternReplaceFilterFactory" pattern="^(.{20})(.*)?" replacement="$1" replace="all"/>