From 831bd84d647209389c8fc403e1e4c06a1e7c75fb Mon Sep 17 00:00:00 2001 From: "huangchunlin@liepiaowang.com" Date: Fri, 1 Sep 2017 16:36:26 +0800 Subject: [PATCH 1/3] =?UTF-8?q?Add=201.9.5=E7=89=88=E6=9C=AC=EF=BC=8C?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E7=83=AD=E6=9B=B4=E6=96=B0=E6=97=B6=EF=BC=8C?= =?UTF-8?q?=E7=94=A8=E6=88=B7=E5=8F=AF=E9=85=8D=E7=BD=AE=E6=98=AF=E5=90=A6?= =?UTF-8?q?=E6=8A=8A=E4=BB=8E=E8=BF=9C=E7=A8=8B=E5=AD=97=E5=85=B8=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=9A=84=E8=AF=8D=E5=AD=98=E5=85=A5=E5=88=B0=E8=87=AA?= =?UTF-8?q?=E5=B7=B1=E7=9A=84=E5=AD=97=E5=85=B8=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/IKAnalyzer.cfg.xml | 8 ++- config/remote.dic | 1 + .../org/wltea/analyzer/dic/Dictionary.java | 65 +++++++++++++------ 3 files changed, 53 insertions(+), 21 deletions(-) create mode 100644 config/remote.dic diff --git a/config/IKAnalyzer.cfg.xml b/config/IKAnalyzer.cfg.xml index 9a935c19..b8d5359f 100644 --- a/config/IKAnalyzer.cfg.xml +++ b/config/IKAnalyzer.cfg.xml @@ -3,11 +3,15 @@ IK Analyzer 扩展配置 - custom/mydict.dic;custom/single_word_low_freq.dic - + custom/mydict.dic;custom/single_word_low_freq.dic;remote.dic + custom/ext_stopword.dic + + false diff --git a/config/remote.dic b/config/remote.dic new file mode 100644 index 00000000..0049df16 --- /dev/null +++ b/config/remote.dic @@ -0,0 +1 @@ +//这是远程扩展字典的备份 \ No newline at end of file diff --git a/src/main/java/org/wltea/analyzer/dic/Dictionary.java b/src/main/java/org/wltea/analyzer/dic/Dictionary.java index d36c8d13..3f99ec50 100644 --- a/src/main/java/org/wltea/analyzer/dic/Dictionary.java +++ b/src/main/java/org/wltea/analyzer/dic/Dictionary.java @@ -25,13 +25,7 @@ */ package org.wltea.analyzer.dic; -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; +import java.io.*; import java.nio.file.Path; import java.util.*; import java.util.concurrent.Executors; @@ -86,15 +80,17 @@ public class Dictionary { public static final String PATH_DIC_SUFFIX = "suffix.dic"; public static final String PATH_DIC_PREP = "preposition.dic"; public static final String PATH_DIC_STOP = "stopword.dic"; + public static final String PATH_DIC_REMOTE = "remote.dic"; private final static String FILE_NAME = "IKAnalyzer.cfg.xml"; private final static String EXT_DICT = "ext_dict"; private final static String REMOTE_EXT_DICT = "remote_ext_dict"; private final static String EXT_STOP = "ext_stopwords"; private final static String REMOTE_EXT_STOP = "remote_ext_stopwords"; + private final static String REBASE_REMOTE_EXT_DICT = "rebase_remote_ext_dict"; - private Path conf_dir; - private Properties props; + private static Path conf_dir; + private static Properties props; private Dictionary(Configuration cfg) { this.configuration = cfg; @@ -128,7 +124,7 @@ private Dictionary(Configuration cfg) { } } - public String getProperty(String key){ + public static String getProperty(String key){ if(props!=null){ return props.getProperty(key); } @@ -137,7 +133,7 @@ public String getProperty(String key){ /** * 词典初始化 由于IK Analyzer的词典采用Dictionary类的静态方法进行词典初始化 * 只有当Dictionary类被实际调用时，才会开始载入词典， 这将延长首次分词操作的时间 该方法提供了一个在应用加载阶段就初始化字典的手段 - * + * * @return Dictionary */ public static synchronized Dictionary initial(Configuration cfg) { @@ -237,14 +233,14 @@ public List getRemoteExtStopWordDictionarys() { return remoteExtStopWordDictFiles; } - public String getDictRoot() { + public static String getDictRoot() { return conf_dir.toAbsolutePath().toString(); } /** * 获取词典单子实例 - * + * * @return Dictionary 单例对象 */ public static Dictionary getSingleton() { @@ -257,7 +253,7 @@ public static Dictionary getSingleton() { /** * 批量加载新词条 - * + * * @param words * Collection词条列表 */ @@ -288,7 +284,7 @@ public void disableWords(Collection words) { /** * 检索匹配主词典 - * + * * @return Hit 匹配结果描述 */ public Hit matchInMainDict(char[] charArray) { @@ -297,7 +293,7 @@ public Hit matchInMainDict(char[] charArray) { /** * 检索匹配主词典 - * + * * @return Hit 匹配结果描述 */ public Hit matchInMainDict(char[] charArray, int begin, int length) { @@ -306,7 +302,7 @@ public Hit matchInMainDict(char[] charArray, int begin, int length) { /** * 检索匹配量词词典 - * + * * @return Hit 匹配结果描述 */ public Hit matchInQuantifierDict(char[] charArray, int begin, int length) { @@ -315,7 +311,7 @@ public Hit matchInQuantifierDict(char[] charArray, int begin, int length) { /** * 从已匹配的Hit中直接取出DictSegment，继续向下匹配 - * + * * @return Hit */ public Hit matchWithHit(char[] charArray, int currentIndex, Hit matchedHit) { @@ -325,7 +321,7 @@ public Hit matchWithHit(char[] charArray, int currentIndex, Hit matchedHit) { /** * 判断是否是停止词 - * + * * @return boolean */ public boolean isStopWord(char[] charArray, int begin, int length) { @@ -481,6 +477,10 @@ private static List getRemoteWords(String location) { } in.close(); response.close(); + String isRebase = getProperty(REBASE_REMOTE_EXT_DICT); + if (isRebase.equals("true")) { + return writeRemoteWords(buffer); + } return buffer; } response.close(); @@ -759,4 +759,31 @@ public void reLoadMainDict() { logger.info("重新加载词典完毕..."); } + /** + * 把远程扩展字典的词备份下来，并放入用户自定义的扩展字典中 + */ + private static List writeRemoteWords(List buffer) throws IOException { + Path path = PathUtils.get(getDictRoot(), Dictionary.PATH_DIC_REMOTE); + File file = path.toFile(); + List words = new ArrayList<>(); + FileReader fr = new FileReader(file); + BufferedReader br = new BufferedReader(fr); + String line; + while ((line = br.readLine()) != null) { + words.add(line); + } + if (buffer.containsAll(words)) { + buffer.removeAll(words); + } + fr.close(); + br.close(); + //写入remote.dic中 + FileWriter fw = new FileWriter(file, true); + for (String buf : buffer) { + fw.append("\n"); + fw.append(buf); + } + fw.close(); + return buffer; + } } From d52922bfde0dbddbf36974c7f8ecf6062d5a0333 Mon Sep 17 00:00:00 2001 From: "huangchunlin@liepiaowang.com" Date: Fri, 1 Sep 2017 18:16:15 +0800 Subject: [PATCH 2/3] Fix bug --- src/main/java/org/wltea/analyzer/dic/Dictionary.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/org/wltea/analyzer/dic/Dictionary.java b/src/main/java/org/wltea/analyzer/dic/Dictionary.java index 3f99ec50..5c6b4ff2 100644 --- a/src/main/java/org/wltea/analyzer/dic/Dictionary.java +++ b/src/main/java/org/wltea/analyzer/dic/Dictionary.java @@ -772,8 +772,10 @@ private static List writeRemoteWords(List buffer) throws IOExcep while ((line = br.readLine()) != null) { words.add(line); } - if (buffer.containsAll(words)) { - buffer.removeAll(words); + for (String str : words) { + if (buffer.contains(str)) { + buffer.remove(str); + } } fr.close(); br.close(); From adac0ac235d87ea34d48ff83db334e5a97321ff1 Mon Sep 17 00:00:00 2001 From: "huangchunlin@liepiaowang.com" Date: Fri, 1 Sep 2017 19:22:36 +0800 Subject: [PATCH 3/3] =?UTF-8?q?Fix=20=E4=BC=98=E5=8C=96=E4=BB=A3=E7=A0=81,?= =?UTF-8?q?=E9=98=B2=E6=AD=A2=E5=86=99=E5=85=A5=E9=87=8D=E5=A4=8D=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/wltea/analyzer/dic/Dictionary.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/wltea/analyzer/dic/Dictionary.java b/src/main/java/org/wltea/analyzer/dic/Dictionary.java index 5c6b4ff2..76d1228d 100644 --- a/src/main/java/org/wltea/analyzer/dic/Dictionary.java +++ b/src/main/java/org/wltea/analyzer/dic/Dictionary.java @@ -763,6 +763,13 @@ public void reLoadMainDict() { * 把远程扩展字典的词备份下来，并放入用户自定义的扩展字典中 */ private static List writeRemoteWords(List buffer) throws IOException { + if (buffer == null || buffer.size() == 0) { + return new ArrayList<>(); + } + Set bufferSet = new HashSet<>(buffer); + if (bufferSet.size() == 0) { + return new ArrayList<>(); + } Path path = PathUtils.get(getDictRoot(), Dictionary.PATH_DIC_REMOTE); File file = path.toFile(); List words = new ArrayList<>(); @@ -773,19 +780,19 @@ private static List writeRemoteWords(List buffer) throws IOExcep words.add(line); } for (String str : words) { - if (buffer.contains(str)) { - buffer.remove(str); + if (bufferSet.contains(str)) { + bufferSet.remove(str); } } fr.close(); br.close(); //写入remote.dic中 FileWriter fw = new FileWriter(file, true); - for (String buf : buffer) { + for (String buf : bufferSet) { fw.append("\n"); fw.append(buf); } fw.close(); - return buffer; + return new ArrayList<>(bufferSet); } }