重构EnumItemDictionary

Author: hankcs

src/main/java/com/hankcs/hanlp/dictionary/common/CommonDictionary.java

@@ -12,12 +12,17 @@
 package com.hankcs.hanlp.dictionary.common;
 
 import com.hankcs.hanlp.collection.trie.DoubleArrayTrie;
+import com.hankcs.hanlp.corpus.io.ByteArray;
 import com.hankcs.hanlp.corpus.io.IOUtil;
-import com.hankcs.hanlp.dictionary.BaseSearcher;
+import com.hankcs.hanlp.utility.TextUtility;
 
-import java.io.*;
+import java.io.BufferedReader;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
 import java.util.*;
 
+import static com.hankcs.hanlp.utility.Predefine.BIN_EXT;
 import static com.hankcs.hanlp.utility.Predefine.logger;
 
 /**
@@ -29,68 +34,84 @@ public abstract class CommonDictionary<V>
 {
     DoubleArrayTrie<V> trie;
 
+    protected abstract V[] loadValueArray(ByteArray byteArray);
+
     public boolean load(String path)
     {
         trie = new DoubleArrayTrie<V>();
         long start = System.currentTimeMillis();
-        V[] valueArray = onLoadValue(path);
-        if (valueArray == null)
-        {
-            logger.info("加载值" + path + ".value.dat失败，耗时" + (System.currentTimeMillis() - start) + "ms");
-            return false;
-        }
-        logger.info("加载值" + path + ".value.dat成功，耗时" + (System.currentTimeMillis() - start) + "ms");
-        start = System.currentTimeMillis();
-        if (loadDat(path + ".trie.dat", valueArray))
+        if (loadDat(ByteArray.createByteArray(path + BIN_EXT)))
         {
-            logger.info("加载键" + path + ".trie.dat成功，耗时" + (System.currentTimeMillis() - start) + "ms");
             return true;
         }
-        List<String> keyList = new ArrayList<String>(valueArray.length);
+        TreeMap<String, V> map = new TreeMap<String, V>();
         try
         {
             BufferedReader br = new BufferedReader(new InputStreamReader(IOUtil.newInputStream(path), "UTF-8"));
             String line;
             while ((line = br.readLine()) != null)
             {
                 String[] paramArray = line.split("\\s");
-                keyList.add(paramArray[0]);
+                map.put(paramArray[0], createValue(paramArray));
             }
             br.close();
         }
         catch (Exception e)
         {
             logger.warning("读取" + path + "失败" + e);
         }
-        int resultCode = trie.build(keyList, valueArray);
+        Set<Map.Entry<String, V>> entrySet = map.entrySet();
+        List<String> keyList = new ArrayList<String>(entrySet.size());
+        List<V> valueList = new ArrayList<V>(entrySet.size());
+        for (Map.Entry<String, V> entry : entrySet)
+        {
+            keyList.add(entry.getKey());
+            valueList.add(entry.getValue());
+        }
+        int resultCode = trie.build(keyList, valueList);
         if (resultCode != 0)
         {
-            logger.warning("trie建立失败" + resultCode + ",正在尝试排序后重载");
-            TreeMap<String, V> map = new TreeMap<String, V>();
-            for (int i = 0; i < valueArray.length; ++i)
-            {
-                map.put(keyList.get(i), valueArray[i]);
-            }
-            trie = new DoubleArrayTrie<V>();
-            trie.build(map);
-            int i = 0;
-            for (V v : map.values())
-            {
-                valueArray[i++] = v;
-            }
+            logger.warning("trie建立失败");
+            return false;
         }
-        trie.save(path + ".trie.dat");
-        onSaveValue(valueArray, path);
-        logger.info(path + "加载成功");
+        logger.info(path + "加载成功，耗时" + (System.currentTimeMillis() - start) + "ms");
+        saveDat(path + BIN_EXT, valueList);
         return true;
     }
 
-    private boolean loadDat(String path, V[] valueArray)
+    protected boolean loadDat(ByteArray byteArray)
+    {
+        V[] valueArray = loadValueArray(byteArray);
+        if (valueArray == null)
+        {
+            return false;
+        }
+        return trie.load(byteArray, valueArray);
+    }
+
+    protected boolean saveDat(String path, List<V> valueArray)
     {
-        if (trie.load(path, valueArray)) return true;
-        return false;
+        try
+        {
+            DataOutputStream out = new DataOutputStream(IOUtil.newOutputStream(path));
+            out.writeInt(valueArray.size());
+            for (V item : valueArray)
+            {
+                saveValue(item, out);
+            }
+            trie.save(out);
+            out.close();
+        }
+        catch (Exception e)
+        {
+            logger.warning("保存失败" + TextUtility.exceptionToString(e));
+            return false;
+        }
+        return true;
     }
 
+    protected abstract void saveValue(V value, DataOutputStream out) throws IOException;
+
     /**
      * 查询一个单词
      *
@@ -123,103 +144,5 @@ public int size()
         return trie.size();
     }
 
-    /**
-     * 排序这个词典
-     *
-     * @param path
-     * @return
-     */
-    public static boolean sort(String path)
-    {
-        TreeMap<String, String> map = new TreeMap<String, String>();
-        try
-        {
-            BufferedReader br = new BufferedReader(new InputStreamReader(IOUtil.newInputStream(path), "UTF-8"));
-            String line;
-            while ((line = br.readLine()) != null)
-            {
-                String[] argArray = line.split("\\s");
-                map.put(argArray[0], line);
-            }
-            br.close();
-            // 输出它们
-            BufferedWriter bw = new BufferedWriter(new OutputStreamWriter(IOUtil.newOutputStream(path)));
-            for (Map.Entry<String, String> entry : map.entrySet())
-            {
-                bw.write(entry.getValue());
-                bw.newLine();
-            }
-            bw.close();
-        }
-        catch (Exception e)
-        {
-            logger.warning("读取" + path + "失败" + e);
-            return false;
-        }
-        return true;
-    }
-
-    /**
-     * 实现此方法来加载值
-     *
-     * @param path
-     * @return
-     */
-    protected abstract V[] onLoadValue(String path);
-
-    protected abstract boolean onSaveValue(V[] valueArray, String path);
-
-    public BaseSearcher getSearcher(String text)
-    {
-        return new Searcher(text);
-    }
-
-    /**
-     * 前缀搜索，长短都可匹配
-     */
-    public class Searcher extends BaseSearcher<V>
-    {
-        /**
-         * 分词从何处开始，这是一个状态
-         */
-        int begin;
-
-        private List<Map.Entry<String, V>> entryList;
-
-        protected Searcher(char[] c)
-        {
-            super(c);
-        }
-
-        protected Searcher(String text)
-        {
-            super(text);
-            entryList = new LinkedList<Map.Entry<String, V>>();
-        }
-
-        @Override
-        public Map.Entry<String, V> next()
-        {
-            // 保证首次调用找到一个词语
-            while (entryList.size() == 0 && begin < c.length)
-            {
-                entryList = trie.commonPrefixSearchWithValue(c, begin);
-                ++begin;
-            }
-            // 之后调用仅在缓存用完的时候调用一次
-            if (entryList.size() == 0 && begin < c.length)
-            {
-                entryList = trie.commonPrefixSearchWithValue(c, begin);
-                ++begin;
-            }
-            if (entryList.size() == 0)
-            {
-                return null;
-            }
-            Map.Entry<String, V> result = entryList.get(0);
-            entryList.remove(0);
-            offset = begin - 1;
-            return result;
-        }
-    }
+    protected abstract V createValue(String[] params);
 }

src/main/java/com/hankcs/hanlp/dictionary/common/EnumItemDictionary.java

@@ -0,0 +1,78 @@
+/*
+ * <author>Hankcs</author>
+ * <email>[email protected]</email>
+ * <create-date>2017-11-14 下午8:32</create-date>
+ *
+ * <copyright file="EnumItemDictionary.java" company="码农场">
+ * Copyright (c) 2017, 码农场. All Right Reserved, http://www.hankcs.com/
+ * This source is subject to Hankcs. Please contact Hankcs to get more information.
+ * </copyright>
+ */
+package com.hankcs.hanlp.dictionary.common;
+
+import com.hankcs.hanlp.corpus.dictionary.item.EnumItem;
+import com.hankcs.hanlp.corpus.io.ByteArray;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * @author hankcs
+ */
+public abstract class EnumItemDictionary<E extends Enum<E>> extends CommonDictionary<EnumItem<E>>
+{
+    @Override
+    protected EnumItem<E> createValue(String[] params)
+    {
+        Map.Entry<String, Map.Entry<String, Integer>[]> args = EnumItem.create(params);
+        EnumItem<E> nrEnumItem = new EnumItem<E>();
+        for (Map.Entry<String, Integer> e : args.getValue())
+        {
+            nrEnumItem.labelMap.put(valueOf(e.getKey()), e.getValue());
+        }
+        return nrEnumItem;
+    }
+
+    protected abstract E valueOf(String name);
+
+    protected abstract E[] values();
+
+    protected abstract EnumItem<E> newItem();
+
+    @Override
+    final protected EnumItem<E>[] loadValueArray(ByteArray byteArray)
+    {
+        if (byteArray == null)
+        {
+            return null;
+        }
+        E[] nrArray = values();
+        int size = byteArray.nextInt();
+        EnumItem<E>[] valueArray = new EnumItem[size];
+        for (int i = 0; i < size; ++i)
+        {
+            int currentSize = byteArray.nextInt();
+            EnumItem<E> item = newItem();
+            for (int j = 0; j < currentSize; ++j)
+            {
+                E nr = nrArray[byteArray.nextInt()];
+                int frequency = byteArray.nextInt();
+                item.labelMap.put(nr, frequency);
+            }
+            valueArray[i] = item;
+        }
+        return valueArray;
+    }
+
+    @Override
+    protected void saveValue(EnumItem<E> item, DataOutputStream out) throws IOException
+    {
+        out.writeInt(item.labelMap.size());
+        for (Map.Entry<E, Integer> entry : item.labelMap.entrySet())
+        {
+            out.writeInt(entry.getKey().ordinal());
+            out.writeInt(entry.getValue());
+        }
+    }
+}