Merge pull request jprante#3 from Automattic/add-merged-average-profile

Add merged-average language profile

Author: stambizzle

scripts/README.md

@@ -1,15 +1,21 @@
+from collections import defaultdict
+from fractions import Fraction
 from io import BytesIO
+import json
+import math
 import os
 import re
+import shutil
 from zipfile import ZipFile
 
 import baker
 import ftfy
 import requests
 import xmltodict
 
-_TEST_RESOURCES_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)),
-                                    '../src/test/resources/org/xbib/elasticsearch/index/mapper/langdetect/')
+_THIS_PATH = os.path.dirname(os.path.abspath(__file__))
+_TEST_RESOURCES_PATH = os.path.join(_THIS_PATH, '../src/test/resources/org/xbib/elasticsearch/index/mapper/langdetect/')
+_MAIN_RESOURCES_PATH = os.path.join(_THIS_PATH, '../src/main/resources/langdetect/')
 
 # Supported languages according to https://github.com/shuyo/language-detection/blob/wiki/LanguageList.md
 _SUPPORTED_LANGUAGES = {
@@ -172,5 +178,57 @@ def generate_wordpress_translations_dataset(out_path=os.path.join(_TEST_RESOURCE
             for _, clean_text in clean_texts_with_len[:texts_per_language]:
                 out_file.write('{}\t{}\n'.format(supported_code, clean_text))
 
+
+@baker.command
+def merge_language_profiles(output_profile_dir='merged-average'):
+    """
+    Preprocess the original default and short-text profile files, averaging the normalized n-gram frequencies from the
+    two profiles to create a merged profile.
+
+    For example, if the trigram "xyz" appears 1 time out of 1000 trigrams in a default profile file and 1 out of 100
+    in a short profile file, its merged-average frequency would be (0.001 + 0.01) / 2 = 0.0055. This frequency is then
+    converted back to an integer, as the plugin's Java code assumes the frequencies are integers.
+
+    :param output_profile_dir: directory name under the main resource path where the merged profile will be saved,
+                               overwriting any existing files in the directory
+    """
+    merged_dir = os.path.join(_MAIN_RESOURCES_PATH, output_profile_dir)
+    shutil.rmtree(merged_dir, ignore_errors=True)
+    os.mkdir(merged_dir)
+    for lang in _SUPPORTED_LANGUAGES:
+        merged_profile = dict(name=lang, freq=defaultdict(Fraction), n_words=[1, 1, 1])
+        profile_paths = []
+        for profile_name in ['default', 'short']:
+            profile_path = os.path.join(_MAIN_RESOURCES_PATH, '' if profile_name == 'default' else 'short-text', lang)
+            if os.path.exists(profile_path):
+                profile_paths.append(profile_path)
+        # Copy the original profile without any processing if only one exists
+        for profile_path in profile_paths:
+            with open(profile_path, encoding='utf-8') as profile_file:
+                profile = json.load(profile_file)
+            # The n_words sums of some profiles are wrong so we fix them here
+            profile['n_words'] = [0, 0, 0]
+            for ngram, count in profile['freq'].items():
+                profile['n_words'][len(ngram) - 1] += count
+            for ngram, count in profile['freq'].items():
+                merged_profile['freq'][ngram] += Fraction(count,
+                                                          profile['n_words'][len(ngram) - 1] * len(profile_paths))
+        # The least common multiplier of the frequency denominators for each n-gram length is the new n_words
+        merged_n_words = merged_profile['n_words']
+        for ngram, freq in merged_profile['freq'].items():
+            n_words_index = len(ngram) - 1
+            merged_n_words[n_words_index] = int(merged_n_words[n_words_index] * freq.denominator /
+                                                math.gcd(merged_n_words[n_words_index], freq.denominator))
+        # Ensure we don't exceed the maximum long value in Java
+        for n_words in merged_n_words:
+            assert n_words < 2 ** 63
+        n_words_check = list(merged_n_words)
+        for ngram, freq in merged_profile['freq'].items():
+            merged_profile['freq'][ngram] = int(merged_n_words[len(ngram) - 1] * freq)
+            n_words_check[len(ngram) - 1] -= merged_profile['freq'][ngram]
+        assert not sum(n_words_check)
+        with open(os.path.join(merged_dir, lang), 'w', encoding='utf-8') as out_file:
+            json.dump(merged_profile, out_file, ensure_ascii=False, separators=',:')
+
 if __name__ == '__main__':
     baker.run()

scripts/run.py

@@ -12,18 +12,34 @@
 import java.util.Map;
 
 public class LangProfile {
+    private final String name;
+    private final Map<String, Long> freq = new HashMap<>();
+    private final List<Long> n_words = new ArrayList<>(NGram.N_GRAM);
 
-    private String name;
-
-    private Map<String, Integer> freq;
-
-    private List<Integer> n_words;
-
-    public LangProfile() {
-        this.freq = new HashMap<>();
-        this.n_words = new ArrayList<>(NGram.N_GRAM);
+    /**
+     * Create an empty language profile.
+     */
+    public LangProfile(String name) {
+        this.name = name;
         for (int i = 0; i < NGram.N_GRAM; i++) {
-            n_words.add(0);
+            n_words.add(0L);
+        }
+    }
+
+    /**
+     * Create a language profile from a JSON input stream.
+     */
+    @SuppressWarnings("unchecked")
+    public LangProfile(InputStream input) throws IOException {
+        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(input);
+        Map<String, Object> map = parser.map();
+        this.name = (String) map.get("name");
+        // Explicity convert the numbers because they may get parsed as Integers or Longs.
+        for (Map.Entry<String, Number> entry : ((Map<String, Number>) map.get("freq")).entrySet()) {
+            freq.put(entry.getKey(), entry.getValue().longValue());
+        }
+        for (Number n : (List<Number>) map.get("n_words")) {
+            n_words.add(n.longValue());
         }
     }
 
@@ -39,37 +55,19 @@ public void add(String gram) {
         if (freq.containsKey(gram)) {
             freq.put(gram, freq.get(gram) + 1);
         } else {
-            freq.put(gram, 1);
+            freq.put(gram, 1L);
         }
     }
 
-    public void setName(String name) {
-        this.name = name;
-    }
-
     public String getName() {
         return name;
     }
 
-    public List<Integer> getNWords() {
+    public List<Long> getNWords() {
         return n_words;
     }
 
-    public void setFreq(Map<String, Integer> freq) {
-        this.freq = freq;
-    }
-
-    public Map<String, Integer> getFreq() {
+    public Map<String, Long> getFreq() {
         return freq;
     }
-
-    @SuppressWarnings("unchecked")
-    public void read(InputStream input) throws IOException {
-        XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(input);
-        Map<String,Object> map = parser.map();
-        freq = (Map<String, Integer>) map.get("freq");
-        name = (String)map.get("name");
-        n_words = (List<Integer>)map.get("n_words");
-    }
-
 }

src/main/java/org/xbib/elasticsearch/common/langdetect/LangProfile.java

@@ -188,9 +188,7 @@ public void loadProfileFromResource(String resource,  int index, int langsize) t
         if (in == null) {
             throw new IOException("profile '" + resource + "' not found");
         }
-        LangProfile langProfile = new LangProfile();
-        langProfile.read(in);
-        addProfile(langProfile, index, langsize);
+        addProfile(new LangProfile(in), index, langsize);
     }
 
     public void addProfile(LangProfile profile, int index, int langsize) throws IOException {

src/main/java/org/xbib/elasticsearch/common/langdetect/LangdetectService.java

@@ -54,7 +54,7 @@ public class DetectLanguageAccuracyTest extends Assert {
     private final String datasetName;
     private final int substringLength;
     private final int sampleSize;
-    private final boolean useShortProfile;
+    private final String profileParam;
     private final boolean useAllLanguages;
     private final Map<String, Double> languageToExpectedAccuracy;
 
@@ -69,20 +69,20 @@ public class DetectLanguageAccuracyTest extends Assert {
      * @param datasetName multi-language dataset name, as read in the setup step (see {@link #setUp()})
      * @param substringLength substring length to test (see {@link #generateSubstringSample(String, int, int)})
      * @param sampleSize number of substrings to test (see {@link #generateSubstringSample(String, int, int)})
-     * @param useShortProfile if true, the short text language profile will be used instead of the default profile
+     * @param profileParam profile name parameter to pass to the detection service 
      * @param useAllLanguages if true, all supported languages will be used instead of  just the default ones
      * @param languageToExpectedAccuracy mapping from language code to expected accuracy 
      */
     public DetectLanguageAccuracyTest(String datasetName,
                                       int substringLength,
                                       int sampleSize,
-                                      boolean useShortProfile,
+                                      String profileParam,
                                       boolean useAllLanguages,
                                       Map<String, Double> languageToExpectedAccuracy) {
         this.datasetName = datasetName;
         this.substringLength = substringLength;
         this.sampleSize = sampleSize;
-        this.useShortProfile = useShortProfile;
+        this.profileParam = profileParam;
         this.useAllLanguages = useAllLanguages;
         this.languageToExpectedAccuracy = languageToExpectedAccuracy;
     }
@@ -104,7 +104,7 @@ public static void setUp() throws IOException {
             // Write column headers
             Files.write(
                 outputPath,
-                Collections.singletonList("datasetName,substringLength,sampleSize,useShortProfile,useAllLanguages," +
+                Collections.singletonList("datasetName,substringLength,sampleSize,profileParam,useAllLanguages," +
                                           ALL_LANGUAGES),
                 StandardCharsets.UTF_8
             );
@@ -121,12 +121,20 @@ public void test() throws IOException {
         // Set up the detection service according to the test's parameters
         String languageSetting = DEFAULT_LANGUAGES;
         if (useAllLanguages) {
-            languageSetting = useShortProfile ? ALL_SHORT_PROFILE_LANGUAGES : ALL_DEFAULT_PROFILE_LANGUAGES;
+            // TODO: This is a bit clunky. LangdetectService should support "all" as a language setting.
+            if (profileParam.isEmpty()) {
+                languageSetting = ALL_DEFAULT_PROFILE_LANGUAGES;
+            } else if (profileParam.equals("short-text")) {
+                languageSetting = ALL_SHORT_PROFILE_LANGUAGES;
+            } else {
+                assertEquals(profileParam, "merged-average");
+                languageSetting = ALL_LANGUAGES;
+            }
         }
         LangdetectService service = new LangdetectService(
             Settings.builder()
                     .put("languages", languageSetting)
-                    .put("profile", useShortProfile ? "short-text" : "")
+                    .put("profile", profileParam)
                     .build()
         );
         Map<String, List<String>> languageToFullTexts = multiLanguageDatasets.get(datasetName);
@@ -159,7 +167,7 @@ public void test() throws IOException {
             }
         } else {
             List<Object> row = new ArrayList<>();
-            Collections.addAll(row, datasetName, substringLength, sampleSize, useShortProfile, useAllLanguages);
+            Collections.addAll(row, datasetName, substringLength, sampleSize, profileParam, useAllLanguages);
             for (String language : ALL_LANGUAGES.split(",")) {
                 row.add(languageToAccuracy.containsKey(language) ? languageToAccuracy.get(language) : Double.NaN);
             }
@@ -175,7 +183,7 @@ public void test() throws IOException {
      *
      * @return the parsed parameters
      */
-    @Parameterized.Parameters(name="{0}: substringLength={1} sampleSize={2} useShortProfile={3} useAllLanguages={4}")
+    @Parameterized.Parameters(name="{0}: substringLength={1} sampleSize={2} profileParam={3} useAllLanguages={4}")
     public static Collection<Object[]> data() throws IOException {
         List<Object[]> data = new ArrayList<>();
         try (BufferedReader br = getResourceReader("accuracies.csv")) {
@@ -191,8 +199,8 @@ public static Collection<Object[]> data() throws IOException {
                     scanner.nextInt(),
                     // sampleSize
                     scanner.nextInt(),
-                    // useShortProfile
-                    scanner.nextBoolean(),
+                    // profileParam
+                    scanner.next(),
                     // useAllLanguages
                     scanner.nextBoolean(),
                     // languageToExpectedAccuracy

src/main/resources/langdetect/merged-average/af

@@ -24,22 +24,19 @@ public static void setUp() throws Exception {
 
         detect = new LangdetectService(Settings.EMPTY);
 
-        LangProfile profile_en = new LangProfile();
-        profile_en.setName("en_test");
+        LangProfile profile_en = new LangProfile("en_test");
         for (String w : TRAINING_EN.split(" ")) {
             profile_en.add(w);
         }
         detect.addProfile(profile_en, 0, 3);
 
-        LangProfile profile_fr = new LangProfile();
-        profile_fr.setName("fr_test");
+        LangProfile profile_fr = new LangProfile("fr_test");
         for (String w : TRAINING_FR.split(" ")) {
             profile_fr.add(w);
         }
         detect.addProfile(profile_fr, 1, 3);
 
-        LangProfile profile_ja = new LangProfile();
-        profile_ja.setName("ja_test");
+        LangProfile profile_ja = new LangProfile("ja_test");
         for (String w : TRAINING_JA.split(" ")) {
             profile_ja.add(w);
         }

src/main/resources/langdetect/merged-average/ar

@@ -8,52 +8,48 @@ public class LangProfileTest extends Assert {
 
     @Test
     public final void testLangProfile() {
-        LangProfile profile = new LangProfile();
+        LangProfile profile = new LangProfile((String) null);
         assertEquals(profile.getName(), null);
     }
 
     @Test
     public final void testLangProfileStringInt() {
-        LangProfile profile = new LangProfile();
-        profile.setName("en");
+        LangProfile profile = new LangProfile("en");
         assertEquals(profile.getName(), "en");
     }
 
     @Test
     public final void testAdd() {
-        LangProfile profile = new LangProfile();
-        profile.setName("en");
+        LangProfile profile = new LangProfile("en");
         profile.add("a");
-        assertEquals((int) profile.getFreq().get("a"), 1);
+        assertEquals((long) profile.getFreq().get("a"), 1);
         profile.add("a");
-        assertEquals((int) profile.getFreq().get("a"), 2);
+        assertEquals((long) profile.getFreq().get("a"), 2);
         //profile.omitLessFreq();
     }
 
     @Test
     public final void testAddIllegally1() {
-        LangProfile profile = new LangProfile();
+        LangProfile profile = new LangProfile((String) null);
         profile.add("a");
         assertEquals(profile.getFreq().get("a"), null);
     }
 
     @Test
     public final void testAddIllegally2() {
-        LangProfile profile = new LangProfile();
-        profile.setName("en");
+        LangProfile profile = new LangProfile("en");
         profile.add("a");
         profile.add("");
         profile.add("abcd");
-        assertEquals((int) profile.getFreq().get("a"), 1);
+        assertEquals((long) profile.getFreq().get("a"), 1);
         assertEquals(profile.getFreq().get(""), null);
         assertEquals(profile.getFreq().get("abcd"), null);
 
     }
 
     @Test
     public final void testOmitLessFreq() {
-        LangProfile profile = new LangProfile();
-        profile.setName("en");
+        LangProfile profile = new LangProfile("en");
         String[] grams = "a b c \u3042 \u3044 \u3046 \u3048 \u304a \u304b \u304c \u304d \u304e \u304f".split(" ");
         for (int i = 0; i < 5; ++i) {
             for (String g : grams) {
@@ -62,9 +58,9 @@ public final void testOmitLessFreq() {
         }
         profile.add("\u3050");
 
-        assertEquals((int) profile.getFreq().get("a"), 5);
-        assertEquals((int) profile.getFreq().get("\u3042"), 5);
-        assertEquals((int) profile.getFreq().get("\u3050"), 1);
+        assertEquals((long) profile.getFreq().get("a"), 5);
+        assertEquals((long) profile.getFreq().get("\u3042"), 5);
+        assertEquals((long) profile.getFreq().get("\u3050"), 1);
         //profile.omitLessFreq();
         //assertEquals(profile.freq.get("a"), null);
         //assertEquals((int) profile.freq.get("\u3042"), 5);