Properties segments (graphhopper#3129)

* Simplify property saving

* Allow to store multiple segments

* Allow to load multiple segments

* Add test case for high amount of props

* Use try-with-resources

* bytePos iter

* Check capacity

Author: otbutz

core/src/main/java/com/graphhopper/storage/StorableProperties.java

@@ -22,6 +22,7 @@
 import org.slf4j.LoggerFactory;
 
 import java.io.*;
+import java.util.Arrays;
 import java.util.LinkedHashMap;
 import java.util.Map;
 
@@ -51,9 +52,18 @@ public synchronized boolean loadExisting() {
         if (!da.loadExisting())
             return false;
 
+        if (da.getCapacity() > Integer.MAX_VALUE) {
+            throw new IllegalStateException("Properties file is too large: " + da.getCapacity());
+        }
         int len = (int) da.getCapacity();
         byte[] bytes = new byte[len];
-        da.getBytes(0, bytes, len);
+        int segmentSize = da.getSegmentSize();
+        for (int bytePos = 0; bytePos < len; bytePos += segmentSize) {
+            int partLen = Math.min(bytes.length - bytePos, segmentSize);
+            byte[] part = new byte[partLen];
+            da.getBytes(bytePos, part, part.length);
+            System.arraycopy(part, 0, bytes, bytePos, partLen);
+        }
         try {
             loadProperties(map, new StringReader(new String(bytes, UTF_CS)));
             return true;
@@ -63,21 +73,23 @@ public synchronized boolean loadExisting() {
     }
 
     public synchronized void flush() {
-        try {
-            StringWriter sw = new StringWriter();
-            saveProperties(map, sw);
-            // TODO at the moment the size is limited to da.segmentSize() !
-            byte[] bytes = sw.toString().getBytes(UTF_CS);
-            da.setBytes(0, bytes, bytes.length);
-            da.flush();
-            // todo: would not be needed if the properties file used a format that is compatible with common text tools
-            if (dir.getDefaultType().isStoring()) {
-                try (BufferedWriter writer = new BufferedWriter(new FileWriter(dir.getLocation() + "/properties.txt"))) {
-                    writer.write(sw.toString());
-                }
+        String props = saveProperties(map);
+        byte[] bytes = props.getBytes(UTF_CS);
+        da.ensureCapacity(bytes.length);
+        int segmentSize = da.getSegmentSize();
+        for (int bytePos = 0; bytePos < bytes.length; bytePos += segmentSize) {
+            int partLen = Math.min(bytes.length - bytePos, segmentSize);
+            byte[] part = Arrays.copyOfRange(bytes, bytePos, bytePos + partLen);
+            da.setBytes(bytePos, part, part.length);
+        }
+        da.flush();
+        // todo: would not be needed if the properties file used a format that is compatible with common text tools
+        if (dir.getDefaultType().isStoring()) {
+            try (BufferedWriter writer = new BufferedWriter(new FileWriter(dir.getLocation() + "/properties.txt"))) {
+                writer.write(props);
+            } catch (IOException e) {
+                throw new RuntimeException(e);
             }
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
         }
     }
 
@@ -147,10 +159,20 @@ public synchronized String toString() {
         return da.toString();
     }
 
+    static String saveProperties(Map<String, String> map) {
+        StringBuilder builder = new StringBuilder();
+        for (Map.Entry<String, String> e : map.entrySet()) {
+            builder.append(e.getKey());
+            builder.append('=');
+            builder.append(e.getValue());
+            builder.append('\n');
+        }
+        return builder.toString();
+    }
+
     static void loadProperties(Map<String, String> map, Reader tmpReader) throws IOException {
-        BufferedReader reader = new BufferedReader(tmpReader);
-        String line;
-        try {
+        try (BufferedReader reader = new BufferedReader(tmpReader)) {
+            String line;
             while ((line = reader.readLine()) != null) {
                 if (line.startsWith("//") || line.startsWith("#")) {
                     continue;
@@ -170,8 +192,6 @@ static void loadProperties(Map<String, String> map, Reader tmpReader) throws IOE
                 String value = line.substring(index + 1);
                 map.put(field.trim(), value.trim());
             }
-        } finally {
-            reader.close();
         }
     }
 }

core/src/test/java/com/graphhopper/storage/StorablePropertiesTest.java

@@ -68,6 +68,30 @@ public void testStore() {
         Helper.removeDir(new File(dir));
     }
 
+    @Test
+    public void testStoreLarge() {
+        String dir = "./target/test";
+        Helper.removeDir(new File(dir));
+        StorableProperties instance = new StorableProperties(createDir(dir, true));
+        instance.create(1000);
+        for (int i = 0; i <= 100_000; i++) {
+            instance.put(Integer.toString(i), "test." + i);
+        }
+
+        instance.flush();
+        long bytesWritten = instance.getCapacity();
+        instance.close();
+
+        instance = new StorableProperties(createDir(dir, true));
+        assertTrue(instance.loadExisting());
+        assertEquals(bytesWritten, instance.getCapacity());
+        assertEquals("test.0", instance.get("0"));
+        assertEquals("test.100000", instance.get("100000"));
+        instance.close();
+
+        Helper.removeDir(new File(dir));
+    }
+
     @Test
     public void testLoadProperties() throws IOException {
         Map<String, String> map = new HashMap<>();

web-api/src/main/java/com/graphhopper/util/Helper.java

@@ -26,7 +26,6 @@
 import java.text.NumberFormat;
 import java.text.SimpleDateFormat;
 import java.util.*;
-import java.util.Map.Entry;
 
 /**
  * @author Peter Karich
@@ -65,20 +64,6 @@ public static String toUpperCase(String string) {
         return string.toUpperCase(Locale.ROOT);
     }
 
-    public static void saveProperties(Map<String, String> map, Writer tmpWriter) throws IOException {
-        BufferedWriter writer = new BufferedWriter(tmpWriter);
-        try {
-            for (Entry<String, String> e : map.entrySet()) {
-                writer.append(e.getKey());
-                writer.append('=');
-                writer.append(e.getValue());
-                writer.append('\n');
-            }
-        } finally {
-            writer.close();
-        }
-    }
-
     public static String readJSONFileWithoutComments(String file) throws IOException {
         return Helper.readFile(file).stream().
                 filter(line -> !line.trim().startsWith("//")).