introduced edge segments which is slightly slower 3% but more robust memory allocation for big graphs

Peter · Peter · commit a558ef22cbd2 · 2012-07-19T18:43:21.000+02:00
diff --git a/core/nbactions.xml b/core/nbactions.xml
@@ -18,7 +18,7 @@
         </goals>
         <properties>
             <exec.classpathScope>runtime</exec.classpathScope>
-            <exec.args>-Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.reader.OSMReader test=false debug=true shortestpath=true graph=graph-unterfranken.osm osm=/media/SAMSUNG/unterfranken.osm</exec.args>
+            <exec.args>-Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.reader.OSMReader test=true debug=true shortestpath=true graph=graph-unterfranken.osm osm=/media/SAMSUNG/unterfranken.osm</exec.args>
             <exec.executable>java</exec.executable>
         </properties>
     </action>
@@ -30,7 +30,7 @@
         </goals>
         <properties>
             <exec.classpathScope>runtime</exec.classpathScope>
-            <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.reader.OSMReader test=false debug=true shortestpath=true graph=graph-unterfranken.osm osm=/media/SAMSUNG/unterfranken.osm</exec.args>
+            <exec.args>-Xdebug -Xrunjdwp:transport=dt_socket,server=n,address=${jpda.address} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.reader.OSMReader test=true debug=true shortestpath=true graph=graph-unterfranken.osm osm=/media/SAMSUNG/unterfranken.osm</exec.args>
             <jpda.listen>true</jpda.listen>
             <exec.executable>java</exec.executable>
         </properties>
@@ -42,7 +42,7 @@
             <goal>org.codehaus.mojo:exec-maven-plugin:1.1.1:exec</goal>
         </goals>
         <properties>
-            <exec.args>${profiler.args} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.reader.OSMReader test=false debug=true shortestpath=true graph=graph-unterfranken.osm osm=/media/SAMSUNG/unterfranken.osm</exec.args>
+            <exec.args>${profiler.args} -Xms1000m -Xmx1000m -XX:PermSize=20m -XX:MaxPermSize=20m -classpath %classpath de.jetsli.graph.reader.OSMReader test=true debug=true shortestpath=true graph=graph-unterfranken.osm osm=/media/SAMSUNG/unterfranken.osm</exec.args>
             <profiler.action>profile</profiler.action>
             <exec.executable>${profiler.java}</exec.executable>
         </properties>
diff --git a/core/pom.xml b/core/pom.xml
@@ -35,23 +35,13 @@
             <groupId>org.slf4j</groupId>
             <artifactId>slf4j-api</artifactId>
             <version>${slf4j.version}</version>
-        </dependency>
+        </dependency>        
         <dependency>
             <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-log4j12</artifactId>
+            <artifactId>slf4j-simple</artifactId>
             <version>${slf4j.version}</version>
             <scope>runtime</scope>
-        </dependency>
-        
-        <!-- use 1.2.14 instead of 1.2.16 because of
-        java.lang.NullPointerException         at org.apache.log4j.helpers.ISO8601DateFormat.format(ISO8601DateFormat.java:70)
-        -->
-        <dependency>
-            <groupId>log4j</groupId>
-            <artifactId>log4j</artifactId>
-            <version>1.2.14</version>
-            <scope>compile</scope>
-        </dependency>
+        </dependency>        
         
         <dependency>
             <groupId>junit</groupId>
@@ -60,20 +50,7 @@
             <version>4.10</version>
             <scope>test</scope>
         </dependency>
-        <!--
-   <dependency>
-       <groupId>org.hamcrest</groupId>
-       <artifactId>hamcrest-core</artifactId>
-       <version>1.3.RC2</version>
-       <scope>test</scope>
-   </dependency>
-   <dependency>
-       <groupId>org.hamcrest</groupId>
-       <artifactId>hamcrest-library</artifactId>
-       <version>1.3.RC2</version>
-       <scope>test</scope>
-   </dependency>           
-        -->     
+        
     </dependencies>
         
     <build>
diff --git a/core/src/main/java/de/jetsli/graph/storage/MemoryGraphSafe.java b/core/src/main/java/de/jetsli/graph/storage/MemoryGraphSafe.java
@@ -41,32 +41,27 @@ public class MemoryGraphSafe implements SaveableGraph {
     private static final int EMPTY_LINK = 0;
     private static final float DIST_UNIT = 10000f;
     private Logger logger = LoggerFactory.getLogger(getClass());
+    // number of integers not edges
+    private static final int MIN_SEGMENT_SIZE = 1 << 13;
     private static final float FACTOR = 1.5f;
     // keep in mind that we address integers here - not bytes!
     private static final int LEN_DIST = 1;
     private static final int LEN_NODEID = 1;
     private static final int LEN_FLAGS = 1;
     private static final int LEN_LINK = 1;
     private static final int LEN_EDGE = LEN_FLAGS + LEN_DIST + LEN_NODEID + LEN_LINK;
-    // or should we use int and a factor? private static double MICRO = 1000000;
     private float[] lats;
     private float[] lons;
-    // private int[] priorities;
     private long creationTime = System.currentTimeMillis();
     private int size;
     private int[] refToEdges;
-    // TODO use a bitset to memorize fragmented edges-area!
-    private int[] edgesArea;
-    private int nextEdgePointer;
-//    private final ReadWriteLock lock;
-//    private final Lock writeLock;
-//    private final Lock readLock;
+    private int[][] edgesSegments;
+    private int edgesSegmentSize;
+    private int edgeCurrentSegment;
+    private int edgeNextGlobalPointer;
     private String storageLocation;
     private MyBitSet deletedNodes;
 
-//    public MemoryGraphSafe() {
-//        this(10);
-//    }
     public MemoryGraphSafe(int cap) {
         this(null, cap);
     }
@@ -83,13 +78,21 @@ public MemoryGraphSafe(String storageDir, int cap, int capEdge) {
         }
     }
 
+    int getSegmentSize() {
+        return edgesSegmentSize;
+    }
+
+    int getSegments() {
+        return edgesSegments.length;
+    }
+
     @Override public int getNodes() {
         // readLock.lock();
         return size;
     }
 
-    public int getMaxEdges() {
-        return edgesArea.length / LEN_EDGE;
+    private int getMaxEdges() {
+        return edgesSegments.length * edgesSegmentSize / LEN_EDGE;
     }
 
     @Override
@@ -101,27 +104,25 @@ public void setNode(int index, double lat, double lon) {
     }
 
     private void initEdges(int cap) {
-        cap *= LEN_EDGE;
-        edgesArea = new int[cap];
-    }
-
-    // Use ONLY within a writer lock area
-    private void ensureEdgesCapacity(int cap) {
-        ensureEdgePointer(cap * LEN_EDGE);
+        int tmp = (int) (Math.log(cap) / Math.log(2));
+        edgesSegmentSize = Math.max((int) Math.pow(2, tmp), MIN_SEGMENT_SIZE);
+        edgesSegments = new int[1][edgesSegmentSize];
     }
 
     // Use ONLY within a writer lock area
     private void ensureEdgePointer(int pointer) {
-        if (pointer + LEN_EDGE < edgesArea.length)
+        if (pointer + LEN_EDGE < edgesSegmentSize * getSegments())
             return;
 
-        // TODO instead of reallocation (memory waste, time to copy), create multiple segments for 
-        // edges and use the highest bits (any number 2^x) as the segment number.
-        // Choose 2^x as segment number so we can use bit operations and avoid modulo for edgeId lookup!
-        pointer = Math.max(10 * LEN_EDGE, Math.round(pointer * FACTOR));
-        logger.info("ensure space to " + (int) (pointer / LEN_EDGE) + " edges (" + (float) 4 * pointer / (1 << 20) + " MB)");
-        int newLen = pointer;
-        edgesArea = Arrays.copyOf(edgesArea, newLen);
+        // create a new segment
+        edgeCurrentSegment++;
+        int[][] tmp = new int[edgeCurrentSegment + 1][];
+        for (int i = 0; i < edgesSegments.length; i++) {
+            tmp[i] = edgesSegments[i];
+        }
+        tmp[edgeCurrentSegment] = new int[edgesSegmentSize];
+        edgesSegments = tmp;
+        // nextEdgePointer = 0;
     }
 
     // Use ONLY within a writer lock area
@@ -181,6 +182,20 @@ public void edge(int a, int b, double distance, boolean bothDirections) {
         internalAdd(b, a, distance, dirFlag);
     }
 
+    private void saveToEdgeArea(int pointer, int data) {
+        // TODO improve speed via bit operations!
+        int segNumber = pointer / edgesSegmentSize;
+        int segPointer = pointer % edgesSegmentSize;
+        edgesSegments[segNumber][segPointer] = data;
+    }
+
+    private int getFromEdgeArea(int pointer) {
+        // TODO improve speed via bit operations!
+        int segNumber = pointer / edgesSegmentSize;
+        int segPointer = pointer % edgesSegmentSize;
+        return edgesSegments[segNumber][segPointer];
+    }
+
     private void internalAdd(int fromNodeId, int toNodeId, double dist, int flags) {
         int edgePointer = refToEdges[fromNodeId];
         int newPos = nextEdgePointer();
@@ -197,7 +212,7 @@ private void internalAdd(int fromNodeId, int toNodeId, double dist, int flags) {
                 throw new IllegalStateException("list cannot be empty for positive edgePointer " + edgePointer + " node:" + fromNodeId);
 
             int linkPointer = getLink(list.get(list.size() - 1));
-            edgesArea[linkPointer] = newPos;
+            saveToEdgeArea(linkPointer, newPos);
         } else
             refToEdges[fromNodeId] = newPos;
 
@@ -211,30 +226,30 @@ private int getLink(int edgePointer) {
     private void writeEdge(int edgePointer, int flags, double dist, int toNodeId, int nextEdgePointer) {
         ensureEdgePointer(edgePointer);
 
-        edgesArea[edgePointer] = flags;
+        saveToEdgeArea(edgePointer, flags);
         edgePointer += LEN_FLAGS;
 
-        edgesArea[edgePointer] = (int) (dist * DIST_UNIT);
+        saveToEdgeArea(edgePointer, (int) (dist * DIST_UNIT));
         edgePointer += LEN_DIST;
 
-        edgesArea[edgePointer] = toNodeId;
+        saveToEdgeArea(edgePointer, toNodeId);
         edgePointer += LEN_NODEID;
 
-        edgesArea[edgePointer] = nextEdgePointer;
+        saveToEdgeArea(edgePointer, nextEdgePointer);
         // edgePointer += LEN_LINK;
     }
 
     private int nextEdgePointer() {
-        nextEdgePointer += LEN_EDGE;
-        return nextEdgePointer;
+        edgeNextGlobalPointer += LEN_EDGE;
+        return edgeNextGlobalPointer;
     }
 
     private TIntArrayList readAllEdges(int edgePointer) {
         TIntArrayList list = new TIntArrayList(5);
         int i = 0;
         for (; i < 1000; i++) {
             list.add(edgePointer);
-            edgePointer = edgesArea[getLink(edgePointer)];
+            edgePointer = getFromEdgeArea(getLink(edgePointer));
             if (edgePointer == EMPTY_LINK)
                 break;
         }
@@ -282,20 +297,20 @@ public EdgeIterable(int node, boolean in, boolean out) {
 
         void readNext() {
             // readLock.lock();
-            flags = edgesArea[pointer];
+            flags = getFromEdgeArea(pointer);
             if (!in && (flags & 1) == 0 || !out && (flags & 2) == 0) {
-                pointer = edgesArea[getLink(pointer)];
+                pointer = getFromEdgeArea(getLink(pointer));
                 return;
             }
             pointer += LEN_FLAGS;
 
-            dist = edgesArea[pointer] / DIST_UNIT;
+            dist = getFromEdgeArea(pointer) / DIST_UNIT;
             pointer += LEN_DIST;
 
-            nodeId = edgesArea[pointer];
+            nodeId = getFromEdgeArea(pointer);
             pointer += LEN_NODEID;
             // next edge
-            pointer = edgesArea[pointer];
+            pointer = getFromEdgeArea(pointer);
             foundNext = true;
         }
 
@@ -334,14 +349,25 @@ public int flags() {
     @Override
     public Graph clone() {
         // readLock.lock();
-        MemoryGraphSafe g = new MemoryGraphSafe(null, refToEdges.length, getMaxEdges());
-        System.arraycopy(lats, 0, g.lats, 0, lats.length);
-        System.arraycopy(lons, 0, g.lons, 0, lons.length);
-        System.arraycopy(refToEdges, 0, g.refToEdges, 0, refToEdges.length);
-        System.arraycopy(edgesArea, 0, g.edgesArea, 0, edgesArea.length);
-        g.nextEdgePointer = nextEdgePointer;
-        g.size = size;
-        return g;
+        MemoryGraphSafe clonedGraph = new MemoryGraphSafe(null, refToEdges.length, getMaxEdges());
+
+        System.arraycopy(lats, 0, clonedGraph.lats, 0, lats.length);
+        System.arraycopy(lons, 0, clonedGraph.lons, 0, lons.length);
+        System.arraycopy(refToEdges, 0, clonedGraph.refToEdges, 0, refToEdges.length);
+
+        clonedGraph.edgesSegments = new int[edgeCurrentSegment + 1][];
+        for (int i = 0; i < clonedGraph.edgesSegments.length; i++) {
+            clonedGraph.edgesSegments[i] = new int[edgesSegmentSize];
+        }
+
+        for (int i = 0; i < edgesSegments.length; i++) {
+            System.arraycopy(edgesSegments[i], 0, clonedGraph.edgesSegments[i], 0, edgesSegmentSize);
+        }
+        clonedGraph.edgeCurrentSegment = edgeCurrentSegment;
+        clonedGraph.edgesSegmentSize = edgesSegmentSize;
+        clonedGraph.edgeNextGlobalPointer = edgeNextGlobalPointer;
+        clonedGraph.size = size;
+        return clonedGraph;
     }
 
     private MyBitSet getDeletedNodes() {
@@ -380,7 +406,7 @@ public void optimize() {
         if (deleted == 0)
             return;
 
-//        if (deleted < size / 5) {
+//        if (deleted < size / 4) {
         inPlaceDelete(deleted);
 //        } else
 //            optimizeIfLotsOfDeletes(deleted);
@@ -524,7 +550,10 @@ void replacingDelete(int deleted) {
         lats = inMemGraph.lats;
         lons = inMemGraph.lons;
         refToEdges = inMemGraph.refToEdges;
-        edgesArea = inMemGraph.edgesArea;
+        for (int i = 0; i < edgesSegments.length; i++) {
+            edgesSegments[i] = inMemGraph.edgesSegments[i];
+        }
+
         size = inMemGraph.size;
         deletedNodes = null;
     }
@@ -541,9 +570,10 @@ public void save() {
             Helper.writeFloats(storageLocation + "/lats", lats);
             Helper.writeFloats(storageLocation + "/lons", lons);
             Helper.writeInts(storageLocation + "/refs", refToEdges);
-            Helper.writeInts(storageLocation + "/edges", edgesArea);
-            // Helper.writeInts(storageLocation + "/priorities", priorities);
-            Helper.writeSettings(storageLocation + "/settings", size, creationTime, nextEdgePointer);
+            for (int i = 0; i < edgesSegments.length; i++) {
+                Helper.writeInts(storageLocation + "/edges" + i, edgesSegments[i]);
+            }
+            Helper.writeSettings(storageLocation + "/settings", size, creationTime, edgeNextGlobalPointer, edgeCurrentSegment, edgesSegmentSize);
         } catch (IOException ex) {
             throw new RuntimeException("Couldn't write data to storage. location was " + storageLocation, ex);
         }
@@ -561,15 +591,22 @@ public boolean loadExisting(String storageDir) {
 
             size = (Integer) ob[0];
             creationTime = (Long) ob[1];
-            nextEdgePointer = (Integer) ob[2];
-            logger.info("found graph " + storageLocation + " with size:" + size
-                    + ", edges:" + nextEdgePointer / LEN_EDGE + ", created-at:" + new Date(creationTime));
+            edgeNextGlobalPointer = (Integer) ob[2];
+            edgeCurrentSegment = (Integer) ob[3];
+            edgesSegmentSize = (Integer) ob[4];
+            logger.info("found graph " + storageLocation + " with nodes:" + size
+                    + ", edges:" + edgeNextGlobalPointer / LEN_EDGE
+                    + ", edges segments:" + (edgeCurrentSegment + 1)
+                    + ", edges segmentSize:" + edgesSegmentSize
+                    + ", created-at:" + new Date(creationTime));
 
             lats = Helper.readFloats(storageLocation + "/lats");
             lons = Helper.readFloats(storageLocation + "/lons");
             refToEdges = Helper.readInts(storageLocation + "/refs");
-            edgesArea = Helper.readInts(storageLocation + "/edges");
-            // priorities = Helper.readInts(storageLocation + "/priorities");
+            edgesSegments = new int[edgeCurrentSegment + 1][];
+            for (int i = 0; i <= edgeCurrentSegment; i++) {
+                edgesSegments[i] = Helper.readInts(storageLocation + "/edges" + i);
+            }
             deletedNodes = new MyOpenBitSet(lats.length);
             return true;
         } catch (IOException ex) {
diff --git a/core/src/main/java/de/jetsli/graph/storage/MemoryGraphSafeStorage.java b/core/src/main/java/de/jetsli/graph/storage/MemoryGraphSafeStorage.java
@@ -33,8 +33,9 @@ public MemoryGraphSafeStorage(String file, int expectedNodes) {
     @Override
     public void createNew() {
         Helper.deleteDir(new File(folder));
-        // in order to avoid reallocation, allocate enough memory for the edges
-        g = new MemoryGraphSafe(folder, osmIdToIndexMap.size(), Math.round(2.2f * osmIdToIndexMap.size()));
+        // in order to avoid reallocation allocate enough memory. 
+        // edges will be incrementally allocated so it is not that important to match the size up front
+        g = new MemoryGraphSafe(folder, osmIdToIndexMap.size(), Math.round(0.8f * osmIdToIndexMap.size()));
     }
 
     @Override
diff --git a/core/src/test/java/de/jetsli/graph/storage/MemoryGraphSafeTest.java b/core/src/test/java/de/jetsli/graph/storage/MemoryGraphSafeTest.java
