store and access pillarNodes; found one more problem with pillar nodes (finding subnetworks)

Author: Peter

config-example.properties

@@ -14,21 +14,14 @@ osmreader.sortGraph=true
 # use levelgraph to speed things up (requires more RAM/disc space for holding the graph)
 #osmreader.levelgraph=true
 #  the following options require levelgraph=true
-#osmreader.towerNodesShortcuts=true
 #osmreader.chShortcuts=true
-osmreader.algoIterations=1000
 
 #  other options than CAR are currently not supported
 osmreader.type=CAR
 
-# how much resolution do you want to resolve lat,lon
-# default should be good for you
-osmreader.locationIndexCapacity=200000
-
-
 osmreader.test=false
 osmreader.runshortestpath=false
-
+osmreader.algoIterations=1000
 
 #  specifiy via command line
 #osmreader.graph-location=

src/main/java/com/graphhopper/reader/OSMReader.java

@@ -20,7 +20,6 @@
 import com.graphhopper.routing.util.AlgorithmPreparation;
 import com.graphhopper.routing.util.FastestCarCalc;
 import com.graphhopper.routing.ch.PrepareContractionHierarchies;
-import com.graphhopper.routing.util.PrepareTowerNodesShortcuts;
 import com.graphhopper.routing.util.PrepareRoutingSubnetworks;
 import com.graphhopper.routing.util.RoutingAlgorithmSpecialAreaTests;
 import com.graphhopper.storage.Directory;
@@ -32,7 +31,6 @@
 import com.graphhopper.storage.Location2IDQuadtree;
 import com.graphhopper.storage.RAMDirectory;
 import com.graphhopper.util.CmdArgs;
-import com.graphhopper.util.DistanceCalc;
 import com.graphhopper.util.GraphUtility;
 import com.graphhopper.util.Helper;
 import com.graphhopper.util.Helper7;
@@ -85,14 +83,13 @@ public static void main(String[] strs) throws Exception {
     private static Logger logger = LoggerFactory.getLogger(OSMReader.class);
     private int locations;
     private int skippedLocations;
-    private int nextEdgeIndex;
+    private int edgeCount;
     private int skippedEdges;
     private GraphStorage graphStorage;
     private OSMReaderHelper helper;
     private int expectedNodes;
     private TLongArrayList tmpLocs = new TLongArrayList(10);
     private Map<String, Object> properties = new HashMap<String, Object>();
-    private DistanceCalc callback = new DistanceCalc();
     private AcceptStreet acceptStreets = new AcceptStreet(true, false, false, false);
     private AlgorithmPreparation prepare;
     private Location2IDQuadtree index;
@@ -151,7 +148,11 @@ public static OSMReader osm2Graph(OSMReader osmReader, CmdArgs args) throws IOEx
         final String algoStr = args.get("osmreader.algo", "astar");
         osmReader.setDefaultAlgoPrepare(Helper.createAlgoPrepare(algoStr));
         osmReader.setSort(args.getBool("osmreader.sortGraph", false));
-        osmReader.setTowerNodeShortcuts(args.getBool("osmreader.towerNodesShortcuts", false));
+        
+        // TODO LATER make this configurable in OSMReaderHelper
+        if (args.getBool("osmreader.towerNodesShortcuts", false))
+            throw new IllegalArgumentException("towerNodes are always automatically created");
+
         osmReader.setCHShortcuts(args.get("osmreader.chShortcuts", "no"));
         if (!osmReader.loadExisting()) {
             String strOsm = args.get("osmreader.osm", "");
@@ -250,7 +251,7 @@ public void flush() {
         if (indexCapacity < 0)
             indexCapacity = Helper.calcIndexSize(graphStorage.getBounds());
         logger.info("now initializing and flushing index with " + indexCapacity);
-        getLocation2IDIndex().prepareIndex(indexCapacity);        
+        getLocation2IDIndex().prepareIndex(indexCapacity);
         index.flush();
     }
 
@@ -278,7 +279,7 @@ public void writeOsm2Graph(InputStream is) {
                         if ("node".equals(sReader.getLocalName())) {
                             processNode(sReader);
                             if (counter % 10000000 == 0) {
-                                logger.info(counter + ", locs:" + locations + " (" + skippedLocations + "), edges:" + nextEdgeIndex
+                                logger.info(counter + ", locs:" + locations + " (" + skippedLocations + "), edges:" + edgeCount
                                         + " (" + skippedEdges + "), " + Helper.getMemInfo());
                             }
                         } else if ("way".equals(sReader.getLocalName())) {
@@ -295,7 +296,7 @@ public void writeOsm2Graph(InputStream is) {
                             }
                             // counter does +=2 until the next loop
                             if ((counter / 2) % 1000000 == 0) {
-                                logger.info(counter + ", locs:" + locations + " (" + skippedLocations + "), edges:" + nextEdgeIndex
+                                logger.info(counter + ", locs:" + locations + " (" + skippedLocations + "), edges:" + edgeCount
                                         + " (" + skippedEdges + "), " + Helper.getMemInfo());
                             }
                         }
@@ -393,18 +394,11 @@ public boolean isHighway(XMLStreamReader sReader) throws XMLStreamException {
 
     public void processHighway(XMLStreamReader sReader) throws XMLStreamException {
         if (isHighway(sReader) && tmpLocs.size() > 1) {
-            int l = tmpLocs.size();
-            long prevOsmId = tmpLocs.get(0);
+            int all = tmpLocs.size();
             int flags = acceptStreets.toFlags(properties);
-            for (int ii = 1; ii < l; ii++) {
-                long currOsmId = tmpLocs.get(ii);
-                boolean ret = helper.addEdge(prevOsmId, currOsmId, flags, callback);
-                if (ret)
-                    nextEdgeIndex++;
-                else
-                    skippedEdges++;
-                prevOsmId = currOsmId;
-            }
+            int successfullAdded = helper.addEdge(tmpLocs, flags);
+            edgeCount += successfullAdded;
+            skippedEdges += all - successfullAdded;
         }
     }
 
@@ -431,15 +425,6 @@ public OSMReader setCHShortcuts(String chShortcuts) {
         return this;
     }
 
-    /**
-     * @param bool if yes then shortcuts will be introduced to skip all nodes with only 2 edges.
-     */
-    public OSMReader setTowerNodeShortcuts(boolean bool) {
-        if (bool)
-            prepare = new PrepareTowerNodesShortcuts();
-        return this;
-    }
-
     private void setDefaultAlgoPrepare(AlgorithmPreparation defaultPrepare) {
         prepare = defaultPrepare;
         prepare.setGraph(graphStorage);

src/main/java/com/graphhopper/reader/OSMReaderHelper.java

@@ -19,7 +19,10 @@
 import com.graphhopper.storage.GraphStorage;
 import com.graphhopper.util.DistanceCalc;
 import com.graphhopper.util.EdgeIterator;
-import com.graphhopper.util.GraphUtility;
+import gnu.trove.list.TDoubleList;
+import gnu.trove.list.TIntList;
+import gnu.trove.list.TLongList;
+import gnu.trove.list.array.TIntArrayList;
 import java.io.InputStream;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -35,53 +38,64 @@ public abstract class OSMReaderHelper {
     protected int zeroCounter = 0;
     protected final Graph g;
     protected final int expectedNodes;
+    private DistanceCalc callback = new DistanceCalc();
 
     public OSMReaderHelper(Graph g, int expectedNodes) {
         this.g = g;
         this.expectedNodes = expectedNodes;
     }
 
+    public void setCallback(DistanceCalc callback) {
+        this.callback = callback;
+    }
+
     public int getExpectedNodes() {
         return expectedNodes;
     }
 
+    public void preProcess(InputStream osmXml) {
+    }
+
     public boolean addNode(long osmId, double lat, double lon) {
         return true;
     }
 
-    public abstract boolean addEdge(long nodeIdFrom, long nodeIdTo, int flags, DistanceCalc callback);
+    public abstract int addEdge(TLongList nodes, int flags);
 
-    public void preProcess(InputStream osmXml) {
-    }
+    public int addEdge(TDoubleList latitudes, TDoubleList longitudes,
+            TIntList allNodes, int flags) {
+        int nodes = allNodes.size();
+        if (latitudes.size() != nodes || longitudes.size() != nodes)
+            throw new IllegalArgumentException("latitudes.size must be equals to longitudes.size and node list size " + nodes);
 
-    public boolean addEdge(double laf, double lof, double lat, double lot,
-            int fromIndex, int toIndex, int flags, DistanceCalc callback) {
-        double dist = callback.calcDist(laf, lof, lat, lot);
-        if (dist == 0) {
+        double towerNodeDistance = 0;
+        double prevLat = latitudes.get(0);
+        double prevLon = longitudes.get(0);
+        double lat;
+        double lon;
+        for (int i = 1; i < nodes; i++) {
+            lat = latitudes.get(i);
+            lon = longitudes.get(i);
+            towerNodeDistance += callback.calcDist(prevLat, prevLon, lat, lon);
+            prevLat = lat;
+            prevLon = lon;
+        }
+        if (towerNodeDistance == 0) {
             // As investigation shows often two paths should have crossed via one identical point 
             // but end up in two very close points. later this will be removed/fixed while 
             // removing short edges where one node is of degree 2
             zeroCounter++;
-            dist = 0.0001;
-        } else if (dist < 0) {
-            logger.info(counter + " - distances negative. " + fromIndex + " (" + laf + ", " + lof + ")->"
-                    + toIndex + "(" + lat + ", " + lot + ") :" + dist);
-            return false;
+            towerNodeDistance = 0.0001;
         }
 
-        EdgeIterator iter = GraphUtility.until(g.getOutgoing(fromIndex), toIndex);
-        if (!iter.isEmpty()) {
-            if (flags == iter.flags() && dist > iter.distance()) {
-                // silently skip if exactly the same way and the new one would be longer
-//                    return true;
-            }
-//                else logger.warn("longer edge already exists " + fromIndex + "->" + toIndex + "!? "
-//                            + "existing: " + iter.distance() + "|" + BitUtil.toBitString(iter.flags(), 8)
-//                            + " new:" + dist + "|" + BitUtil.toBitString(flags, 8));
+        int fromIndex = allNodes.get(0);
+        int toIndex = allNodes.get(nodes - 1);
+        EdgeIterator iter = g.edge(fromIndex, toIndex, towerNodeDistance, flags);
+        if (nodes > 2) {
+            TIntList pillarNodes = allNodes.subList(1, nodes - 1);
+            iter.pillarNodes(pillarNodes);
         }
-
-        g.edge(fromIndex, toIndex, dist, flags);
-        return true;
+        return nodes;
     }
 
     public String getInfo() {

src/main/java/com/graphhopper/reader/OSMReaderHelperDoubleParse.java

@@ -18,11 +18,14 @@
 import com.graphhopper.storage.DataAccess;
 import com.graphhopper.storage.Directory;
 import com.graphhopper.storage.GraphStorage;
-import com.graphhopper.util.DistanceCalc;
 import com.graphhopper.util.Helper;
 import com.graphhopper.util.Helper7;
+import gnu.trove.list.TLongList;
+import gnu.trove.list.array.TDoubleArrayList;
+import gnu.trove.list.array.TIntArrayList;
 import gnu.trove.list.array.TLongArrayList;
 import gnu.trove.map.hash.TIntIntHashMap;
+import gnu.trove.map.hash.TLongIntHashMap;
 import java.io.InputStream;
 import java.util.HashMap;
 import java.util.Map;
@@ -43,7 +46,7 @@
 public class OSMReaderHelperDoubleParse extends OSMReaderHelper {
 
     private final Logger logger = LoggerFactory.getLogger(getClass());
-    private TIntIntHashMap osmIdToIndexMap;
+    private TLongIntHashMap osmIdToIndexMap;
     // very slow: private SparseLongLongArray osmIdToIndexMap;
     // not applicable as ways introduces the nodes in 'wrong' order: new OSMIDSegmentedMap
     private int internalId = 0;
@@ -55,20 +58,21 @@ public class OSMReaderHelperDoubleParse extends OSMReaderHelper {
     public OSMReaderHelperDoubleParse(GraphStorage storage, int expectedNodes) {
         super(storage, expectedNodes);
         dir = storage.getDirectory();
-        indexToCount = dir.findCreate("tmpOSMReaderMap");        
-        osmIdToIndexMap = new TIntIntHashMap(expectedNodes, 1.4f, -1, -1);
+        indexToCount = dir.findCreate("tmpOSMReaderMap");
+        // we expect 4 nodes between two tower nodes => 2/6 => 1/3
+        osmIdToIndexMap = new TLongIntHashMap(expectedNodes / 3, 1.4f, -1, -1);
     }
 
     @Override
     public boolean addNode(long osmId, double lat, double lon) {
-        int count = osmIdToIndexMap.get((int) osmId);
+        int count = osmIdToIndexMap.get(osmId);
         if (count > FILLED)
             return false;
 
         g.setNode(internalId, lat, lon);
         indexToCount.ensureCapacity(4 * (internalId + 1));
         indexToCount.setInt(internalId, 1 - (count - FILLED));
-        osmIdToIndexMap.put((int) osmId, internalId);
+        osmIdToIndexMap.put(osmId, internalId);
         internalId++;
         return true;
     }
@@ -79,21 +83,35 @@ public int getExpectedNodes() {
     }
 
     @Override
-    public boolean addEdge(long nodeIdFrom, long nodeIdTo, int flags, DistanceCalc callback) {
-        int fromIndex = (int) osmIdToIndexMap.get((int) nodeIdFrom);
-        int toIndex = (int) osmIdToIndexMap.get((int) nodeIdTo);
-        if (fromIndex < 0 || toIndex < 0)
-            return false;
-
-        try {
-            double laf = g.getLatitude(fromIndex);
-            double lof = g.getLongitude(fromIndex);
-            double lat = g.getLatitude(toIndex);
-            double lot = g.getLongitude(toIndex);
-            return addEdge(laf, lof, lat, lot, fromIndex, toIndex, flags, callback);
-        } catch (Exception ex) {
-            throw new RuntimeException("Problem to add edge! with node " + fromIndex + "->" + toIndex + " osm:" + nodeIdFrom + "->" + nodeIdTo, ex);
+    public int addEdge(TLongList nodes, int flags) {
+        TDoubleArrayList longitudes = new TDoubleArrayList(nodes.size());
+        TDoubleArrayList latitudes = new TDoubleArrayList(nodes.size());
+        TIntArrayList allNodes = new TIntArrayList(nodes.size());
+        int successfullyAdded = 0;
+        for (int i = 0; i < nodes.size(); i++) {
+            int tmpNode = osmIdToIndexMap.get(nodes.get(i));
+            if (tmpNode < 0)
+                continue;
+
+            allNodes.add(tmpNode);
+            latitudes.add(g.getLatitude(tmpNode));
+            longitudes.add(g.getLongitude(tmpNode));
+            // split way into more than one edge!
+            if (allNodes.size() > 1 && indexToCount.getInt(tmpNode) > 1) {
+                successfullyAdded += addEdge(latitudes, longitudes, allNodes, flags);
+                latitudes.clear();
+                longitudes.clear();
+                allNodes.clear();
+                allNodes.add(tmpNode);
+                latitudes.add(g.getLatitude(tmpNode));
+                longitudes.add(g.getLongitude(tmpNode));
+            }
         }
+        if (allNodes.size() > 1)
+            successfullyAdded += addEdge(latitudes, longitudes, allNodes, flags);
+        return successfullyAdded;
+//       throw new RuntimeException("Problem to add edge! with node "
+//              + fromIndex + "->" + toIndex + " osm:" + nodeIdFrom + "->" + nodeIdTo, ex);        
     }
 
     @Override
@@ -126,7 +144,7 @@ public String toString() {
      */
     @Override
     public void preProcess(InputStream osmXml) {
-        indexToCount.createNew(expectedNodes);        
+        indexToCount.createNew(expectedNodes);
         if (osmXml == null)
             throw new IllegalStateException("Stream cannot be empty");
 

src/main/java/com/graphhopper/reader/OSMReaderHelperSingleParse.java

@@ -21,6 +21,7 @@
 import com.graphhopper.util.DistanceCalc;
 import com.graphhopper.util.Helper;
 import com.graphhopper.util.shapes.CoordTrig;
+import gnu.trove.list.TLongList;
 import org.slf4j.LoggerFactory;
 
 /**
@@ -57,25 +58,26 @@ public void startWayProcessing() {
     }
 
     @Override
-    public boolean addEdge(long nodeIdFrom, long nodeIdTo, int flags, DistanceCalc callback) {
-        int fromIndex = (int) osmIdToIndexMap.get(nodeIdFrom);
-        int toIndex = (int) osmIdToIndexMap.get(nodeIdTo);
-        if (fromIndex == osmIdToIndexMap.getNoEntryValue() || toIndex == osmIdToIndexMap.getNoEntryValue())
-            return false;
-
-        try {
-            CoordTrig from = arr.get(fromIndex);
-            CoordTrig to = arr.get(toIndex);
-            if (from == null || to == null)
-                return false;
-
-            g.setNode(fromIndex, from.lat, from.lon);
-            g.setNode(toIndex, to.lat, to.lon);
-            return addEdge(from.lat, from.lon, to.lat, to.lon, fromIndex, toIndex, flags, callback);
-        } catch (Exception ex) {
-            throw new RuntimeException("Problem to add edge! with node ids " + fromIndex + "->" + toIndex
-                    + " vs. osm ids:" + nodeIdFrom + "->" + nodeIdTo, ex);
-        }
+    public int addEdge(TLongList nodes, int flags) {
+        throw new UnsupportedOperationException("todo");
+//        int fromIndex = (int) osmIdToIndexMap.get(nodeIdFrom);
+//        int toIndex = (int) osmIdToIndexMap.get(nodeIdTo);
+//        if (fromIndex == osmIdToIndexMap.getNoEntryValue() || toIndex == osmIdToIndexMap.getNoEntryValue())
+//            return false;
+//
+//        try {
+//            CoordTrig from = arr.get(fromIndex);
+//            CoordTrig to = arr.get(toIndex);
+//            if (from == null || to == null)
+//                return false;
+//
+//            g.setNode(fromIndex, from.lat, from.lon);
+//            g.setNode(toIndex, to.lat, to.lon);
+//            return addEdge(from.lat, from.lon, to.lat, to.lon, fromIndex, toIndex, flags, callback);
+//        } catch (Exception ex) {
+//            throw new RuntimeException("Problem to add edge! with node ids " + fromIndex + "->" + toIndex
+//                    + " vs. osm ids:" + nodeIdFrom + "->" + nodeIdTo, ex);
+//        }
     }
 
     @Override