fixed code (list->queue) and tests after commit for graphhopper#916

Author: karussell

core/src/test/java/com/graphhopper/storage/GraphHopperStorageWithTurnCostsTest.java

@@ -93,7 +93,7 @@ public void testEnsureCapacity() throws IOException {
         graph.create(100); // 100 is the minimum size
 
         // assert that turnCostStorage can hold 104 turn cost entries at the beginning
-        assertEquals(104, turnCostStorage.getCapacity() / 16);
+        assertEquals(128, turnCostStorage.getCapacity());
 
         Random r = new Random();
 

graphhopper.sh

@@ -174,9 +174,7 @@ NAME="${FILE%.*}"
 
 if [ "$FILE" == "-" ]; then
    OSM_FILE=
-elif [ ${FILE: -4} == ".osm" ]; then
-   OSM_FILE="$FILE"
-elif [ ${FILE: -4} == ".pbf" ]; then
+elif [ ${FILE: -4} == ".osm" ] || [ ${FILE: -4} == ".xml" ] || [ ${FILE: -4} == ".pbf" ]; then
    OSM_FILE="$FILE"
 elif [ ${FILE: -7} == ".osm.gz" ]; then
    OSM_FILE="$FILE"

reader-osm/src/main/java/com/graphhopper/reader/osm/OSMInputFile.java

@@ -54,7 +54,7 @@ public class OSMInputFile implements Sink, Closeable {
 
     public OSMInputFile(File file) throws IOException {
         bis = decode(file);
-        itemQueue = new LinkedBlockingQueue<ReaderElement>(50000);
+        itemQueue = new LinkedBlockingQueue<>(50_000);
     }
 
     public OSMInputFile open() throws XMLStreamException {
@@ -251,9 +251,10 @@ public void process(ReaderElement item) {
         } catch (InterruptedException ex) {
             throw new RuntimeException(ex);
         }
+    }
 
-        // throw exception if full
-        // itemQueue.add(item);
+    public int getUnprocessedElements() {
+        return itemQueue.size();
     }
 
     @Override

reader-osm/src/main/java/com/graphhopper/reader/osm/OSMReader.java

@@ -93,7 +93,7 @@ public class OSMReader implements DataReader {
     private long locations;
     private long skippedLocations;
     private EncodingManager encodingManager = null;
-    private int workerThreads = -1;
+    private int workerThreads = 2;
     // Using the correct Map<Long, Integer> is hard. We need a memory efficient and fast solution for big data sets!
     //
     // very slow: new SparseLongLongArray
@@ -176,7 +176,7 @@ void preProcess(File osmFile) {
                             prepareHighwayNode(wayNodes.get(index));
                         }
 
-                        if (++tmpWayCounter % 5000000 == 0) {
+                        if (++tmpWayCounter % 10_000_000 == 0) {
                             LOGGER.info(nf(tmpWayCounter) + " (preprocess), osmIdMap:" + nf(getNodeMap().getSize()) + " ("
                                     + getNodeMap().getMemoryUsage() + "MB) " + Helper.getMemInfo());
                         }
@@ -189,7 +189,7 @@ void preProcess(File osmFile) {
                     if (relation.hasTag("type", "restriction"))
                         prepareRestrictionRelation(relation);
 
-                    if (++tmpRelationCounter % 50000 == 0) {
+                    if (++tmpRelationCounter % 100_000 == 0) {
                         LOGGER.info(nf(tmpRelationCounter) + " (preprocess), osmWayMap:" + nf(getRelFlagsMap().size())
                                 + " " + Helper.getMemInfo());
                     }
@@ -248,7 +248,7 @@ boolean filterWay(ReaderWay item) {
     }
 
     /**
-     * Creates the edges and nodes files from the specified osm file.
+     * Creates the graph with edges and nodes from the specified osm file.
      */
     private void writeOsm2Graph(File osmFile) {
         int tmp = (int) Math.max(getNodeMap().getSize() / 50, 100);
@@ -290,11 +290,14 @@ private void writeOsm2Graph(File osmFile) {
                     default:
                         throw new IllegalStateException("Unknown type " + item.getType());
                 }
-                if (++counter % 100000000 == 0) {
+                if (++counter % 200_000_000 == 0) {
                     LOGGER.info(nf(counter) + ", locs:" + nf(locations) + " (" + skippedLocations + ") " + Helper.getMemInfo());
                 }
             }
 
+            if (in.getUnprocessedElements() > 0)
+                throw new IllegalStateException("Still unprocessed elements in reader queue " + in.getUnprocessedElements());
+
             // logger.info("storage nodes:" + storage.nodes() + " vs. graph nodes:" + storage.getGraph().nodes());
         } catch (Exception ex) {
             throw new RuntimeException("Couldn't process file " + osmFile + ", error: " + ex.getMessage(), ex);
@@ -304,7 +307,7 @@ private void writeOsm2Graph(File osmFile) {
 
         finishedReading();
         if (graph.getNodes() == 0)
-            throw new IllegalStateException("osm must not be empty. read " + counter + " lines and " + locations + " locations");
+            throw new RuntimeException("Graph after reading OSM must not be empty. Read " + counter + " items and " + locations + " locations");
     }
 
     /**

reader-osm/src/main/java/com/graphhopper/reader/osm/pbf/PbfDecoder.java

@@ -7,6 +7,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
+import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.locks.Condition;
 import java.util.concurrent.locks.Lock;
@@ -26,7 +27,7 @@ public class PbfDecoder implements Runnable {
     private final Sink sink;
     private final Lock lock;
     private final Condition dataWaitCondition;
-    private final List<PbfBlobResult> blobResults;
+    private final Queue<PbfBlobResult> blobResults;
 
     /**
      * Creates a new instance.
@@ -49,7 +50,8 @@ public PbfDecoder(PbfStreamSplitter streamSplitter, ExecutorService executorServ
         dataWaitCondition = lock.newCondition();
 
         // Create the queue of blobs being decoded.
-        blobResults = new ArrayList<>();
+        // TODO #916 blobResults = new ArrayBlockingQueue<>(1000);
+        blobResults = new LinkedList<>();
     }
 
     /**
@@ -75,7 +77,7 @@ private void signalUpdate() {
     private void sendResultsToSink(int targetQueueSize) {
         while (blobResults.size() > targetQueueSize) {
             // Get the next result from the queue and wait for it to complete.
-            PbfBlobResult blobResult = blobResults.remove(blobResults.size() - 1);
+            PbfBlobResult blobResult = blobResults.remove();
             while (!blobResult.isComplete()) {
                 // The thread hasn't finished processing yet so wait for an
                 // update from another thread before checking again.

reader-osm/src/test/java/com/graphhopper/reader/osm/OSMReaderTest.java

@@ -421,8 +421,8 @@ public void testRelation() {
         OSMReader reader = new OSMReader(ghStorage).
                 setEncodingManager(manager);
         ReaderRelation osmRel = new ReaderRelation(1);
-        osmRel.getMembers().add(new ReaderRelation.Member(ReaderRelation.WAY, 1, ""));
-        osmRel.getMembers().add(new ReaderRelation.Member(ReaderRelation.WAY, 2, ""));
+        osmRel.add(new ReaderRelation.Member(ReaderRelation.WAY, 1, ""));
+        osmRel.add(new ReaderRelation.Member(ReaderRelation.WAY, 2, ""));
 
         osmRel.setTag("route", "bicycle");
         osmRel.setTag("network", "lcn");
@@ -723,7 +723,7 @@ public void testPreferredLanguage() {
 
     @Test
     public void testDataDateWithinPBF() {
-        GraphHopper hopper = new GraphHopperTest(file6).importOrLoad();
+        GraphHopper hopper = new GraphHopperTest("test-osm6.pbf").importOrLoad();
         GraphHopperStorage graph = hopper.getGraphHopperStorage();
 
         assertEquals("2014-01-02T00:10:14Z", graph.getProperties().get("datareader.data.date"));
@@ -745,14 +745,15 @@ public void testCrossBoundary_issue667() {
         assertEquals(56, qr.getClosestEdge().getDistance() / 1000, 1);
     }
 
+    @Test
     public void testRoutingRequestFails_issue665() {
         GraphHopper hopper = new GraphHopperOSM()
-                .setDataReaderFile("src/test/resources/com/graphhopper/reader/" + file7)
+                .setDataReaderFile(getClass().getResource(file7).getFile())
                 .setEncodingManager(new EncodingManager("car,motorcycle"))
                 .setGraphHopperLocation(dir);
         hopper.getCHFactoryDecorator().setEnabled(false);
         hopper.importOrLoad();
-        GHRequest req = new GHRequest(48.97725592769741, 8.256896138191223, 48.978875552977684, 8.25486302375793).
+        GHRequest req = new GHRequest(48.977277, 8.256896, 48.978876, 8.254884).
                 setWeighting("curvature").
                 setVehicle("motorcycle");