Remove detail data from map-necessary data in an attempt to speed things up.

Author: swigder

generate_web_data.py

@@ -36,14 +36,22 @@ def get_num_fatalities(people):
 
 
 def convert_to_web_data(df):
-    items = []
+    geojson_items = []
+    items_details = {}
     for index, row in df.iterrows():
-        items.append(geojson.Feature(geometry=geojson.Point((row['LONGITUD'], row['LATITUDE'])), properties={
+        case_id = f'{row["CASEYEAR"]}-{row["STATE"]}-{row["ST_CASE"]}'
+
+        geojson_items.append(geojson.Feature(geometry=geojson.Point((row['LONGITUD'], row['LATITUDE'])), properties={
+            'id': case_id,
             'year': row['CASEYEAR'],
-            'case_id': row['ST_CASE'],
             'harm': get_category(row['Person']),
-            'injuries': get_injuries(row['Person']),
-            'num_vehicles': row['Vehicle'],
             'num_fatalities': get_num_fatalities(row['Person']),
         }))
-    return geojson.FeatureCollection(features=items)
+
+        items_details[case_id] = {
+            'year': row['CASEYEAR'],
+            'num_vehicles': row['Vehicle'],
+            'injuries': get_injuries(row['Person'])
+        }
+
+    return geojson.FeatureCollection(features=geojson_items), items_details

main.py

@@ -1,5 +1,6 @@
 import glob
 import json
+import math
 
 import pandas as pd
 import requests
@@ -88,22 +89,26 @@ def merge_data(year):
 def generate_web_data():
     all_pickles = glob.glob('data/df-*.pkl')
     dfs = []
-    for filename in all_pickles:
-        df = pd.read_pickle(filename)
+    for filename_geojson in all_pickles:
+        df = pd.read_pickle(filename_geojson)
         dfs.append(df)
     df = pd.concat(dfs, axis=0)
 
     latlong_interval = 2
 
     grouped = df.groupby(
         df[['LATITUDE', 'LONGITUD']].agg(
-            lambda ys: '_'.join([str(int(y / latlong_interval) * latlong_interval) for y in ys]), axis=1))
+            lambda ys: '_'.join([str(math.floor(y / latlong_interval) * latlong_interval) for y in ys]), axis=1))
     filenames = []
     for name, group in grouped:
-        filename = f'{DATA_BASE_DIR}/data-{name}.json'
-        with open(f'{WEB_BASE_DIR}/{filename}', 'w') as outfile:
-            json.dump(convert_to_web_data(group.reset_index()), outfile)
-        filenames.append(filename)
+        geojson, full = convert_to_web_data(group.reset_index())
+        filename_geojson = f'{DATA_BASE_DIR}/data-{name}.json'
+        with open(f'{WEB_BASE_DIR}/{filename_geojson}', 'w') as outfile:
+            json.dump(geojson, outfile)
+        filenames.append(filename_geojson)
+        filename_full = f'{DATA_BASE_DIR}/data-{name}-full.json'
+        with open(f'{WEB_BASE_DIR}/{filename_full}', 'w') as outfile:
+            json.dump(full, outfile)
 
     df = df.reset_index()
     metadata = {

web/data/data-18_-154.json

@@ -0,0 +1 @@
+{"2015-15-150015": {"year": 2015, "num_vehicles": 2, "injuries": [{"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Fatal Injury (K)", "person": "Passenger"}, {"severity": "Suspected Serious Injury (A)", "person": "Passenger"}, {"severity": "Fatal Injury (K)", "person": "Passenger"}, {"severity": "Suspected Serious Injury (A)", "person": "Driver"}]}, "2015-15-150024": {"year": 2015, "num_vehicles": 2, "injuries": [{"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}]}, "2015-15-150030": {"year": 2015, "num_vehicles": 3, "injuries": [{"severity": "No Apparent Injury (O)", "person": "Driver"}, {"severity": "No Apparent Injury (O)", "person": "Passenger"}, {"severity": "No Apparent Injury (O)", "person": "Passenger"}, {"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}, {"severity": "No Apparent Injury (O)", "person": "Passenger"}]}, "2015-15-150031": {"year": 2015, "num_vehicles": 3, "injuries": [{"severity": "Suspected Minor Injury (B)", "person": "Driver"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}, {"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Fatal Injury (K)", "person": "Passenger"}]}, "2015-15-150054": {"year": 2015, "num_vehicles": 2, "injuries": [{"severity": "Suspected Minor Injury (B)", "person": "Driver"}, {"severity": "Fatal Injury (K)", "person": "Driver"}]}, "2017-15-150005": {"year": 2017, "num_vehicles": 1, "injuries": [{"severity": "Fatal Injury (K)", "person": "Bicyclist"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}]}, "2017-15-150033": {"year": 2017, "num_vehicles": 2, "injuries": [{"severity": "No Apparent Injury (O)", "person": "Driver"}, {"severity": "No Apparent Injury (O)", "person": "Passenger"}, {"severity": "Fatal Injury (K)", "person": "Driver"}]}, "2017-15-150034": {"year": 2017, "num_vehicles": 2, "injuries": [{"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Fatal Injury (K)", "person": "Passenger"}, {"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Suspected Serious Injury (A)", "person": "Passenger"}]}, "2017-15-150056": {"year": 2017, "num_vehicles": 1, "injuries": [{"severity": "Fatal Injury (K)", "person": "Pedestrian"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}]}, "2017-15-150079": {"year": 2017, "num_vehicles": 1, "injuries": [{"severity": "Fatal Injury (K)", "person": "Bicyclist"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}]}, "2016-15-150033": {"year": 2016, "num_vehicles": 1, "injuries": [{"severity": "Fatal Injury (K)", "person": "Driver"}]}, "2012-15-150063": {"year": 2012, "num_vehicles": 2, "injuries": [{"severity": "Non-incapacitating Evident Injury (B)", "person": "Driver"}, {"severity": "Non-incapacitating Evident Injury (B)", "person": "Passenger"}, {"severity": "Fatal Injury (K)", "person": "Passenger"}, {"severity": "Non-incapacitating Evident Injury (B)", "person": "Passenger"}, {"severity": "Non-incapacitating Evident Injury (B)", "person": "Passenger"}, {"severity": "Possible Injury (C)", "person": "Driver"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}]}, "2012-15-150070": {"year": 2012, "num_vehicles": 2, "injuries": [{"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Possible Injury (C)", "person": "Driver"}]}, "2012-15-150071": {"year": 2012, "num_vehicles": 2, "injuries": [{"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Non-incapacitating Evident Injury (B)", "person": "Driver"}]}, "2010-15-150107": {"year": 2010, "num_vehicles": 2, "injuries": [{"severity": "No Injury (O)", "person": "Driver"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "No Injury (O)", "person": "Passenger"}, {"severity": "Fatal Injury (K)", "person": "Driver"}]}, "2018-15-150017": {"year": 2018, "num_vehicles": 1, "injuries": [{"severity": "Fatal Injury (K)", "person": "Pedestrian"}, {"severity": "No Apparent Injury (O)", "person": "Driver"}]}, "2019-15-150085": {"year": 2019, "num_vehicles": 2, "injuries": [{"severity": "Suspected Minor Injury (B)", "person": "Driver"}, {"severity": "Fatal Injury (K)", "person": "Driver"}, {"severity": "Suspected Serious Injury (A)", "person": "Passenger"}, {"severity": "Suspected Serious Injury (A)", "person": "Passenger"}, {"severity": "Possible Injury (C)", "person": "Passenger"}]}}

web/data/data-18_-156-full.json

@@ -0,0 +1 @@
+{"type": "FeatureCollection", "features": [{"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.022492, 19.689678]}, "properties": {"id": "2015-15-150015", "year": 2015, "harm": "car", "num_fatalities": 3}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.015267, 19.671328]}, "properties": {"id": "2015-15-150024", "year": 2015, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.017119, 19.69265]}, "properties": {"id": "2015-15-150030", "year": 2015, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.033044, 19.707397]}, "properties": {"id": "2015-15-150031", "year": 2015, "harm": "car", "num_fatalities": 2}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.024769, 19.725497]}, "properties": {"id": "2015-15-150054", "year": 2015, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.014117, 19.668933]}, "properties": {"id": "2017-15-150005", "year": 2017, "harm": "bike", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.017125, 19.692669]}, "properties": {"id": "2017-15-150033", "year": 2017, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.024847, 19.761394]}, "properties": {"id": "2017-15-150034", "year": 2017, "harm": "car", "num_fatalities": 3}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.018942, 19.723006]}, "properties": {"id": "2017-15-150056", "year": 2017, "harm": "ped", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.030381, 19.732747]}, "properties": {"id": "2017-15-150079", "year": 2017, "harm": "bike", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.01495, 19.670919]}, "properties": {"id": "2016-15-150033", "year": 2016, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.023958, 19.764183]}, "properties": {"id": "2012-15-150063", "year": 2012, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.031722, 19.724794]}, "properties": {"id": "2012-15-150070", "year": 2012, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.002928, 19.672931]}, "properties": {"id": "2012-15-150071", "year": 2012, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.015558, 19.671281]}, "properties": {"id": "2010-15-150107", "year": 2010, "harm": "car", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.019522, 19.678636]}, "properties": {"id": "2018-15-150017", "year": 2018, "harm": "ped", "num_fatalities": 1}}, {"type": "Feature", "geometry": {"type": "Point", "coordinates": [-156.017275, 19.776744]}, "properties": {"id": "2019-15-150085", "year": 2019, "harm": "car", "num_fatalities": 1}}]}