From e7f674548bf8f1bb7531d25d57106faf1b3d55bb Mon Sep 17 00:00:00 2001 From: jacob6838 Date: Tue, 30 Dec 2025 11:39:54 -0700 Subject: [PATCH 1/2] Finalizing wzdx field device experimental combination module --- README.md | 66 +- .../field_devices/field_device_hwy-159.json | 19 + .../field_devices/field_device_nowhere.json | 19 + .../icone_standard_hwy-159.json | 0 .../icone_standard_nowhere.json | 0 .../wzdx_combination_co-9.json | 0 .../wzdx_combination_hwy-159.json | 0 .../wzdx_combination_nowhere.json | 0 .../wzdx_combination_nowhere_2.json | 0 .../field_devices_experimental_test.py | 139 +++ .../icone_experimental_test.py | 130 --- tests/models/field_device_feed_test.py | 65 +- tests/raw_to_standard/icone_test.py | 97 -- .../standard_to_wzdx/icone_translator_test.py | 251 ------ .../{icone.py => field_devices.py} | 839 +++++++++--------- .../field_device_core_details.py | 21 +- .../field_device_feed/field_device_feature.py | 7 +- .../field_device_feed/field_device_type.py | 21 +- wzdx/raw_to_standard/icone.py | 371 -------- .../field_device_feed/icone_2025_12_29.json | 117 +++ wzdx/standard_to_wzdx/icone_translator.py | 510 ----------- wzdx/tools/cdot_geospatial_api.py | 2 +- 22 files changed, 834 insertions(+), 1840 deletions(-) create mode 100644 tests/data/experimental_combination/field_devices/field_device_hwy-159.json create mode 100644 tests/data/experimental_combination/field_devices/field_device_nowhere.json rename tests/data/experimental_combination/{icone => field_devices}/icone_standard_hwy-159.json (100%) rename tests/data/experimental_combination/{icone => field_devices}/icone_standard_nowhere.json (100%) rename tests/data/experimental_combination/{icone => field_devices}/wzdx_combination_co-9.json (100%) rename tests/data/experimental_combination/{icone => field_devices}/wzdx_combination_hwy-159.json (100%) rename tests/data/experimental_combination/{icone => field_devices}/wzdx_combination_nowhere.json (100%) rename tests/data/experimental_combination/{icone => field_devices}/wzdx_combination_nowhere_2.json (100%) create mode 100644 tests/experimental_combination/field_devices_experimental_test.py delete mode 100644 tests/experimental_combination/icone_experimental_test.py delete mode 100644 tests/raw_to_standard/icone_test.py delete mode 100644 tests/standard_to_wzdx/icone_translator_test.py rename wzdx/experimental_combination/{icone.py => field_devices.py} (50%) delete mode 100644 wzdx/raw_to_standard/icone.py create mode 100644 wzdx/sample_files/field_device_feed/icone_2025_12_29.json delete mode 100644 wzdx/standard_to_wzdx/icone_translator.py diff --git a/README.md b/README.md index 8504095b..56acabbe 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ The build package tar.gz file will be located in the dist folder. ## Running the Translators Locally -This set of CWZ and WZDx message translators is set up to be implemented in GCP with App Engines and Dataflows. It is also set up with raw, standard, and enhanced data feeds. This means that to take a raw icone document and generate a CWZ or WZDx message, the raw icone xml document must first be converted to 1 or multiple standard json messages (based on CDOT RTDH specification), and then each standard message may be converted into a single enhanced message. At this point, this data can be combined with other CWZ/WZDx messages, through the [combination scripts](wzdx/experimental_combination/) +This set of CWZ and WZDx message translators is set up to be implemented in GCP with App Engines and Dataflows. It is also set up with raw, standard, and enhanced data feeds. This means that to take a CDOT planned events document and generate a CWZ or WZDx message, the raw planned event document must first be converted to 1 or multiple standard json messages (based on CDOT RTDH specification), and then each standard message may be converted into a single enhanced message. At this point, this data can be combined with other CWZ/WZDx messages, through the [combination scripts](wzdx/experimental_combination/) ### Prerequisites @@ -58,13 +58,13 @@ Please set up the following environment variable for your local computer before Runtime Environment Variables: -| Name | Value | Description | -| :--------------------------- | :---------------------------------------------------------------------------------------------------: | ---------------------------------------------------: | -| contact_name | Heather Pickering-Hilgers | name of WZDx feed contact | -| contact_email | heather.pickeringhilgers@state.co.us | email of WZDx feed contact | -| publisher | CDOT | name of the organization issuing the WZDx feed | -| CDOT_GEOSPATIAL_API_BASE_URL | https://dtdapps.codot.gov/server/rest/services/LRS/Routes_withDEC/MapServer/exts/CdotLrsAccessRounded | GIS server endpoint used for geospatial api | -| NAMESPACE_UUID | 00000000-0000-0000-0000-000000000000 | UUID used to pseudo-randomly tag all UUIDs generated | +| Name | Value | Description | +| :--------------------------- | :-----------------------------------------------------------------------------------------------: | ---------------------------------------------------: | +| contact_name | Heather Pickering-Hilgers | name of WZDx feed contact | +| contact_email | heather.pickeringhilgers@state.co.us | email of WZDx feed contact | +| publisher | CDOT | name of the organization issuing the WZDx feed | +| CDOT_GEOSPATIAL_API_BASE_URL | https://dtdapps.codot.gov/server/rest/services/LRS/Routes_withDEC/MapServer/exts/LrsServerRounded | GIS server endpoint used for geospatial api | +| NAMESPACE_UUID | 00000000-0000-0000-0000-000000000000 | UUID used to pseudo-randomly tag all UUIDs generated | Example usage: for mac computer run the following script to initialize the environment variable: @@ -109,44 +109,6 @@ Example usage: python -m wzdx.standard_to_wzdx.planned_events_translator 'wzdx/sample_files/standard/planned_events/standard_planned_event_OpenTMS-Event20643308360_westbound.json' ``` -### Execution for iCone translator - -#### Raw to Standard Conversion - -``` -python -m wzdx.raw_to_standard.icone inputfile.json --outputDir outputDirectory -``` - -Example usage: - -``` -python -m wzdx.raw_to_standard.icone 'wzdx/sample_files/raw/icone/icone_ftp_20241107-235100.xml' -``` - -#### Standard to CWZ Conversion - -``` -python -m wzdx.standard_to_cwz.icone_translator inputfile.json --outputFile outputfile.geojson -``` - -Example usage: - -``` -python -m wzdx.standard_to_cwz.icone_translator 'wzdx/sample_files/standard/icone/standard_icone_U13632784_20241107235100_1731023924_unknown.json' -``` - -#### Standard to WZDx Conversion - -``` -python -m wzdx.standard_to_wzdx.icone_translator inputfile.json --outputFile outputfile.geojson -``` - -Example usage: - -``` -python -m wzdx.standard_to_wzdx.icone_translator 'wzdx/sample_files/standard/icone/standard_icone_U13632784_20241107235100_1731023924_unknown.json' -``` - ### Execution for NavJoy 568 translator This translator reads in a NavJoy 568 speed reduction form and translates it into a WZDx message. Many of the 568 messages cover 2 directions of traffic, and are thus expanded into 2 WZDx messages, one for each direction. @@ -191,19 +153,19 @@ python -m wzdx.standard_to_wzdx.navjoy_translator 'wzdx/sample_files/standard/na ### Combine WZDx Messages -These combination scripts take in a base WZDx message and an additional icone/navjoy WZDx or Geotab JSON message, and generate an enhanced WZDx message as output. +These combination scripts take in a base WZDx message and an additional field device/navjoy WZDx or Geotab JSON message, and generate an enhanced WZDx message as output. -### iCone +### Field Device Feed -Edit the files read in for iCone and WZDx messages in the main method, then run the combination script: +Insert names for the files read in for field device feed and WZDx messages in the below combination script and then run: ``` -python icone.py wzdxFile.geojson ./iconeDirectory --outputDir ./ --updateDates true +python -m wzdx.experimental_combination.field_devices wzdxFile.geojson deviceFeedFile.geojson --outputDir ./ --updateDates false ``` ### Navjoy 568 form -Edit the files read in for navjoy and WZDx messages in the main method, then run the combination script: +Insert names for the files read in for navjoy-568 and WZDx messages in the below combination script and then run: ``` python navjoy.py wzdxFile.geojson navjoyWzdxFile.geojson --outputDir ./ --updateDates true @@ -211,7 +173,7 @@ python navjoy.py wzdxFile.geojson navjoyWzdxFile.geojson --outputDir ./ --update ### Geotab Vehicle (ATMA) -Edit the files read in for geotab_avl and WZDx messages in the main method, then run the combination script: +Insert names for the files read in for geotab_avl and WZDx messages in the below combination script and then run: ``` python attenuator.py wzdxFile.geojson geotabFile.json --outputDir ./ --updateDates true diff --git a/tests/data/experimental_combination/field_devices/field_device_hwy-159.json b/tests/data/experimental_combination/field_devices/field_device_hwy-159.json new file mode 100644 index 00000000..58f5fba0 --- /dev/null +++ b/tests/data/experimental_combination/field_devices/field_device_hwy-159.json @@ -0,0 +1,19 @@ +{ + "id": "04435BD5-83C9-44C5-968A-7CF54C089530", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-105.43859934199997, 37.19252041900006] + }, + "properties": { + "core_details": { + "device_type": "arrow-board", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2022-07-27T22:00:00Z", + "has_automatic_location": true, + "description": "Roadwork - Caution" + }, + "pattern": "four-corners-flashing" + } +} diff --git a/tests/data/experimental_combination/field_devices/field_device_nowhere.json b/tests/data/experimental_combination/field_devices/field_device_nowhere.json new file mode 100644 index 00000000..e01ec30f --- /dev/null +++ b/tests/data/experimental_combination/field_devices/field_device_nowhere.json @@ -0,0 +1,19 @@ +{ + "id": "04435BD5-83C9-44C5-968A-7CF54C089530", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-106.94366455078125, 39.455546939606066] + }, + "properties": { + "core_details": { + "device_type": "arrow-board", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2022-07-27T20:00:00Z", + "has_automatic_location": true, + "description": "Roadwork - Caution" + }, + "pattern": "four-corners-flashing" + } +} diff --git a/tests/data/experimental_combination/icone/icone_standard_hwy-159.json b/tests/data/experimental_combination/field_devices/icone_standard_hwy-159.json similarity index 100% rename from tests/data/experimental_combination/icone/icone_standard_hwy-159.json rename to tests/data/experimental_combination/field_devices/icone_standard_hwy-159.json diff --git a/tests/data/experimental_combination/icone/icone_standard_nowhere.json b/tests/data/experimental_combination/field_devices/icone_standard_nowhere.json similarity index 100% rename from tests/data/experimental_combination/icone/icone_standard_nowhere.json rename to tests/data/experimental_combination/field_devices/icone_standard_nowhere.json diff --git a/tests/data/experimental_combination/icone/wzdx_combination_co-9.json b/tests/data/experimental_combination/field_devices/wzdx_combination_co-9.json similarity index 100% rename from tests/data/experimental_combination/icone/wzdx_combination_co-9.json rename to tests/data/experimental_combination/field_devices/wzdx_combination_co-9.json diff --git a/tests/data/experimental_combination/icone/wzdx_combination_hwy-159.json b/tests/data/experimental_combination/field_devices/wzdx_combination_hwy-159.json similarity index 100% rename from tests/data/experimental_combination/icone/wzdx_combination_hwy-159.json rename to tests/data/experimental_combination/field_devices/wzdx_combination_hwy-159.json diff --git a/tests/data/experimental_combination/icone/wzdx_combination_nowhere.json b/tests/data/experimental_combination/field_devices/wzdx_combination_nowhere.json similarity index 100% rename from tests/data/experimental_combination/icone/wzdx_combination_nowhere.json rename to tests/data/experimental_combination/field_devices/wzdx_combination_nowhere.json diff --git a/tests/data/experimental_combination/icone/wzdx_combination_nowhere_2.json b/tests/data/experimental_combination/field_devices/wzdx_combination_nowhere_2.json similarity index 100% rename from tests/data/experimental_combination/icone/wzdx_combination_nowhere_2.json rename to tests/data/experimental_combination/field_devices/wzdx_combination_nowhere_2.json diff --git a/tests/experimental_combination/field_devices_experimental_test.py b/tests/experimental_combination/field_devices_experimental_test.py new file mode 100644 index 00000000..f25e68e0 --- /dev/null +++ b/tests/experimental_combination/field_devices_experimental_test.py @@ -0,0 +1,139 @@ +from wzdx.experimental_combination import field_devices +from wzdx.models.field_device_feed.field_device_feature import FieldDeviceFeature +import json +import time_machine +import datetime + + +def test_get_direction_from_route_details(): + route_details = {"Direction": "a"} + expected = "a" + + actual = field_devices.get_direction_from_route_details(route_details) + + assert actual == expected + + +def test_get_direction(): + street = "I-25N" + coords = [ + [-106.07316970825195, 39.190971392168045], + [-106.07331991195677, 39.18659739731203], + ] + route_details = {"Direction": "northbound"} + + expected = "northbound" + actual = field_devices.get_direction(street, []) + assert actual == expected + + expected = "southbound" + actual = field_devices.get_direction("", coords) + assert actual == expected + + expected = "northbound" + actual = field_devices.get_direction("", [], route_details) + assert actual == expected + + expected = "northbound" + actual = field_devices.get_direction(street, coords, route_details) + assert actual == expected + + +def test_get_combined_events_valid(): + icone_msgs = [ + field_devices.pre_process_field_device_feature( + FieldDeviceFeature.model_validate_json( + open( + "./tests/data/experimental_combination/field_devices/field_device_hwy-159.json" + ).read() + ) + ) + ] + wzdx = [ + json.loads( + open( + "./tests/data/experimental_combination/field_devices/wzdx_combination_hwy-159.json" + ).read() + ) + ] + + with time_machine.travel( + datetime.datetime(2022, 7, 27, 22, 0, 0, 0, tzinfo=datetime.timezone.utc) + ): + expected = field_devices.get_combined_events(icone_msgs, wzdx) + assert len(expected) == 1 + + +def test_get_combined_events_invalid(): + icone_msgs = [ + field_devices.pre_process_field_device_feature( + FieldDeviceFeature.model_validate_json( + open( + "./tests/data/experimental_combination/field_devices/field_device_hwy-159.json" + ).read() + ) + ) + ] + wzdx = [ + json.loads( + open( + "./tests/data/experimental_combination/field_devices/wzdx_combination_nowhere.json" + ).read() + ) + ] + + with time_machine.travel( + datetime.datetime(2022, 7, 27, 20, 0, 0, 0, tzinfo=datetime.timezone.utc) + ): + expected = field_devices.get_combined_events(icone_msgs, wzdx) + assert len(expected) == 0 + + +def test_get_combined_events_invalid_different_routes(): + icone_msgs = [ + field_devices.pre_process_field_device_feature( + FieldDeviceFeature.model_validate_json( + open( + "./tests/data/experimental_combination/field_devices/field_device_hwy-159.json" + ).read() + ) + ) + ] + wzdx = [ + json.loads( + open( + "./tests/data/experimental_combination/field_devices/wzdx_combination_co-9.json" + ).read() + ) + ] + + with time_machine.travel( + datetime.datetime(2022, 7, 27, 20, 0, 0, 0, tzinfo=datetime.timezone.utc) + ): + expected = field_devices.get_combined_events(icone_msgs, wzdx) + assert len(expected) == 0 + + +def test_get_combined_events_invalid_no_routes(): + icone_msgs = [ + field_devices.pre_process_field_device_feature( + FieldDeviceFeature.model_validate_json( + open( + "./tests/data/experimental_combination/field_devices/field_device_nowhere.json" + ).read() + ) + ) + ] + wzdx = [ + json.loads( + open( + "./tests/data/experimental_combination/field_devices/wzdx_combination_nowhere_2.json" + ).read() + ) + ] + + with time_machine.travel( + datetime.datetime(2022, 7, 27, 20, 0, 0, 0, tzinfo=datetime.timezone.utc) + ): + expected = field_devices.get_combined_events(icone_msgs, wzdx) + assert len(expected) == 0 diff --git a/tests/experimental_combination/icone_experimental_test.py b/tests/experimental_combination/icone_experimental_test.py deleted file mode 100644 index 2cc3b2a1..00000000 --- a/tests/experimental_combination/icone_experimental_test.py +++ /dev/null @@ -1,130 +0,0 @@ -from wzdx.experimental_combination import icone -import json -import time_machine -import datetime - - -def test_get_direction_from_route_details(): - route_details = {"Direction": "a"} - expected = "a" - - actual = icone.get_direction_from_route_details(route_details) - - assert actual == expected - - -def test_get_direction(): - street = "I-25N" - coords = [ - [-106.07316970825195, 39.190971392168045], - [-106.07331991195677, 39.18659739731203], - ] - route_details = {"Direction": "northbound"} - - expected = "northbound" - actual = icone.get_direction(street, []) - assert actual == expected - - expected = "southbound" - actual = icone.get_direction("", coords) - assert actual == expected - - expected = "northbound" - actual = icone.get_direction("", [], route_details) - assert actual == expected - - expected = "northbound" - actual = icone.get_direction(street, coords, route_details) - assert actual == expected - - -def test_get_combined_events_valid(): - icone_msgs = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/icone_standard_hwy-159.json" - ).read() - ) - ] - wzdx = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/wzdx_combination_hwy-159.json" - ).read() - ) - ] - - with time_machine.travel( - datetime.datetime(2022, 7, 27, 22, 0, 0, 0, tzinfo=datetime.timezone.utc) - ): - expected = icone.get_combined_events(icone_msgs, wzdx) - assert len(expected) == 1 - - -def test_get_combined_events_invalid(): - icone_msgs = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/icone_standard_hwy-159.json" - ).read() - ) - ] - wzdx = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/wzdx_combination_nowhere.json" - ).read() - ) - ] - - with time_machine.travel( - datetime.datetime(2022, 7, 27, 20, 0, 0, 0, tzinfo=datetime.timezone.utc) - ): - expected = icone.get_combined_events(icone_msgs, wzdx) - assert len(expected) == 0 - - -def test_get_combined_events_invalid_different_routes(): - icone_msgs = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/icone_standard_hwy-159.json" - ).read() - ) - ] - wzdx = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/wzdx_combination_co-9.json" - ).read() - ) - ] - - with time_machine.travel( - datetime.datetime(2022, 7, 27, 20, 0, 0, 0, tzinfo=datetime.timezone.utc) - ): - expected = icone.get_combined_events(icone_msgs, wzdx) - assert len(expected) == 0 - - -def test_get_combined_events_invalid_no_routes(): - icone_msgs = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/icone_standard_nowhere.json" - ).read() - ) - ] - wzdx = [ - json.loads( - open( - "./tests/data/experimental_combination/icone/wzdx_combination_nowhere_2.json" - ).read() - ) - ] - - with time_machine.travel( - datetime.datetime(2022, 7, 27, 20, 0, 0, 0, tzinfo=datetime.timezone.utc) - ): - expected = icone.get_combined_events(icone_msgs, wzdx) - assert len(expected) == 0 diff --git a/tests/models/field_device_feed_test.py b/tests/models/field_device_feed_test.py index fcc14e10..2f560a78 100644 --- a/tests/models/field_device_feed_test.py +++ b/tests/models/field_device_feed_test.py @@ -1,6 +1,7 @@ from pydantic import TypeAdapter from wzdx.models.field_device_feed.device_feed import DeviceFeed + def test_deserialization(): # Deserialize from JSON string json_string = """ @@ -117,4 +118,66 @@ def test_deserialization(): ) print("JSON Output", json_output) - assert False + + expected_object = { + "feed_info": { + "update_date": "2025-12-18T20:34:51.150000Z", + "publisher": "iCone Products LLC", + "contact_email": "support@iconeproducts.com", + "version": "4.2", + "data_sources": [ + { + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "update_date": "2025-12-18T20:34:51.150000Z", + "organization_name": "iCone Products LLC", + "contact_email": "support@iconeproducts.com", + } + ], + }, + "type": "FeatureCollection", + "features": [ + { + "id": "E595E296-B1DE-4911-9454-1F2D54AC2EBD", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-104.7752009, 39.4983242], + }, + "properties": { + "core_details": { + "device_type": "arrow-board", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2025-12-18T20:30:27Z", + "has_automatic_location": True, + "description": "Roadwork - Caution", + }, + "pattern": "four-corners-flashing", + }, + }, + { + "id": "0E1E3B5B-D06E-4390-ABB3-C89091E246F0", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-106.0079266, 39.6531149], + }, + "properties": { + "core_details": { + "device_type": "location-marker", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2025-12-18T20:19:13Z", + "has_automatic_location": True, + "description": "Roadwork Active", + }, + "marked_locations": [{"type": "work-truck-with-lights-flashing"}], + }, + }, + ], + } + + assert ( + device_feed_list[0].model_dump(by_alias=True, exclude_none=True, mode="json") + == expected_object + ) diff --git a/tests/raw_to_standard/icone_test.py b/tests/raw_to_standard/icone_test.py deleted file mode 100644 index a1165758..00000000 --- a/tests/raw_to_standard/icone_test.py +++ /dev/null @@ -1,97 +0,0 @@ -from wzdx.raw_to_standard import icone -from tests.data.raw_to_standard import icone_test_expected_results as expected_results -import uuid -import json -import argparse -from unittest.mock import Mock, patch -import time_machine -from datetime import datetime - - -@patch.object(argparse, "ArgumentParser") -def test_parse_navjoy_arguments(argparse_mock): - navjoyFile, outputFile = icone.parse_rtdh_arguments() - assert navjoyFile is not None and outputFile is not None - - -# --------------------------------------------------------------------------------Unit test for parse_icone_polyline function-------------------------------------------------------------------------------- -def test_parse_icone_polyline_valid_data(): - test_polyline = "34.8380671,-114.1450650,34.8380671,-114.1450650" - test_coordinates = icone.parse_icone_polyline(test_polyline) - valid_coordinates = [[-114.145065, 34.8380671], [-114.145065, 34.8380671]] - assert test_coordinates == valid_coordinates - - -# --------------------------------------------------------------------------------Unit test for validate_incident function-------------------------------------------------------------------------------- -def test_validate_incident_valid_data(): - test_valid_output = { - "@id": "U13631595_202012160845", - "updatetime": "2020-12-16T17:18:00Z", - "starttime": "2020-12-07T14:18:00Z", - "description": "Road constructions are going on", - "creationtime": "2020-12-13T14:18:00Z", - "location": { - "polyline": "34.8380671,-114.1450650,34.8380671,-114.1450650", - "street": "I-70 N", - }, - } - assert icone.validate_incident(test_valid_output) is True - - -def test_validate_incident_missing_required_field_description(): - test_valid_output = { - "@id": "U13631595_202012160845", - "updatetime": "2020-12-16T17:18:00Z", - "starttime": "2020-12-07T14:18:00Z", - "creationtime": "2020-12-13T14:18:00Z", - "location": {"polyline": "34.8380671,-114.1450650,34.8380671,-114.1450650"}, - } - assert icone.validate_incident(test_valid_output) is False - - -def test_validate_incident_invalid_start_time(): - test_valid_output = { - "@id": "U13631595_202012160845", - "updatetime": "2020-12-16T17:18:00Z", - "starttime": "dsafsaf", - "description": "Road constructions are going on", - "creationtime": "2020-12-13T14:18:00Z", - "location": { - "polyline": "34.8380671,-114.1450650,34.8380671,-114.1450650", - "street": "I-70 N", - }, - } - assert icone.validate_incident(test_valid_output) is False - - -def test_validate_incident_invalid(): - test_valid_output = "invalid output" - assert icone.validate_incident(test_valid_output) is False - - -def test_validate_incident_no_data(): - test_valid_output = None - assert icone.validate_incident(test_valid_output) is False - - -@patch("uuid.uuid4") -def test_generate_standard_messages_from_string(mockuuid): - uuid.uuid4 = Mock() - uuid.uuid4.side_effect = ["we234de", "23wsg54h"] - - with time_machine.travel(datetime(2021, 4, 13, 0, 0, 0, 0)): - actual_standard = json.loads( - json.dumps( - icone.generate_standard_messages_from_string( - expected_results.test_generate_standard_messages_from_string_input - ) - ) - ) - - # Removing rtdh_timestamp because mocking it was not working. Kept having incorrect decimal values, weird floating point errors? - for i in actual_standard: - del i["rtdh_timestamp"] - expected = expected_results.test_generate_standard_messages_from_string_expected - for i in expected: - del i["rtdh_timestamp"] - assert actual_standard == expected diff --git a/tests/standard_to_wzdx/icone_translator_test.py b/tests/standard_to_wzdx/icone_translator_test.py deleted file mode 100644 index 6521563f..00000000 --- a/tests/standard_to_wzdx/icone_translator_test.py +++ /dev/null @@ -1,251 +0,0 @@ -import os -import uuid -import argparse -from datetime import datetime -from unittest.mock import Mock, patch - -import time_machine -import xmltodict -from wzdx.standard_to_wzdx import icone_translator -from wzdx.tools import wzdx_translator - -from tests.data.standard_to_wzdx import icone_translator_data - - -@patch.object(argparse, "ArgumentParser") -def test_parse_planned_events_arguments(argparse_mock): - iconeFile, outputFile = icone_translator.parse_icone_arguments() - assert iconeFile is not None and outputFile is not None - - -# --------------------------------------------------------------------------------Unit test for get_vehicle_impact function-------------------------------------------------------------------------------- -def test_get_vehicle_impact_some_lanes_closed(): - test_description = "Roadwork - Lane Closed, MERGE LEFT [Trafficade, iCone]" - test_vehicle_impact = icone_translator.get_vehicle_impact(test_description) - expected_vehicle_impact = "some-lanes-closed" - assert test_vehicle_impact == expected_vehicle_impact - - -def test_get_vehicle_impact_all_lanes_open(): - test_description = ( - "Road Ranger Emergency Personnel On-Scene. Move over - Caution [DBi, iCone]" - ) - test_vehicle_impact = icone_translator.get_vehicle_impact(test_description) - expected_vehicle_impact = "all-lanes-open" - assert test_vehicle_impact == expected_vehicle_impact - - -# --------------------------------------------------------------------------------Unit test for wzdx_creator function-------------------------------------------------------------------------------- -def test_wzdx_creator_empty_icone_object(): - icone_obj = None - test_wzdx = icone_translator.wzdx_creator(icone_obj) - assert test_wzdx is None - - -def test_wzdx_creator_invalid_info_object(): - icone_obj = { - "rtdh_timestamp": 1638894543.6077065, - "rtdh_message_id": "b33b2851-0475-4c8c-8bb7-c63e449190a9", - "event": { - "type": "CONSTRUCTION", - "source": { - "id": "1245", - "creation_timestamp": 1572916940000, - "last_updated_timestamp": 1636142163000, - }, - "geometry": [ - [-84.1238971, 37.1686478], - [-84.1238971, 37.1686478], - [-84.145861, 37.1913], - [-84.145861, 37.1913], - [-84.157105, 37.201197], - [-84.167033, 37.206079], - [-84.204074, 37.21931], - ], - "header": { - "description": "19-1245: Roadwork between MP 40 and MP 48", - "start_timestamp": 1623183301000, - "end_timestamp": "None", - }, - "detail": { - "road_name": "I-75 N", - "road_number": "I-75 N", - "direction": "northbound", - }, - "additional_info": {}, - }, - } - - test_invalid_info_object = { - "contact_name": "Heather Pickering-Hilgers", - "contact_email": "heather.pickeringhilgers@state.co.us", - "publisher": "iCone", - } - - test_wzdx = icone_translator.wzdx_creator(icone_obj, test_invalid_info_object) - assert test_wzdx is None - - -@patch.dict( - os.environ, - { - "contact_name": "Heather Pickering-Hilgers", - "contact_email": "heather.pickeringhilgers@state.co.us", - "publisher": "CDOT", - }, -) -@patch("uuid.uuid4") -def test_wzdx_creator(mockUuid): - uuid.uuid4 = Mock() - uuid.uuid4.side_effect = "we234de" - - icone_obj = { - "rtdh_timestamp": 1638894543.6077065, - "rtdh_message_id": "b33b2851-0475-4c8c-8bb7-c63e449190a9", - "event": { - "type": "CONSTRUCTION", - "source": { - "id": "1245", - "creation_timestamp": 1572916940000, - "last_updated_timestamp": 1636142163000, - }, - "geometry": [ - [-84.1238971, 37.1686478], - [-84.1238971, 37.1686478], - [-84.145861, 37.1913], - [-84.145861, 37.1913], - [-84.157105, 37.201197], - [-84.167033, 37.206079], - [-84.204074, 37.21931], - ], - "header": { - "description": "19-1245: Roadwork between MP 40 and MP 48", - "start_timestamp": 1623183301000, - "end_timestamp": 1623186301000, - }, - "detail": { - "road_name": "I-75 N", - "road_number": "I-75 N", - "direction": "northbound", - }, - }, - } - - expected_wzdx = icone_translator_data.test_wzdx_creator_expected - - with time_machine.travel(datetime(2021, 4, 13, 0, 0, 0)): - test_wzdx = icone_translator.wzdx_creator(icone_obj) - test_wzdx = wzdx_translator.remove_unnecessary_fields(test_wzdx) - assert expected_wzdx == test_wzdx - - -# --------------------------------------------------------------------------------unit test for parse_icone_sensor function-------------------------------------------------------------------------------- -def test_parse_icone_sensor(): - valid_description = { - "type": "iCone", - "id": "#4", - "location": [41.3883260, -81.9707500], - "radar": { - "average_speed": 63.52, - "std_dev_speed": 7.32, - "timestamp": "2020-08-21T15:55:00Z", - }, - } - test_sensor = icone_translator_data.test_parse_icone_sensor_test_sensor_1 - output_description = icone_translator.parse_icone_sensor(test_sensor) - assert output_description == valid_description - - valid_description = { - "type": "iCone", - "id": "#4", - "location": [41.3883260, -81.9707500], - "radar": { - "average_speed": 64.32, - "std_dev_speed": 6.11, - "timestamp": "2020-08-21T15:40:00Z", - }, - } - test_sensor = icone_translator_data.test_parse_icone_sensor_test_sensor_2 - output_description = icone_translator.parse_icone_sensor(test_sensor) - assert output_description == valid_description - - -# --------------------------------------------------------------------------------unit test for parse_pcms_sensor function-------------------------------------------------------------------------------- -def test_parse_pcms_sensor(): - valid_description = { - "type": "PCMS", - "id": "I-75 NB - MP 48.3", - "timestamp": "2020-08-21T15:48:25Z", - "location": [37.2182000, -84.2027000], - "messages": [" ROADWORK / 5 MILES / AHEAD"], - } - test_sensor = { - "@type": "PCMS", - "@id": "I-75 NB - MP 48.3", - "@latitude": "37.2182000", - "@longitude": "-84.2027000", - "message": { - "@verified": "2020-08-21T15:48:25Z", - "@latitude": "37.2178100", - "@longitude": "-84.2024390", - "@text": " ROADWORK / 5 MILES / AHEAD", - }, - } - output_description = icone_translator.parse_pcms_sensor(test_sensor) - assert output_description == valid_description - - -# --------------------------------------------------------------------------------unit test for create_description function-------------------------------------------------------------------------------- -def test_create_description(): - test_description = icone_translator_data.test_create_description_description - test_incident = """ - 2019-11-05T01:32:44Z - 2020-08-21T15:52:02Z - CONSTRUCTION - 19-1245: Roadwork between MP 48 and MP 40 - - I-75 S - ONE_DIRECTION - 37.2066700,-84.1691290,37.2012230,-84.1573460,37.1858150,-84.1404820,37.1663450,-84.1214250,37.1478080,-84.1115880 - - 2019-11-22T23:02:21Z - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - """ - output_description = icone_translator.create_description( - xmltodict.parse(test_incident)["incident"] - ) - assert output_description == test_description diff --git a/wzdx/experimental_combination/icone.py b/wzdx/experimental_combination/field_devices.py similarity index 50% rename from wzdx/experimental_combination/icone.py rename to wzdx/experimental_combination/field_devices.py index c43d8146..ec3053c7 100644 --- a/wzdx/experimental_combination/icone.py +++ b/wzdx/experimental_combination/field_devices.py @@ -1,406 +1,433 @@ -import argparse -import json -import logging -from datetime import datetime, timedelta -import glob -from typing import Literal - -from ..tools import combination, wzdx_translator, geospatial_tools, date_tools - -PROGRAM_NAME = "ExperimentalCombinationIcone" -PROGRAM_VERSION = "1.0" - -ISO_8601_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ" -START_TIME_THRESHOLD_MILLISECONDS = 1000 * 60 * 60 * 24 * 31 # 31 days -END_TIME_THRESHOLD_MILLISECONDS = 1000 * 60 * 60 * 24 * 31 # 31 days - - -def main(outputPath="./tests/data/output/wzdx_icone_combined.json"): - wzdxFile, iconeDirectory, output_dir, updateDates = parse_rtdh_arguments() - wzdx = json.loads(open(wzdxFile, "r").read()) - icone = [ - json.loads(open(f_name).read()) - for f_name in glob.glob(f"${iconeDirectory}/*.json") - ] - outputPath = output_dir + "/wzdx_experimental.geojson" - if updateDates == "true": - for i in icone: - i["rtdh_timestamp"] = date_tools.get_current_ts_millis() / 1000 - i["event"]["header"]["start_timestamp"] = ( - date_tools.get_iso_string_from_datetime( - datetime.now() - timedelta(days=1) - ) - ) - i["features"][0]["properties"]["end_date"] = ( - date_tools.get_iso_string_from_datetime( - datetime.now() + timedelta(hours=4) - ) - ) - wzdx[0]["features"][0]["properties"]["start_date"] = ( - date_tools.get_iso_string_from_datetime(datetime.now() - timedelta(days=2)) - ) - wzdx[0]["features"][0]["properties"]["end_date"] = ( - date_tools.get_iso_string_from_datetime(datetime.now() - timedelta(hours=2)) - ) - - combined_events = get_combined_events(icone, wzdx) - - if len(combined_events) == 0: - print( - "No overlapping events found between WZDx and iCone data. See logs for more information." - ) - else: - with open(outputPath, "w+") as f: - f.write(json.dumps(combined_events, indent=2)) - print(f"Successfully wrote combined WZDx file to: {outputPath}") - - -# parse script command line arguments -def parse_rtdh_arguments() -> tuple[str, str, str, str]: - """Parse command line arguments for icone WZDx combination script - - Returns: - str: WZDx file path - str: iCone directory path - str: output directory path - str: updateDates flag (true/false) - """ - parser = argparse.ArgumentParser( - description="Combine WZDx and iCone arrow board data" - ) - parser.add_argument( - "--version", action="version", version=f"{PROGRAM_NAME} {PROGRAM_VERSION}" - ) - parser.add_argument("wzdxFile", help="planned event file path") - parser.add_argument("iconeJsonDirectory", help="planned event file path") - parser.add_argument( - "--outputDir", required=False, default="./", help="output directory" - ) - parser.add_argument( - "--updateDates", - required=False, - default="false", - help="Boolean (true/false), Update dates to the current date to pass time filter", - ) - - args = parser.parse_args() - return args.wzdxFile, args.iconeDirectory, args.outputDir, args.updateDates - - -def get_direction_from_route_details(route_details: dict) -> str: - """Get direction from GIS route details - - Args: - route_details (dict): GIS route details - - Returns: - str: direction | "unknown" - """ - return route_details.get("Direction") - - -def get_direction( - street: str, coords: list[list[float]], route_details: dict = None -) -> str: - """Get road direction from street name, coordinates, or route details - - Args: - street (str): Street name, like "I-25 NB" - coords (list[list[float]]): List of coordinates, to pull direction from - route_details (dict, optional): GIS route details, defaults to None - - Returns: - Literal['unknown', 'eastbound', 'westbound', 'northbound', 'southbound']: Road direction - """ - direction = wzdx_translator.parse_direction_from_street_name(street) - if not direction and route_details: - direction = get_direction_from_route_details(route_details) - if not direction: - direction = geospatial_tools.get_road_direction_from_coordinates(coords) - return direction - - -def get_combined_events( - icone_standard_msgs: list[dict], wzdx_msgs: list[dict] -) -> list[dict]: - """Combine/integrate overlapping iCone messages into WZDx messages - - Args: - icone_standard_msgs (list[dict]): iCone RTDH standard messages - wzdx_msgs (list[dict]): WZDx messages - - Returns: - list[dict]: Combined WZDx messages - """ - combined_events = [] - - filtered_wzdx_msgs = wzdx_translator.filter_wzdx_by_event_status( - wzdx_msgs, ["pending", "completed_recently"] - ) - - for i in identify_overlapping_features_icone( - icone_standard_msgs, filtered_wzdx_msgs - ): - icone_msg, wzdx_msg = i - event_status = wzdx_translator.get_event_status(wzdx_msg["features"][0]) - if event_status in ["pending", "completed_recently"]: - wzdx = combine_icone_with_wzdx(icone_msg, wzdx_msg, event_status) - if wzdx: - combined_events.append(wzdx) - return combined_events - - -def combine_icone_with_wzdx( - icone_standard: dict, - wzdx_wzdx: dict, - event_status: Literal["active", "pending", "planned", "completed"], -) -> dict: - """Combine iCone message with WZDx message - - Args: - icone_standard (dict): iCone RTDH standard message - wzdx_wzdx (dict): WZDx message - event_status (Literal["active", "pending", "planned", "completed"]): WZDx event status - - Returns: - dict: Combined WZDx message - """ - combined_event = wzdx_wzdx - updated = False - - if event_status == "pending": - combined_event["features"][0]["properties"]["start_date"] = ( - date_tools.get_iso_string_from_unix( - icone_standard["event"]["header"]["start_timestamp"] - ) - ) - updated = True - elif event_status == "completed_recently": - combined_event["features"][0]["properties"]["end_date"] = ( - date_tools.get_iso_string_from_unix( - icone_standard["event"]["header"]["start_timestamp"] + 60 * 60 - ) - ) - combined_event["features"][0]["properties"]["core_details"]["description"] += ( - " " + icone_standard["event"]["header"]["description"] - ) - updated = True - logging.debug("Updated: " + str(updated)) - if updated: - update_date = date_tools.get_iso_string_from_datetime(datetime.now()) - combined_event["features"][0]["properties"]["core_details"][ - "update_date" - ] = update_date - combined_event["feed_info"]["data_sources"][0]["update_date"] = update_date - - combined_event["features"][0]["properties"][ - "experimental_source_type" - ] = "icone" - combined_event["features"][0]["properties"]["experimental_source_id"] = ( - icone_standard["rtdh_message_id"] - ) - combined_event["features"][0]["properties"]["icone_id"] = icone_standard[ - "event" - ]["source"]["id"] - combined_event["features"][0]["properties"]["icone_message"] = icone_standard - - for i in ["route_details_start", "route_details_end"]: - if i in combined_event: - del combined_event[i] - return combined_event - else: - return None - - -def get_route_details_for_icone(coordinates: list[list[float]]) -> tuple[dict, dict]: - """Get route details for iCone message - - Args: - coordinates (list[list[float]]): List of coordinates - - Returns: - tuple[dict, dict]: Route details for start and end coordinates - """ - route_details_start = combination.get_route_details( - coordinates[0][1], coordinates[0][0] - ) - - if len(coordinates) == 1 or ( - len(coordinates) == 2 and coordinates[0] == coordinates[1] - ): - route_details_end = None - else: - route_details_end = combination.get_route_details( - coordinates[-1][1], coordinates[-1][0] - ) - - return route_details_start, route_details_end - - -def validate_directionality_wzdx_icone(icone: dict, wzdx: dict) -> bool: - """Validate directionality between iCone and WZDx messages - - Args: - icone (dict): iCone RTDH standard message - wzdx (dict): WZDx message - - Returns: - bool: Directionality match - """ - direction_1 = icone["event"]["detail"]["direction"] - direction_2 = wzdx["features"][0]["properties"]["core_details"]["direction"] - - return direction_1 in [None, "unknown", "undefined"] or direction_1 == direction_2 - - -# Filter out iCone and WZDx messages which are not within the time interval -def validate_dates(icone: dict, wzdx: dict) -> bool: - """Validate date overlap between iCone and WZDx messages - - Args: - icone (dict): iCone RTDH standard message - wzdx (dict): WZDx message - - Returns: - bool: Date match - """ - wzdx_start_date = date_tools.get_unix_from_iso_string( - wzdx["features"][0]["properties"]["start_date"] - ) - wzdx_end_date = date_tools.get_unix_from_iso_string( - wzdx["features"][0]["properties"]["end_date"] - ) - icone_start_date = icone["event"]["header"]["start_timestamp"] * 1000 - icone_end_date = ( - icone["event"]["header"]["end_timestamp"] * 100 - if icone["event"]["header"]["end_timestamp"] - else None - ) - return wzdx_start_date - icone_start_date < START_TIME_THRESHOLD_MILLISECONDS or ( - icone_end_date is None - or icone_end_date - wzdx_end_date < END_TIME_THRESHOLD_MILLISECONDS - ) - - -def identify_overlapping_features_icone( - icone_standard_msgs: list[dict], wzdx_msgs: list[dict] -) -> list[tuple[dict, dict]]: - """Identify overlapping iCone and WZDx messages - - Args: - icone_standard_msgs (list[dict]): iCone RTDH standard messages - wzdx_msgs (list[dict]): WZDx messages - - Returns: - list[tuple[dict, dict]]: Overlapping iCone and WZDx messages - """ - icone_routes = {} - wzdx_routes = {} - matching_routes = [] - - # Step 1: Add route info to iCone messages - for icone in icone_standard_msgs: - icone["route_details_start"] = ( - icone["event"].get("additional_info", {}).get("route_details_start") - ) - icone["route_details_end"] = ( - icone["event"].get("additional_info", {}).get("route_details_end") - ) - route_details_start, route_details_end = get_route_details_for_icone( - icone["event"]["geometry"] - ) - - if not route_details_start: - logging.debug( - f"Invalid route details for iCone: {icone['event']['source']['id']}" - ) - continue - icone["route_details_start"] = route_details_start - icone["route_details_end"] = route_details_end - - if ( - icone["route_details_end"] - and route_details_start["Route"] != route_details_end["Route"] - ): - logging.debug( - f"Mismatched routes for iCone feature {icone['event']['source']['id']}" - ) - continue - - if route_details_start["Route"] in icone_routes: - icone_routes[route_details_start["Route"]].append(icone) - else: - icone_routes[route_details_start["Route"]] = [icone] - - # Step 2: Add route info to WZDx messages - for wzdx in wzdx_msgs: - wzdx["route_details_start"] = wzdx["features"][0]["properties"].get( - "route_details_start" - ) - wzdx["route_details_end"] = wzdx["features"][0]["properties"].get( - "route_details_end" - ) - if ( - wzdx.get("route_details_start") - and not wzdx.get("route_details_end") - or not wzdx.get("route_details_start") - and wzdx.get("route_details_end") - ): - logging.debug( - f"Missing route_details for WZDx object: {wzdx['features'][0]['id']}" - ) - continue - if not wzdx.get("route_details_start") and not wzdx.get("route_details_end"): - route_details_start, route_details_end = ( - combination.get_route_details_for_wzdx(wzdx["features"][0]) - ) - - if not route_details_start or not route_details_end: - logging.debug(f"Missing WZDx route details {wzdx['features'][0]['id']}") - continue - wzdx["route_details_start"] = route_details_start - wzdx["route_details_end"] = route_details_end - else: - route_details_start = wzdx["route_details_start"] - route_details_end = wzdx["route_details_end"] - - if route_details_start["Route"] != route_details_end["Route"]: - logging.debug(f"Mismatched routes for feature {wzdx['features'][0]['id']}") - continue - - logging.debug( - "Route details: " + str(route_details_start) + str(route_details_end) - ) - - if route_details_start["Route"] in wzdx_routes: - wzdx_routes[route_details_start["Route"]].append(wzdx) - else: - wzdx_routes[route_details_start["Route"]] = [wzdx] - - if not icone_routes: - logging.debug("No routes found for icone") - return [] - if not wzdx_routes: - logging.debug("No routes found for wzdx") - return [] - - # Step 3: Identify overlapping events - for wzdx_route_id, wzdx_matched_msgs in wzdx_routes.items(): - matching_icone_routes = icone_routes.get(wzdx_route_id, []) - - for match_icone in matching_icone_routes: - for match_wzdx in wzdx_matched_msgs: - # require routes to overlap, directionality to match, and dates to match - if ( - combination.does_route_overlap(match_icone, match_wzdx) - and validate_directionality_wzdx_icone(match_icone, match_wzdx) - and validate_dates(match_icone, match_wzdx) - ): - - matching_routes.append((match_icone, match_wzdx)) - - return matching_routes - - -if __name__ == "__main__": - main() +import argparse +import json +import logging +from datetime import datetime, timedelta, timezone +from typing import Literal +from wzdx.models.field_device_feed.device_feed import DeviceFeed +from wzdx.models.field_device_feed.field_device_feature import FieldDeviceFeature +from wzdx.models.geometry.geojson_geometry import GeoJsonGeometry + +from ..tools import combination, wzdx_translator, geospatial_tools, date_tools + +PROGRAM_NAME = "ExperimentalCombinationFieldDevices" +PROGRAM_VERSION = "1.0" + +ISO_8601_FORMAT_STRING = "%Y-%m-%dT%H:%M:%SZ" +START_TIME_THRESHOLD_MILLISECONDS = 1000 * 60 * 60 * 24 * 31 # 31 days +END_TIME_THRESHOLD_MILLISECONDS = 1000 * 60 * 60 * 24 * 31 # 31 days + + +def main(outputPath="./tests/data/output/wzdx_field_devices_combined.json"): + wzdxFile, fieldDevicesFile, output_dir, updateDates = parse_rtdh_arguments() + wzdx = json.loads(open(wzdxFile, "r").read()) + field_device_collection: DeviceFeed = DeviceFeed.model_validate_json( + open(fieldDevicesFile).read() + ) + outputPath = output_dir + "/wzdx_experimental.geojson" + if updateDates == "true": + for i in field_device_collection.features: + i.properties.core_details.update_date = datetime.now(timezone.utc) + wzdx[0]["features"][0]["properties"]["start_date"] = ( + date_tools.get_iso_string_from_datetime( + datetime.now(timezone.utc) - timedelta(days=3) + ) + ) + wzdx[0]["features"][0]["properties"]["end_date"] = ( + date_tools.get_iso_string_from_datetime( + datetime.now(timezone.utc) - timedelta(days=1) + ) + ) + + field_device_collection.features = [ + pre_process_field_device_feature(i) for i in field_device_collection.features + ] + + combined_events = get_combined_events(field_device_collection.features, wzdx) + + if len(combined_events) == 0: + print( + "No overlapping events found between WZDx and iCone data. See logs for more information." + ) + else: + with open(outputPath, "w+") as f: + f.write(json.dumps(combined_events, indent=2)) + print(f"Successfully wrote combined WZDx file to: {outputPath}") + + +# parse script command line arguments +def parse_rtdh_arguments() -> tuple[str, str, str, str]: + """Parse command line arguments for WZDx field device combination script + + Returns: + str: WZDx event file path + str: WZDx field device file path + str: output directory path + str: updateDates flag (true/false) + """ + parser = argparse.ArgumentParser( + description="Combine WZDx event and WZDx field device data" + ) + parser.add_argument( + "--version", action="version", version=f"{PROGRAM_NAME} {PROGRAM_VERSION}" + ) + parser.add_argument("wzdxFile", help="Incoming WZDx event file path (JSON)") + parser.add_argument("fieldDevicesFile", help="WZDx field device file path (JSON)") + parser.add_argument( + "--outputDir", required=False, default="./", help="output directory" + ) + parser.add_argument( + "--updateDates", + required=False, + default="false", + help="Boolean (true/false), Update dates to the current date to pass time filter", + ) + + args = parser.parse_args() + return args.wzdxFile, args.fieldDevicesFile, args.outputDir, args.updateDates + + +def parse_field_device_feature(raw: str) -> FieldDeviceFeature: + """Parse raw JSON string into WZDx field device feature + + Args: + raw (str): Raw JSON string + + Returns: + FieldDeviceFeature: WZDx field device feature + """ + return FieldDeviceFeature.model_validate_json(raw) + + +def pre_process_field_device_feature( + field_device_feature: FieldDeviceFeature, +) -> FieldDeviceFeature: + """Pre-process field device feature for combination + + Args: + field_device_feature (FieldDeviceFeature): WZDx field device feature + Returns: + FieldDeviceFeature: Pre-processed WZDx field device feature + """ + route_details_start, route_details_end = get_route_details_for_feature( + field_device_feature.geometry + ) + field_device_feature.route_details_start = route_details_start + field_device_feature.route_details_end = route_details_end + return field_device_feature + + +def get_direction_from_route_details(route_details: dict) -> str: + """Get direction from GIS route details + + Args: + route_details (dict): GIS route details + + Returns: + str: direction | "unknown" + """ + return route_details.get("Direction") + + +def get_direction( + street: str, coords: list[list[float]], route_details: dict = None +) -> str: + """Get road direction from street name, coordinates, or route details + + Args: + street (str): Street name, like "I-25 NB" + coords (list[list[float]]): List of coordinates, to pull direction from + route_details (dict, optional): GIS route details, defaults to None + + Returns: + Literal['unknown', 'eastbound', 'westbound', 'northbound', 'southbound']: Road direction + """ + direction = wzdx_translator.parse_direction_from_street_name(street) + if not direction and route_details: + direction = get_direction_from_route_details(route_details) + if not direction: + direction = geospatial_tools.get_road_direction_from_coordinates(coords) + return direction + + +def get_combined_events( + field_device_features: list[FieldDeviceFeature], wzdx_msgs: list[dict] +) -> list[dict]: + """Combine/integrate overlapping iCone messages into WZDx messages + + Args: + field_device_features (list[FieldDeviceFeature]): WZDx field device features + wzdx_msgs (list[dict]): WZDx messages + + Returns: + list[dict]: Combined WZDx messages + """ + combined_events: list[dict] = [] + filtered_wzdx_msgs = wzdx_translator.filter_wzdx_by_event_status( + wzdx_msgs, ["pending", "completed_recently"] + ) + + for i in identify_overlapping_features(field_device_features, filtered_wzdx_msgs): + icone_msg, wzdx_msg = i + event_status = wzdx_translator.get_event_status(wzdx_msg["features"][0]) + wzdx = combine_field_device_with_wzdx(icone_msg, wzdx_msg, event_status) + if wzdx: + combined_events.append(wzdx) + return combined_events + + +def combine_field_device_with_wzdx( + field_device_feature: FieldDeviceFeature, + wzdx_wzdx: dict, + event_status: Literal["active", "pending", "planned", "completed"], +) -> dict: + """Combine iCone message with WZDx message + + Args: + field_device_feature (FieldDeviceFeature): WZDx field device feature + wzdx_wzdx (dict): WZDx message + event_status (Literal["active", "pending", "planned", "completed"]): WZDx event status + + Returns: + dict: Combined WZDx message + """ + combined_event = wzdx_wzdx + updated = False + + if event_status == "pending": + # Start event early + # Field device has already been verified to be updated within the past 2 hours + combined_event["features"][0]["properties"]["start_date"] = ( + date_tools.get_iso_string_from_datetime(datetime.now(timezone.utc)) + ) + updated = True + elif event_status == "completed_recently": + # Keep event open longer + # Field device has already been verified to be updated within the past 2 hours + combined_event["features"][0]["properties"]["end_date"] = ( + date_tools.get_iso_string_from_datetime( + datetime.now(timezone.utc) + timedelta(hours=2) + ) + ) + combined_event["features"][0]["properties"]["core_details"]["description"] += ( + " " + field_device_feature.properties.core_details.description + ) + updated = True + if updated: + update_date = date_tools.get_iso_string_from_datetime( + datetime.now(timezone.utc) + ) + combined_event["features"][0]["properties"]["core_details"][ + "update_date" + ] = update_date + combined_event["feed_info"]["data_sources"][0]["update_date"] = update_date + + combined_event["features"][0]["properties"][ + "experimental_source_type" + ] = "icone" + combined_event["features"][0]["properties"][ + "experimental_source_id" + ] = field_device_feature.id + combined_event["features"][0]["properties"][ + "icone_id" + ] = field_device_feature.id + combined_event["features"][0]["properties"]["icone_message"] = ( + field_device_feature.model_dump( + by_alias=True, exclude_none=True, mode="json" + ) + ) + + for i in ["route_details_start", "route_details_end"]: + if i in combined_event: + del combined_event[i] + return combined_event + else: + return None + + +def get_route_details_for_feature(geometry: GeoJsonGeometry) -> tuple[dict, dict]: + """Get route details for iCone message + + Args: + geometry (GeoJsonGeometry): Geometry object + + Returns: + tuple[dict, dict]: Route details for start and end coordinates + """ + + match geometry.type: + case "Point": + coordinates = [geometry.coordinates] + case "MultiPoint": + coordinates = geometry.coordinates + case "LineString": + coordinates = [geometry.coordinates[0], geometry.coordinates[-1]] + case "Polygon": + coordinates = [geometry.coordinates[0][0], geometry.coordinates[-1][-1]] + + route_details_start = combination.get_route_details( + coordinates[0][1], coordinates[0][0] + ) + + if len(coordinates) == 1 or ( + len(coordinates) == 2 and coordinates[0] == coordinates[1] + ): + route_details_end = None + else: + route_details_end = combination.get_route_details( + coordinates[-1][1], coordinates[-1][0] + ) + + return route_details_start, route_details_end + + +def validate_directionality_wzdx_field_device( + field_device: FieldDeviceFeature, wzdx: dict +) -> bool: + """Validate directionality between field device and WZDx messages + + Args: + field_device (FieldDeviceFeature): Field device feature + wzdx (dict): WZDx message + + Returns: + bool: Directionality match + """ + direction_1 = field_device.properties.core_details.road_direction + direction_2 = wzdx["features"][0]["properties"]["core_details"]["direction"] + + return direction_1 in [None, "unknown", "undefined"] or direction_1 == direction_2 + + +def verify_recent(field_device: FieldDeviceFeature) -> bool: + """Validate that the field device information is recent (within 2 hours) + + Args: + field_device (FieldDeviceFeature): Field device feature + + Returns: + bool: Is recent information + """ + return field_device.properties.core_details.update_date > datetime.now( + timezone.utc + ) - timedelta(hours=2) + + +def identify_overlapping_features( + field_device_features: list[FieldDeviceFeature], wzdx_msgs: list[dict] +) -> list[tuple[dict, dict]]: + """Identify overlapping WZDx events and field devices + + Args: + field_device_features (list[FieldDeviceFeature]): Field device features + wzdx_msgs (list[dict]): WZDx messages + + Returns: + list[tuple[dict, dict]]: Overlapping field device and WZDx messages + """ + field_device_routes: dict[str, list[FieldDeviceFeature]] = {} + wzdx_event_routes: dict[str, list[dict]] = {} + matching_routes: list[tuple[dict, dict]] = [] + + recent_field_device_features = [ + i for i in field_device_features if verify_recent(i) + ] + + # Step 1: Add route info to field device features + for field_device_feature in recent_field_device_features: + if not field_device_feature.route_details_start: + logging.debug( + f"Invalid route details for field device: {field_device_feature.id}" + ) + continue + + if ( + field_device_feature.route_details_end + and field_device_feature.route_details_start["Route"] + != field_device_feature.route_details_end["Route"] + ): + logging.debug( + f"Mismatched routes for field device feature {field_device_feature.id}" + ) + continue + + if field_device_feature.route_details_start["Route"] in field_device_routes: + field_device_routes[ + field_device_feature.route_details_start["Route"] + ].append(field_device_feature) + else: + field_device_routes[field_device_feature.route_details_start["Route"]] = [ + field_device_feature + ] + + # Step 2: Add route info to WZDx messages + for wzdx in wzdx_msgs: + wzdx["route_details_start"] = wzdx["features"][0]["properties"].get( + "route_details_start" + ) + wzdx["route_details_end"] = wzdx["features"][0]["properties"].get( + "route_details_end" + ) + if ( + wzdx.get("route_details_start") + and not wzdx.get("route_details_end") + or not wzdx.get("route_details_start") + and wzdx.get("route_details_end") + ): + logging.debug( + f"Missing route_details for WZDx object: {wzdx['features'][0]['id']}" + ) + continue + if not wzdx.get("route_details_start") and not wzdx.get("route_details_end"): + route_details_start, route_details_end = ( + combination.get_route_details_for_wzdx(wzdx["features"][0]) + ) + + if not route_details_start or not route_details_end: + logging.debug(f"Missing WZDx route details {wzdx['features'][0]['id']}") + continue + wzdx["route_details_start"] = route_details_start + wzdx["route_details_end"] = route_details_end + else: + route_details_start = wzdx["route_details_start"] + route_details_end = wzdx["route_details_end"] + + if route_details_start["Route"] != route_details_end["Route"]: + logging.debug(f"Mismatched routes for feature {wzdx['features'][0]['id']}") + continue + + logging.debug( + "Route details: " + str(route_details_start) + str(route_details_end) + ) + + if route_details_start["Route"] in wzdx_event_routes: + wzdx_event_routes[route_details_start["Route"]].append(wzdx) + else: + wzdx_event_routes[route_details_start["Route"]] = [wzdx] + + if not field_device_routes: + logging.debug("No routes found for icone") + return [] + if not wzdx_event_routes: + logging.debug("No routes found for wzdx") + return [] + + # Step 3: Identify overlapping events + for wzdx_route_id, wzdx_matched_msgs in wzdx_event_routes.items(): + matching_field_device_routes = field_device_routes.get(wzdx_route_id, []) + + for match_field_device in matching_field_device_routes: + for match_wzdx in wzdx_matched_msgs: + # require routes to overlap, directionality to match, and dates to match + if combination.does_route_overlap( + match_field_device.model_dump(), match_wzdx + ) and validate_directionality_wzdx_field_device( + match_field_device, match_wzdx + ): + + matching_routes.append((match_field_device, match_wzdx)) + + return matching_routes + + +if __name__ == "__main__": + main() diff --git a/wzdx/models/field_device_feed/field_device_core_details.py b/wzdx/models/field_device_feed/field_device_core_details.py index 294dc415..8dc878fa 100644 --- a/wzdx/models/field_device_feed/field_device_core_details.py +++ b/wzdx/models/field_device_feed/field_device_core_details.py @@ -6,22 +6,23 @@ from .field_device_status import FieldDeviceStatus from ..enums import Direction + class FieldDeviceCoreDetails(BaseModel): - device_type: FieldDeviceType = Field(None, alias="device_type") - data_source_id: str = Field(None, alias="data_source_id") - device_status: FieldDeviceStatus = Field(None, alias="device_status") - update_date: datetime = Field(None, alias="update_date") - has_automatic_location: bool = Field(None, alias="has_automatic_location") + device_type: FieldDeviceType = Field(alias="device_type") + data_source_id: str = Field(alias="data_source_id") + device_status: FieldDeviceStatus = Field(alias="device_status") + update_date: datetime = Field(alias="update_date") + has_automatic_location: bool = Field(alias="has_automatic_location") road_direction: Optional[Direction] = Field(None, alias="road_direction") road_names: Optional[list[str]] = Field(None, alias="road_names") - name: Optional[str] = None - description: Optional[str] = None + name: Optional[str] = Field(None, alias="name") + description: Optional[str] = Field(None, alias="description") status_messages: Optional[list[str]] = Field(None, alias="status_messages") is_moving: Optional[bool] = Field(None, alias="is_moving") road_event_ids: Optional[list[str]] = Field(None, alias="road_event_ids") - milepost: Optional[float] = None - make: Optional[str] = None - model: Optional[str] = None + milepost: Optional[float] = Field(None, alias="milepost") + make: Optional[str] = Field(None, alias="make") + model: Optional[str] = Field(None, alias="model") serial_number: Optional[str] = Field(None, alias="serial_number") firmware_version: Optional[str] = Field(None, alias="firmware_version") velocity_kph: Optional[float] = Field(None, alias="velocity_kph") diff --git a/wzdx/models/field_device_feed/field_device_feature.py b/wzdx/models/field_device_feed/field_device_feature.py index 47ba2a33..59725df9 100644 --- a/wzdx/models/field_device_feed/field_device_feature.py +++ b/wzdx/models/field_device_feed/field_device_feature.py @@ -5,9 +5,14 @@ from ..geometry.geojson_geometry import GeoJsonGeometry from .properties.field_device_properties import FieldDeviceProperties + class FieldDeviceFeature(BaseModel): id: str type: str properties: FieldDeviceProperties geometry: GeoJsonGeometry # GeoJSON geometry object - bbox: Optional[list[float]] = None \ No newline at end of file + bbox: Optional[list[float]] = None + + # Custom Fields + route_details_start: Optional[dict] = None + route_details_end: Optional[dict] = None diff --git a/wzdx/models/field_device_feed/field_device_type.py b/wzdx/models/field_device_feed/field_device_type.py index 8c86d858..1eae56c7 100644 --- a/wzdx/models/field_device_feed/field_device_type.py +++ b/wzdx/models/field_device_feed/field_device_type.py @@ -1,12 +1,13 @@ # device_feed/field_device_type.py -from enum import Enum +from typing_extensions import Literal -class FieldDeviceType(str, Enum): - ARROW_BOARD = "arrow-board" - CAMERA = "camera" - DYNAMIC_MESSAGE_SIGN = "dynamic-message-sign" - FLASHING_BEACON = "flashing-beacon" - HYBRID_SIGN = "hybrid-sign" - LOCATION_MARKER = "location-marker" - TRAFFIC_SENSOR = "traffic-sensor" - TRAFFIC_SIGNAL = "traffic-signal" \ No newline at end of file +FieldDeviceType = Literal[ + "arrow-board", + "camera", + "dynamic-message-sign", + "flashing-beacon", + "hybrid-sign", + "location-marker", + "traffic-sensor", + "traffic-signal", +] diff --git a/wzdx/raw_to_standard/icone.py b/wzdx/raw_to_standard/icone.py deleted file mode 100644 index 84743e60..00000000 --- a/wzdx/raw_to_standard/icone.py +++ /dev/null @@ -1,371 +0,0 @@ -import argparse -import datetime -import json -import logging -import time -import uuid -import xml.etree.ElementTree as ET -from collections import OrderedDict - -from pydantic import TypeAdapter - -from wzdx.models.field_device_feed.device_feed import DeviceFeed -from wzdx.models.field_device_feed.field_device_feature import FieldDeviceFeature - -from ..tools import date_tools, geospatial_tools, wzdx_translator, combination -from ..util.collections import PathDict - -PROGRAM_NAME = "iConeRawToStandard" -PROGRAM_VERSION = "1.0" - - -def main(): - input_file, output_dir = parse_rtdh_arguments() - input_file_contents = open(input_file, "r").read() - generated_messages = generate_standard_messages_from_string(input_file_contents) - - generated_files_list = [] - features = [] - - for message in generated_messages: - output_path = f"{output_dir}/icone_{message['event']['source']['id']}_{round(message['rtdh_timestamp'])}_{message['event']['detail']['direction']}.json" - open(output_path, "w+").write(json.dumps(message, indent=2)) - generated_files_list.append(output_path) - - features.append( - { - "type": "Feature", - "properties": { - "id": message["event"]["source"]["id"], - "route_details": message["event"]["additional_info"][ - "route_details_start" - ], - }, - "geometry": { - "type": "Point", - "coordinates": message["event"]["geometry"][0], - }, - } - ) - - open(f"{output_dir}/icone_feature_collection.geojson", "w+").write( - json.dumps(features, indent=2) - ) - - if generated_files_list: - print(f"Successfully generated standard message files: {generated_files_list}") - else: - print( - "Warning: Standard message generation failed. See logging messages printed above" - ) - - -def generate_standard_messages_from_string(field_device_feed_json: str): - """Generate RTDH standard messages from iCone XML string - - Args: - input_file_contents: iCone XML string data - """ - device_feed = parse_device_feed(field_device_feed_json) - standard_messages = [] - for feature in device_feed.features: - standard_messages.append(create_rtdh_standard_msg(feature)) - return standard_messages - - -def retrieve_device_feed(auth_token: str, url: str) -> DeviceFeed: - """Retrieve Device Feed from URL with authentication token - - Args: - auth_token: Authentication token for accessing the device feed - url: URL of the device feed - """ - import requests - - headers = {"Authorization": f"Bearer {auth_token}"} - response = requests.get(url, headers=headers) - response.raise_for_status() - json_string = response.text - - return parse_device_feed(json_string) - - -def parse_device_feed(json_string: str) -> DeviceFeed: - """Parse iCone XML string and return list of validated xml incidents - - Args: - message: iCone XML string data - """ - adapter = TypeAdapter(list[DeviceFeed]) - device_feed: list[DeviceFeed] = adapter.validate_json(json_string) - - if device_feed: - return device_feed[0] - - -# parse script command line arguments -def parse_rtdh_arguments() -> tuple[str, str]: - """Parse command line arguments for iCone to RTDH Standard translation - - Returns: - tuple[str, str]: iCone file path, output directory - """ - parser = argparse.ArgumentParser( - description="Translate iCone data to RTDH Standard" - ) - parser.add_argument( - "--version", action="version", version=f"{PROGRAM_NAME} {PROGRAM_VERSION}" - ) - parser.add_argument("iconeFile", help="icone file path") - parser.add_argument( - "--outputDir", required=False, default="./", help="output directory" - ) - - args = parser.parse_args() - return args.iconeFile, args.outputDir - - -# function to parse polyline to geometry line string -# input: "37.1571990,-84.1128540,37.1686478,-84.1238971" (lat, long) -# output: [[-84.1128540, 37.1571990], [-84.1238971, 37.1686478]] (long, lat) -def parse_icone_polyline(polylineString: list[float]): - """Parse iCone polyline string to geometry line string - - Args: - polylineString: iCone polyline string - """ - if not polylineString or type(polylineString) is not str: - return None - # polyline right now is a list which has an empty string in it. - polyline = polylineString.split(",") - coordinates = [] - for i in range(0, len(polyline) - 1, 2): - try: - coordinates.append([float(polyline[i + 1]), float(polyline[i])]) - except ValueError: - logging.warning("failed to parse polyline!") - return [] - return coordinates - - -def get_sensor_list(incident: dict | OrderedDict): - """Get list of sensors from iCone incident - - Args: - incident: iCone incident object - """ - devices = [] - for key in ["sensor", "radar", "display", "message", "marker", "status"]: - obj = incident.get(f"{key}") - if type(obj) in [dict, OrderedDict]: - devices.append({"sensor_type": key, "details": obj}) - elif type(obj) is list: - for i in obj: - devices.append({"sensor_type": key, "details": i}) - return devices - - -def create_rtdh_standard_msg(pd: PathDict): - """Create RTDH standard message from iCone incident pathDict - - Args: - pd: iCone incident pathDict - """ - devices = get_sensor_list(pd.get("incident")) - start_time = pd.get( - "incident/starttime", date_tools.parse_datetime_from_iso_string, default=None - ) - end_time = pd.get( - "incident/endtime", date_tools.parse_datetime_from_iso_string, default=None - ) - if not end_time: - if start_time > datetime.datetime.now(datetime.timezone.utc): - end_time = start_time + datetime.timedelta(days=7) - else: - end_time = datetime.datetime.now( - datetime.timezone.utc - ) + datetime.timedelta(days=7) - # Added for unit test - end_time = end_time.replace(second=0, microsecond=0) - - coordinates = pd.get("incident/location/polyline", parse_icone_polyline) - - route_details_start, route_details_end = ( - combination.get_route_details_for_coordinates_lngLat(coordinates) - ) - - direction = get_direction( - pd.get("incident/location/street"), coordinates, route_details_start - ) - - road_name = pd.get("incident/location/street") - if not road_name: - road_name = get_road_name(route_details_start) - - return { - "rtdh_timestamp": time.time(), - "rtdh_message_id": str(uuid.uuid4()), - "event": { - "type": pd.get("incident/type", default=""), - "source": { - "id": pd.get("incident/@id", default=""), - "creation_timestamp": pd.get( - "incident/creationtime", - date_tools.get_unix_from_iso_string, - default=0, - ), - "last_updated_timestamp": pd.get( - "incident/updatetime", - date_tools.get_unix_from_iso_string, - default=0, - ), - }, - "geometry": pd.get("incident/location/polyline", parse_icone_polyline), - "header": { - "description": pd.get("incident/description", default=""), - "start_timestamp": date_tools.date_to_unix(start_time), - "end_timestamp": date_tools.date_to_unix(end_time), - }, - "detail": { - "road_name": road_name, - "road_number": road_name, - "direction": direction, - }, - "additional_info": { - "devices": devices, - "directionality": pd.get("incident/location/direction"), - "route_details_start": route_details_start, - "route_details_end": route_details_end, - "condition_1": True, - }, - }, - } - - -def get_direction(street: str, coords: list[float], route_details=None): - """Get road direction from street, coordinates, or route details - - Args: - street: Roadway name to pull direction from (I-25 NB, I-25 SB, etc.) - coords: Coordinates to pull direction from - route_details: Optional GIS route details to pull direction from. Defaults to None. - - Returns: - Literal['unknown', 'eastbound', 'westbound', 'northbound', 'southbound']: direction of roadway - """ - direction = wzdx_translator.parse_direction_from_street_name(street) - if (not direction or direction == "unknown") and route_details: - direction = get_direction_from_route_details(route_details) - if not direction or direction == "unknown": - direction = geospatial_tools.get_road_direction_from_coordinates(coords) - return direction - - -def get_road_name(route_details: dict) -> str | None: - """Get road name from GIS route details - - Args: - route_details: GIS route details - Returns: - str | None: road name - """ - return route_details.get("Route") - - -def get_direction_from_route_details(route_details: dict) -> str: - """Get direction from GIS route details - - Args: - route_details (dict): GIS route details - - Returns: - str: direction | "unknown" - """ - return route_details.get("Direction", "unknown") - - -# function to validate the incident -def validate_incident(incident: dict | OrderedDict): - """Validate iCone Incident against predefined set of rules (see below) - - Args: - incident: iCone incident object - - Returns: - bool: True if incident is valid, False otherwise - - Validation Rules: - - Incident must have a location object - - Incident must have a polyline object - - Incident must have a starttime field - - Incident must have a description field - - Incident must have a creationtime field - - Incident must have an updatetime field - - Incident must have a valid direction (parsable from street name or polyline) - - Incident must have a valid start time (parsable from ISO string) - - Incident must have a valid end time (parsable from ISO string) - """ - if not incident or ( - type(incident) is not dict and type(incident) is not OrderedDict - ): - logging.warning("incident is empty or has invalid type") - return False - - id = incident.get("@id") - - location = incident.get("location") - if not location: - logging.warning(f"Invalid incident with id = {id}. Location object not present") - return False - - polyline = location.get("polyline") - coords = parse_icone_polyline(polyline) - street = location.get("street", "") - - starttime_string = incident.get("starttime") - endtime_string = incident.get("endtime") - description = incident.get("description") - creationtime = incident.get("creationtime") - updatetime = incident.get("updatetime") - direction = get_direction(street, coords) - if not direction: - logging.warning( - f"Invalid incident with id = {id}. unable to parse direction from street name or polyline" - ) - return False - - required_fields = [ - location, - polyline, - coords, - starttime_string, - description, - creationtime, - updatetime, - ] - for field in required_fields: - if not field: - logging.warning( - f"""Invalid incident with id = {id}. Not all required fields are present. Required fields are: - location, polyline, starttime, description, creationtime, and updatetime""" - ) - return False - - start_time = date_tools.parse_datetime_from_iso_string(starttime_string) - end_time = date_tools.parse_datetime_from_iso_string(endtime_string) - if not start_time: - logging.warning( - f"Invalid incident with id = {id}. Unsupported start time format: {start_time}" - ) - return False - elif endtime_string and not end_time: - logging.warning( - f"Invalid incident with id = {id}. Unsupported end time format: {end_time}" - ) - return False - - return True - - -if __name__ == "__main__": - main() diff --git a/wzdx/sample_files/field_device_feed/icone_2025_12_29.json b/wzdx/sample_files/field_device_feed/icone_2025_12_29.json new file mode 100644 index 00000000..22914183 --- /dev/null +++ b/wzdx/sample_files/field_device_feed/icone_2025_12_29.json @@ -0,0 +1,117 @@ +{ + "feed_info": { + "update_date": "2025-12-29T16:34:16.5500000Z", + "publisher": "iCone Products LLC", + "contact_email": "support@iconeproducts.com", + "version": "4.2", + "data_sources": [ + { + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "update_date": "2025-12-29T16:34:16.5500000Z", + "organization_name": "iCone Products LLC", + "contact_email": "support@iconeproducts.com" + } + ], + "custom": { + "oldest_feature": "2025-12-28T16:34:14.3466667Z", + "oldest_location": "2025-12-28T16:34:14.3466667Z", + "username": "cdotfeeds", + "active_only": false, + "require_location": false, + "allow_custom_enums": true, + "include_custom": true, + "force_spec_required": false + } + }, + "type": "FeatureCollection", + "features": [ + { + "id": "04435BD5-83C9-44C5-968A-7CF54C089530", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-105.138771, 39.7756546] + }, + "properties": { + "core_details": { + "device_type": "arrow-board", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2025-12-29T16:31:45Z", + "has_automatic_location": true, + "description": "Roadwork - Caution" + }, + "pattern": "four-corners-flashing", + "custom": { + "start_date": "2025-12-23T19:58:46", + "waze_incident": { + "type": "HAZARD", + "description": "Roadwork - Caution" + } + } + } + }, + { + "id": "B5856293-1B04-4588-A924-D18F28D1D3F7", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-106.0714436, 39.6256957] + }, + "properties": { + "core_details": { + "device_type": "location-marker", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2025-12-29T16:32:51Z", + "has_automatic_location": true, + "description": "Roadwork Active" + }, + "marked_locations": [ + { + "type": "work-truck-with-lights-flashing" + } + ], + "custom": { + "isActive": true, + "start_date": "2025-12-29T16:28:34.4100000", + "waze_incident": { + "type": "HAZARD", + "description": "Roadwork Active" + } + } + } + }, + { + "id": "EE3612F8-0B81-4F46-B644-3F096A16AE98", + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [-105.0708308, 40.5873306] + }, + "properties": { + "core_details": { + "device_type": "location-marker", + "data_source_id": "67899A97-0F3E-4683-B169-75C09C3B8F67", + "device_status": "ok", + "update_date": "2025-12-29T16:33:42Z", + "has_automatic_location": true, + "description": "Roadwork Active" + }, + "marked_locations": [ + { + "type": "work-truck-with-lights-flashing" + } + ], + "custom": { + "isActive": true, + "start_date": "2025-12-29T16:33:38", + "waze_incident": { + "type": "HAZARD", + "description": "Roadwork Active" + } + } + } + } + ] +} diff --git a/wzdx/standard_to_wzdx/icone_translator.py b/wzdx/standard_to_wzdx/icone_translator.py deleted file mode 100644 index b9d5cc99..00000000 --- a/wzdx/standard_to_wzdx/icone_translator.py +++ /dev/null @@ -1,510 +0,0 @@ -import argparse -import json -import logging -import copy -from typing import Literal -import uuid - -from wzdx.models.enums import EventType, LocationMethod - -from ..tools import date_tools, wzdx_translator - -PROGRAM_NAME = "WZDxIconeTranslator" -PROGRAM_VERSION = "1.0" - - -def main(): - input_file, output_file = parse_icone_arguments() - - icone_obj = json.loads(open(input_file, "r").read()) - wzdx = wzdx_creator(icone_obj) - - if not wzdx: - print("Error: WZDx message generation failed, see logs for more information.") - else: - with open(output_file, "w") as fWzdx: - fWzdx.write(json.dumps(wzdx, indent=2)) - print( - "Your wzdx message was successfully generated and is located here: " - + str(output_file) - ) - - -# parse script command line arguments -def parse_icone_arguments() -> tuple[str, str]: - """Parse command line arguments for standard RTDH iCone data translation to WZDx - - Returns: - tuple[str, str]: iCone file path, output file path - """ - parser = argparse.ArgumentParser(description="Translate iCone data to WZDx") - parser.add_argument( - "--version", action="version", version=f"{PROGRAM_NAME} {PROGRAM_VERSION}" - ) - parser.add_argument("iconeFile", help="icone file path") - parser.add_argument( - "--outputFile", - required=False, - default="icone_wzdx_translated_output_message.geojson", - help="output file path", - ) - - args = parser.parse_args() - return args.iconeFile, args.outputFile - - -def wzdx_creator(message: dict, info: dict = None) -> dict: - """Translate standard RTDH iCone data to WZDx - - Args: - message (dict): iCone data - info (dict, optional): WZDx info object. Defaults to None. - - Returns: - dict: WZDx object - """ - if not message or not validate_standard_msg(message): - return None - - if not info: - info = wzdx_translator.initialize_info() - if not wzdx_translator.validate_info(info): - return None - - wzdx = wzdx_translator.initialize_wzdx_object(info) - - # Parse Incident to WZDx Feature - feature = parse_incident(message) - if feature: - wzdx.get("features").append(feature) - - if not wzdx.get("features"): - return None - wzdx = wzdx_translator.add_ids(wzdx) - - if not wzdx_translator.validate_wzdx(wzdx): - logging.warning("WZDx message failed validation") - return None - - return wzdx - - -#################### Sample Incident #################### -# -# 2020-12-16T17:17:00Z -# 2020-12-16T17:47:00Z -# CONSTRUCTION -# Roadwork - Lane Closed, MERGE LEFT [iCone] -# -# ONE_DIRECTION -# [28.8060608,-96.9916512,28.8060608,-96.9916512] -# -# 2020-12-16T17:17:00Z -# - - -# { -# "rtdh_timestamp": 1633097202.1872184, -# "rtdh_message_id": "bffd71cd-d35a-45c2-ba4d-a86e1ff12847", -# "event": { -# "type": "CONSTRUCTION", -# "source": { -# "id": 1245, -# "last_updated_timestamp": 1598046722 -# }, -# "geometry": "", -# "header": { -# "description": "19-1245: Roadwork between MP 40 and MP 48", -# "start_timestamp": 1581725296, -# "end_timestamp": null -# }, -# "detail": { -# "road_name": "I-75 N", -# "road_number": "I-75 N", -# "direction": null -# } -# } -# } - - -# function to calculate vehicle impact -def get_vehicle_impact( - description: str, -) -> Literal["all-lanes-open", "some-lanes-closed"]: - """Calculate vehicle impact based on description - - Args: - description (str): Incident description - - Returns: - Literal["all-lanes-open", "some-lanes-closed"]: Vehicle impact - """ - vehicle_impact = "all-lanes-open" - if "lane closed" in description.lower(): - vehicle_impact = "some-lanes-closed" - return vehicle_impact - - -# function to get description -def create_description(incident: dict) -> str: - """Create description from incident - - Args: - incident (dict): RTDH standard incident object - - Returns: - str: Description - """ - description = incident.get("description") - - if incident.get("sensor"): - description += "\n sensors: " - for sensor in incident.get("sensor"): - if not isinstance(sensor, str): - if sensor["@type"] == "iCone": - description += "\n" + json.dumps( - parse_icone_sensor(sensor), indent=2 - ) - else: - sensor = incident.get("sensor") - if sensor["@type"] == "iCone": - description += "\n" + json.dumps( - parse_icone_sensor(sensor), indent=2 - ) - - if incident.get("display"): - description += "\n displays: " - for display in incident.get("display"): - if not isinstance(display, str): - if display["@type"] == "PCMS": - description += "\n" + json.dumps( - parse_pcms_sensor(display), indent=2 - ) # add baton,ab,truck beacon,ipin,signal - else: - display = incident.get("display") - if display["@type"] == "PCMS": - description += "\n" + json.dumps( - parse_pcms_sensor(display), indent=2 - ) # add baton,ab,truck beacon,ipin,signal - - return description - - -def parse_icone_sensor(sensor: dict) -> dict: - """Parse iCone sensor data - - Args: - sensor (dict): iCone sensor data - - Returns: - dict: Parsed iCone sensor data - """ - icone = {} - icone["type"] = sensor.get("@type") - icone["id"] = sensor.get("@id") - icone["location"] = [ - float(sensor.get("@latitude")), - float(sensor.get("@longitude")), - ] - - if sensor.get("radar", None): - avg_speed = 0 - std_dev_speed = 0 - num_reads = 0 - for radar in sensor.get("radar"): - timestamp = "" - if not isinstance(radar, str): - curr_reads = int(radar.get("@numReads")) - if curr_reads == 0: - continue - curr_avg_speed = float(radar.get("@avgSpeed")) - curr_dev_speed = float(radar.get("@stDevSpeed")) - total_num_reads = num_reads + curr_reads - avg_speed = ( - avg_speed * num_reads + curr_avg_speed * curr_reads - ) / total_num_reads - std_dev_speed = ( - std_dev_speed * num_reads + curr_dev_speed * curr_reads - ) / total_num_reads - num_reads = total_num_reads - timestamp = radar.get("@intervalEnd") - else: - radar = sensor.get("radar") - avg_speed = float(radar.get("@avgSpeed")) - std_dev_speed = float(radar.get("@stDevSpeed")) - timestamp = radar.get("@intervalEnd") - - radar = {} - - radar["average_speed"] = round(avg_speed, 2) - radar["std_dev_speed"] = round(std_dev_speed, 2) - radar["timestamp"] = timestamp - icone["radar"] = radar - return icone - - -def parse_pcms_sensor(sensor: dict) -> dict: - """Parse PCMS sensor data - - Args: - sensor (dict): iCone PCMS sensor data - - Returns: - dict: Parsed PCMS sensor data - """ - pcms = {} - pcms["type"] = sensor.get("@type") - pcms["id"] = sensor.get("@id") - pcms["timestamp"] = sensor.get("@id") - pcms["location"] = [float(sensor.get("@latitude")), float(sensor.get("@longitude"))] - if sensor.get("message", None): - pcms["messages"] = [] - for message in sensor.get("message"): - if not isinstance(message, str): - pcms["timestamp"] = message.get("@verified") - if message.get("@text") not in pcms.get("messages"): - pcms.get("messages").append(message.get("@text")) - else: - message = sensor.get("message") - pcms["timestamp"] = message.get("@verified") - if message["@text"] not in pcms.get("messages"): - pcms.get("messages").append(message.get("@text")) - return pcms - - -# Parse Icone Incident to WZDx -def parse_incident(incident: dict) -> dict: - """Parse iCone incident to WZDx feature - - Args: - incident (dict): standard RTDH iCone incident - - Returns: - dict: WZDx feature - """ - event = incident.get("event") - - source = event.get("source") - header = event.get("header") - detail = event.get("detail") - additional_info = event.get("additional_info", {}) - - geometry = {} - geometry["type"] = "LineString" - geometry["coordinates"] = event.get("geometry") - properties = wzdx_translator.initialize_feature_properties() - - # I included a skeleton of the message, fill out all required fields and as many optional fields as you can. Below is a link to the spec page for a road event - # https://github.com/usdot-jpo-ode/jpo-wzdx/blob/master/spec-content/objects/RoadEvent.md - - core_details = properties["core_details"] - - # Event Type ['work-zone', 'detour'] - core_details["event_type"] = EventType.WORK_ZONE.value - - # data_source_id - Leave this empty, it will be populated by add_ids - core_details["data_source_id"] = "" - - # road_name - road_names = [detail.get("road_name")] - core_details["road_names"] = road_names - - # direction - core_details["direction"] = detail.get("direction") - - # related_road_events - current approach generates a individual disconnected events, so no links are generated - core_details["related_road_events"] = [] - - # description - core_details["description"] = header.get("description") - - # creation_date - core_details["creation_date"] = date_tools.get_iso_string_from_unix( - source.get("creation_timestamp") - ) - - # update_date - core_details["update_date"] = date_tools.get_iso_string_from_unix( - source.get("last_updated_timestamp") - ) - - # core_details - properties["core_details"] = core_details - - start_time = date_tools.parse_datetime_from_unix(header.get("start_timestamp")) - end_time = date_tools.parse_datetime_from_unix(header.get("end_timestamp")) - - # start_date - properties["start_date"] = date_tools.get_iso_string_from_datetime(start_time) - - # end_date - if end_time: - properties["end_date"] = date_tools.get_iso_string_from_datetime(end_time) - else: - properties["end_date"] = None - - # is_start_date_verified - properties["is_start_date_verified"] = False - - # is_end_date_verified - properties["is_end_date_verified"] = False - - # is_start_position_verified - properties["is_start_position_verified"] = False - - # is_end_position_verified - properties["is_end_position_verified"] = False - - # location_method - properties["location_method"] = LocationMethod.CHANNEL_DEVICE_METHOD.value - - # vehicle impact - properties["vehicle_impact"] = get_vehicle_impact(header.get("description")) - - # lanes - properties["lanes"] = [] - - # beginning_cross_street - properties["beginning_cross_street"] = "" - - # beginning_cross_street - properties["ending_cross_street"] = "" - - # beginning_cross_street - properties["beginning_milepost"] = "" - - # beginning_cross_street - properties["ending_milepost"] = "" - - # type_of_work - # maintenance, minor-road-defect-repair, roadside-work, overhead-work, below-road-work, barrier-work, surface-work, painting, roadway-relocation, roadway-creation - properties["types_of_work"] = [] - - # worker_presence - not available - - # reduced_speed_limit_kph - not available - - # restrictions - properties["restrictions"] = [] - - properties["route_details_start"] = additional_info.get("route_details_start") - properties["route_details_end"] = additional_info.get("route_details_end") - - properties["condition_1"] = additional_info.get("condition_1", True) - - filtered_properties = copy.deepcopy(properties) - - INVALID_PROPERTIES = [None, "", []] - - for key, value in properties.items(): - if value in INVALID_PROPERTIES: - del filtered_properties[key] - - for key, value in properties["core_details"].items(): - if not value and key not in ["data_source_id"]: - del filtered_properties["core_details"][key] - - feature = {} - feature["id"] = event.get("source", {}).get("id", uuid.uuid4()) - feature["type"] = "Feature" - feature["properties"] = filtered_properties - feature["geometry"] = geometry - - return feature - - -# function to validate the event -def validate_standard_msg(msg: dict) -> bool: - """Validate the event - - Args: - msg (dict): Event message - - Returns: - bool: True if event is valid, False otherwise - """ - if not msg or type(msg) is not dict: - logging.warning("event is empty or has invalid type") - return False - - event = msg.get("event") - - source = event.get("source") - header = event.get("header") - detail = event.get("detail") - - id = source.get("id") - try: - - event = msg.get("event") - - source = event.get("source") - header = event.get("header") - detail = event.get("detail") - - id = source.get("id") - - geometry = event.get("geometry") - road_name = detail.get("road_name") - - start_time = header.get("start_timestamp") - end_time = header.get("end_timestamp") - description = header.get("description") - update_time = source.get("last_updated_timestamp") - direction = detail.get("direction") - - if not (type(geometry) is list and len(geometry) >= 0): - logging.warning( - f"""Invalid event with id = {id}. Invalid geometry: {geometry}""" - ) - return False - if not (type(road_name) is str and len(road_name) >= 0): - logging.warning( - f"""Invalid event with id = {id}. Invalid road_name: {road_name}""" - ) - return False - if not (type(start_time) is float or type(start_time) is int): - logging.warning( - f"""Invalid event with id = {id}. Invalid start_time: {start_time}""" - ) - return False - if not (type(end_time) is float or type(end_time) is int or end_time is None): - logging.warning( - f"""Invalid event with id = {id}. Invalid end_time: {end_time}""" - ) - return False - if not (type(update_time) is float or type(update_time) is int): - logging.warning( - f"""Invalid event with id = {id}. Invalid update_time: {update_time}""" - ) - return False - if not ( - type(direction) is str - and direction - in [ - "unknown", - "undefined", - "northbound", - "southbound", - "eastbound", - "westbound", - ] - ): - logging.warning( - f"""Invalid event with id = {id}. Invalid direction: {direction}""" - ) - return False - if not (type(description) is str and len(description) >= 0): - logging.warning( - f"""Invalid event with id = {id}. Invalid description: {description}""" - ) - return False - - return True - except Exception as e: - logging.warning(f"""Invalid event with id = {id}. Error in validation: {e}""") - return False - - -if __name__ == "__main__": - main() diff --git a/wzdx/tools/cdot_geospatial_api.py b/wzdx/tools/cdot_geospatial_api.py index 02fbb8c6..c181a812 100644 --- a/wzdx/tools/cdot_geospatial_api.py +++ b/wzdx/tools/cdot_geospatial_api.py @@ -18,7 +18,7 @@ def __init__( setCachedRequest: Callable[[str, str], None] = lambda url, response: None, BASE_URL: str = os.getenv( "CDOT_GEOSPATIAL_API_BASE_URL", - "https://dtdapps.coloradodot.info/arcgis/rest/services/LRS/Routes/MapServer/exts/LrsServerRounded", + "https://dtdapps.codot.gov/server/rest/services/LRS/Routes_withDEC/MapServer/exts/LrsServerRounded", ), ): """Initialize the Geospatial API From 132c93fdbae9efaa17e2bbd66a25e7175e9c4c54 Mon Sep 17 00:00:00 2001 From: jacob6838 Date: Tue, 30 Dec 2025 11:40:20 -0700 Subject: [PATCH 2/2] Removing unused icone test data files --- .../field_devices/icone_standard_hwy-159.json | 25 ------------------- .../field_devices/icone_standard_nowhere.json | 25 ------------------- 2 files changed, 50 deletions(-) delete mode 100644 tests/data/experimental_combination/field_devices/icone_standard_hwy-159.json delete mode 100644 tests/data/experimental_combination/field_devices/icone_standard_nowhere.json diff --git a/tests/data/experimental_combination/field_devices/icone_standard_hwy-159.json b/tests/data/experimental_combination/field_devices/icone_standard_hwy-159.json deleted file mode 100644 index 2fcd44d5..00000000 --- a/tests/data/experimental_combination/field_devices/icone_standard_hwy-159.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "rtdh_timestamp": 1658952000.4258583, - "rtdh_message_id": "4c94f97e-e0ea-4336-86ab-49c5bc1317d1", - "event": { - "type": "CONSTRUCTION", - "source": { - "id": 1246, - "last_updated_timestamp": 1658952000 - }, - "geometry": [ - [-105.43859934199997, 37.19252041900006], - [-105.42712700799996, 37.19764377300004] - ], - "header": { - "description": "19-1245: Roadwork between MP 48 and MP 40", - "start_timestamp": 1658952000, - "end_timestamp": null - }, - "detail": { - "road_name": "I-75 S", - "road_number": "I-75 S", - "direction": "southbound" - } - } -} diff --git a/tests/data/experimental_combination/field_devices/icone_standard_nowhere.json b/tests/data/experimental_combination/field_devices/icone_standard_nowhere.json deleted file mode 100644 index 39c2890f..00000000 --- a/tests/data/experimental_combination/field_devices/icone_standard_nowhere.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "rtdh_timestamp": 1658952000.4258583, - "rtdh_message_id": "4c94f97e-e0ea-4336-86ab-49c5bc1317d1", - "event": { - "type": "CONSTRUCTION", - "source": { - "id": 1246, - "last_updated_timestamp": 1658952000 - }, - "geometry": [ - [-106.94366455078125, 39.455546939606066], - [-106.93954467773438, 39.45316112807394] - ], - "header": { - "description": "19-1245: Roadwork between MP 48 and MP 40", - "start_timestamp": 1658952000, - "end_timestamp": null - }, - "detail": { - "road_name": "I-75 S", - "road_number": "I-75 S", - "direction": "southbound" - } - } -}