From 030173a7ae5878d6bca3176a140f48283e28842c Mon Sep 17 00:00:00 2001 From: jecarr Date: Tue, 22 Jul 2025 17:46:59 +0100 Subject: [PATCH 1/4] Nav-JSON resp rewrite + AFB resp skeleton --- main.py | 1 + threadcomponents/handlers/web_api.py | 23 ++++++++++++++++-- threadcomponents/reports/afb_exporter.py | 10 ++++++++ threadcomponents/reports/report_exporter.py | 12 +++++++--- threadcomponents/service/web_svc.py | 3 ++- webapp/html/columns.html | 11 +++++---- webapp/theme/scripts/basics.js | 26 --------------------- 7 files changed, 49 insertions(+), 37 deletions(-) create mode 100644 threadcomponents/reports/afb_exporter.py diff --git a/main.py b/main.py index f46e194..480403d 100644 --- a/main.py +++ b/main.py @@ -115,6 +115,7 @@ async def init(host, port, app_setup_func=None): app.router.add_route("*", web_svc.get_route(WebService.REST_KEY), website_handler.rest_api) app.router.add_route("GET", web_svc.get_route(WebService.EXPORT_PDF_KEY), website_handler.pdf_export) app.router.add_route("GET", web_svc.get_route(WebService.EXPORT_NAV_KEY), website_handler.nav_export) + app.router.add_route("GET", web_svc.get_route(WebService.EXPORT_AFB_KEY), website_handler.afb_export) app.router.add_route("GET", web_svc.get_route(WebService.COOKIE_KEY), website_handler.accept_cookies) if not web_svc.is_local: app.router.add_route("GET", web_svc.get_route(WebService.WHAT_TO_SUBMIT_KEY), website_handler.what_to_submit) diff --git a/threadcomponents/handlers/web_api.py b/threadcomponents/handlers/web_api.py index 9aa59f5..e6972f1 100644 --- a/threadcomponents/handlers/web_api.py +++ b/threadcomponents/handlers/web_api.py @@ -4,6 +4,7 @@ import logging +from aiohttp import web as aiohttp_web from aiohttp.web_exceptions import HTTPException from aiohttp_jinja2 import template, web from aiohttp_security import authorized_userid @@ -324,6 +325,7 @@ async def edit(self, request): final_html = await self.web_svc.build_final_html(original_html, sentences) pdf_link = self.web_svc.get_route(self.web_svc.EXPORT_PDF_KEY, param=title_quoted) nav_link = self.web_svc.get_route(self.web_svc.EXPORT_NAV_KEY, param=title_quoted) + afb_link = self.web_svc.get_route(self.web_svc.EXPORT_AFB_KEY, param=title_quoted) # Add some help-text completed_info, private_info, sen_limit_help = None, None, None is_completed = int(report_status == ReportStatus.COMPLETED.value) @@ -358,6 +360,7 @@ async def edit(self, request): original_html=original_html, pdf_link=pdf_link, nav_link=nav_link, + afb_link=afb_link, unchecked=unchecked, completed_help_text=completed_info, private_help_text=private_info, @@ -388,14 +391,30 @@ async def edit(self, request): template_data.update(same_dates=True) return template_data + @staticmethod + def get_downloadable_export_response(file_data, media_type, filename): + """Returns a response with a downloadable file.""" + return aiohttp_web.Response( + body=file_data, + headers={ + "Content-Type": f"application/{media_type}", + "Content-Disposition": f"attachment; filename={filename}", + }, + ) + + async def afb_export(self, request): + """Function to export report in AFB format.""" + json_str = await self.report_exporter.afb_export(request) + return self.get_downloadable_export_response(json_str, "octet-stream", "data.afb") + async def nav_export(self, request): """ Function to export confirmed sentences in layer json format :param request: The title of the report information :return: the layer json """ - json_str = await self.report_exporter.nav_export(request) - return web.json_response(json_str) + filename, json_str = await self.report_exporter.nav_export(request) + return self.get_downloadable_export_response(json_str, "json", filename) async def pdf_export(self, request): """ diff --git a/threadcomponents/reports/afb_exporter.py b/threadcomponents/reports/afb_exporter.py new file mode 100644 index 0000000..c77daf3 --- /dev/null +++ b/threadcomponents/reports/afb_exporter.py @@ -0,0 +1,10 @@ +class AFBExporter: + """A class to export report mappings in an AFB format.""" + def __init__(self): + pass + + async def export(self): + """Executes the export.""" + data = {} + + return data diff --git a/threadcomponents/reports/report_exporter.py b/threadcomponents/reports/report_exporter.py index 29dc049..93ebcaf 100644 --- a/threadcomponents/reports/report_exporter.py +++ b/threadcomponents/reports/report_exporter.py @@ -3,6 +3,7 @@ from aiohttp_jinja2 import web from threadcomponents.enums import ReportStatus from threadcomponents.constants import UID, URL, TITLE +from threadcomponents.reports.afb_exporter import AFBExporter STATUS = "current_status" DATE_WRITTEN = "date_written_str" @@ -56,6 +57,11 @@ async def check_request_for_export(self, request, action): raise web.HTTPNotFound() return report + async def afb_export(self, request): + """Exports a report in an AFB format.""" + data = await AFBExporter().export() + return json.dumps(data, indent=4) + async def nav_export(self, request): """Exports a report in a navigator-friendly format.""" report = await self.check_request_for_export(request, "nav-export") @@ -76,7 +82,6 @@ async def nav_export(self, request): # Enterprise navigator layer enterprise_layer = { - "filename": sanitise_filename(report_title), "name": report_title, "domain": "mitre-enterprise", "description": enterprise_layer_description, @@ -100,8 +105,9 @@ async def nav_export(self, request): for technique in techniques: enterprise_layer["techniques"].append(technique) - # Return as a JSON string - return json.dumps(enterprise_layer) + # Return as a filename, JSON string tuple + filename = f"{sanitise_filename(report_title)}.json" + return filename, json.dumps(enterprise_layer, indent=4) async def pdf_export(self, request): report = await self.check_request_for_export(request, "pdf-export") diff --git a/threadcomponents/service/web_svc.py b/threadcomponents/service/web_svc.py index 1e9e604..7d9d0f6 100644 --- a/threadcomponents/service/web_svc.py +++ b/threadcomponents/service/web_svc.py @@ -21,7 +21,7 @@ class WebService: # Static class variables for the keys in app_routes HOME_KEY, COOKIE_KEY, EDIT_KEY, ABOUT_KEY, REST_KEY = "home", "cookies", "edit", "about", "rest" - EXPORT_PDF_KEY, EXPORT_NAV_KEY, STATIC_KEY = "export_pdf", "export_nav", "static" + EXPORT_PDF_KEY, EXPORT_NAV_KEY, EXPORT_AFB_KEY, STATIC_KEY = "export_pdf", "export_nav", "export_afb", "static" HOW_IT_WORKS_KEY, WHAT_TO_SUBMIT_KEY = "how_it_works", "what_to_submit" REPORT_PARAM = "file" # Variations of punctuation we want to note @@ -88,6 +88,7 @@ def _initialise_route_values(self, route_prefix_param=None): self.REST_KEY: route_prefix + "/rest", self.EXPORT_PDF_KEY: route_prefix + "/export/pdf/{%s}" % self.REPORT_PARAM, self.EXPORT_NAV_KEY: route_prefix + "/export/nav/{%s}" % self.REPORT_PARAM, + self.EXPORT_AFB_KEY: route_prefix + "/export/afb/{%s}" % self.REPORT_PARAM, self.HOW_IT_WORKS_KEY: route_prefix + "/how-thread-works", self.STATIC_KEY: route_prefix + "/theme/", } diff --git a/webapp/html/columns.html b/webapp/html/columns.html index d56af89..7dcbb48 100644 --- a/webapp/html/columns.html +++ b/webapp/html/columns.html @@ -48,12 +48,13 @@

{{ file }}

+
- - + Export Navigator JSON +
+ +
+ Export AFB
diff --git a/webapp/theme/scripts/basics.js b/webapp/theme/scripts/basics.js index 32b5d1f..055d19c 100644 --- a/webapp/theme/scripts/basics.js +++ b/webapp/theme/scripts/basics.js @@ -681,32 +681,6 @@ function downloadPDF(data) { generatedPDF.download(data["info"]["title"]); } -function downloadLayer(data) { - // Create the name of the JSON download file from the name of the report - var json = JSON.parse(data) - var filename = json["filename"] + ".json"; - // We don't need to include the filename property within the file - delete json["filename"]; - // Encode updated json as a uri component - var dataStr = "text/json;charset=utf-8," + encodeURIComponent(JSON.stringify(json)); - // Create temporary DOM element with attribute values needed to perform the download - var a = document.createElement("a"); - a.href = "data:" + dataStr; - a.download = filename; - a.innerHTML = "download JSON"; - // Add the temporary element to the DOM - var container = document.getElementById("dropdownMenu"); - container.appendChild(a); - // Download the JSON document - a.click(); - // Remove the temporary element from the DOM - a.remove(); -} - -function viewLayer(data) { - console.info("viewLayer: " + data) -} - function divSentenceReload() { $("#sentenceContextSection").load(document.URL + " #sentenceContextSection"); } From 53aabdeda84f103d7875e4b5d671f5875a8102f9 Mon Sep 17 00:00:00 2001 From: jecarr Date: Thu, 24 Jul 2025 15:15:02 +0100 Subject: [PATCH 2/4] AFB export --- threadcomponents/constants.py | 3 + threadcomponents/handlers/web_api.py | 4 +- threadcomponents/reports/afb_exporter.py | 247 +++++++++++++++++++- threadcomponents/reports/report_exporter.py | 10 +- threadcomponents/service/data_svc.py | 20 ++ 5 files changed, 275 insertions(+), 9 deletions(-) diff --git a/threadcomponents/constants.py b/threadcomponents/constants.py index 8173845..49f7495 100644 --- a/threadcomponents/constants.py +++ b/threadcomponents/constants.py @@ -5,3 +5,6 @@ UID = "uid" URL = "url" TITLE = "title" + +TTP = "ttp" +IOC = "ioc" diff --git a/threadcomponents/handlers/web_api.py b/threadcomponents/handlers/web_api.py index e6972f1..2f26234 100644 --- a/threadcomponents/handlers/web_api.py +++ b/threadcomponents/handlers/web_api.py @@ -404,8 +404,8 @@ def get_downloadable_export_response(file_data, media_type, filename): async def afb_export(self, request): """Function to export report in AFB format.""" - json_str = await self.report_exporter.afb_export(request) - return self.get_downloadable_export_response(json_str, "octet-stream", "data.afb") + filename, json_str = await self.report_exporter.afb_export(request) + return self.get_downloadable_export_response(json_str, "json", filename) async def nav_export(self, request): """ diff --git a/threadcomponents/reports/afb_exporter.py b/threadcomponents/reports/afb_exporter.py index c77daf3..e958ad4 100644 --- a/threadcomponents/reports/afb_exporter.py +++ b/threadcomponents/reports/afb_exporter.py @@ -1,10 +1,247 @@ +from datetime import datetime +from ipaddress import ip_address, IPv4Address, IPv6Address +from threadcomponents.constants import TTP, IOC +from uuid import uuid4 + +START_X = 0 +START_Y = -2000 +TILE_SHIFT = 400 +TILES_PER_ROW = 3 + +NAME_CHAR_LIMIT = 60 + + class AFBExporter: """A class to export report mappings in an AFB format.""" - def __init__(self): - pass - async def export(self): + def __init__(self, report_title, report_data): + self.report_title = report_title + self.report_data = report_data + self.reset() + + def reset(self): + self.exported = { + "schema": "attack_flow_v2", + } + self.objects = [] + self.current_ids = set() + + self.layout = {} + self.current_x = START_X + self.current_y = START_Y + + self.name = "" + self.description = None + self.errors = [] + + def export(self): """Executes the export.""" - data = {} + self.reset() + self._set_name_and_description() + self._build_objects_list() + self._add_layout_object() + self._add_camera_object() + return self.exported + + def _build_objects_list(self): + """Build the objects list to be included in the exported data.""" + for data in self.report_data: + data_type = data.get("type") + mapping_key = data.get("tid") or "IOC" + text_truncated = data.get("text", "").replace("\n", "")[:10] + entry_str = f"{mapping_key} - {text_truncated}" + + tile_id = self._generate_tile_id(entry_str) + if not tile_id: + continue + + added = False + + if data_type == IOC: + added = self._add_ioc_object(tile_id, data) + + elif data_type == TTP: + if mapping_key.startswith("T"): + added = self._add_ttp_object(tile_id, data) + + elif mapping_key.startswith("S"): + added = self._add_malware_object(tile_id, data) + + if added: + self._add_tile_to_layout(tile_id) + + self._finalise_objects_list() + + def _set_name_and_description(self): + """Sets the name and description to be exported.""" + if len(self.report_title) >= NAME_CHAR_LIMIT: + self.name = f"{self.report_title[: NAME_CHAR_LIMIT - 1]}-" + self.description = self.report_title + + else: + self.name = self.report_title + + def _finalise_description(self): + """Finalises the description to include any errors that occurred during export.""" + error_str = "\n".join(self.errors) + + if error_str: + self.description = f"{self.description}\n\n" if self.description else "" + self.description += error_str + + def _finalise_objects_list(self): + """Finalises the objects-list to be exported.""" + flow_entry = self._generate_flow_object() + self.exported["objects"] = [flow_entry] + self.objects + + def _add_layout_object(self): + """Adds the layout-entry to the exported data.""" + self.exported["layout"] = self.layout + + def _add_camera_object(self): + """Adds the camera-entry to the exported data.""" + self.exported["camera"] = { + "x": START_X + TILE_SHIFT, + "y": START_Y + TILE_SHIFT, + "k": 0.5, + } + + def _generate_flow_object(self): + """Generates and returns the flow-entry to be added to the exported data.""" + self._finalise_description() + + return { + "id": "flow", + "properties": [ + ["name", self.name], + ["description", self.description], + [ + "author", + [ + ["name", None], + ["identity_class", "individual"], + ], + ], + ["scope", "attack-tree"], + ["created", datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ")], + ], + "objects": list(self.current_ids), + } + + def _add_ioc_object(self, tile_id, data): + """Adds the IoC-entry to the objects-list.""" + try: + return self._add_ip_object(tile_id, data) + + except ValueError: + return self._add_command_line_object(tile_id, data) + + def _add_ip_object(self, tile_id, data): + """Adds the IP-entry to the objects-list.""" + ioc_cleaned_text = data.get("refanged_text", "") + ioc_text = data.get("text", "") + + ip_obj = ip_address(ioc_cleaned_text) + tile_type = None + + if isinstance(ip_obj, IPv4Address): + tile_type = "ipv4_addr" + elif isinstance(ip_obj, IPv6Address): + tile_type = "ipv6_addr" + + if tile_type: + self._add_object( + tile_id, + tile_type, + properties=[ + ["value", ioc_text], + ], + ) + return True + + def _add_command_line_object(self, tile_id, data): + """Adds the command-line-entry to the objects-list.""" + executed = data.get("text", "") + + self._add_object( + tile_id, + "process", + properties=[ + ["command_line", executed], + ], + ) + return True + + def _add_ttp_object(self, tile_id, data): + """Adds the TTP-entry to the objects-list.""" + tech_name = data.get("name") + tech_tid = data.get("tid") + tech_uid = data.get("attack_uid") + + self._add_object( + tile_id, + "action", + properties=[ + ["name", tech_name], + ["technique_id", tech_tid], + ["technique_ref", tech_uid], + [ + "ttp", + [ + ["technique", tech_tid], + ], + ], + ], + ) + + return True + + def _add_malware_object(self, tile_id, data): + """Adds the malware-entry to the objects-list.""" + malware_name = data.get("name") + + self._add_object( + tile_id, + "malware", + properties=[ + ["name", malware_name], + ], + ) + return True + + def _add_object(self, tile_id, id_val, properties=None): + """Adds an-entry to the objects-list.""" + properties = properties or [] + + self.objects.append( + { + "id": id_val, + "instance": tile_id, + "properties": properties, + "anchors": {}, + } + ) + + def _add_tile_to_layout(self, tile_id): + """Updates the layout object to place the given tile_id.""" + self.layout[tile_id] = [self.current_x, self.current_y] + self._update_current_position() + + def _update_current_position(self): + """Updates the current position in the layout.""" + self.current_x += TILE_SHIFT + + if self.current_x > (TILE_SHIFT * TILES_PER_ROW): + self.current_x = 0 + self.current_y += TILE_SHIFT + + def _generate_tile_id(self, log_missing): + """Generates and returns a new ID.""" + for _ in range(5): + new_id = str(uuid4()) + + if new_id not in self.current_ids: + self.current_ids.add(new_id) + return new_id - return data + self.errors.append(f"Missing {log_missing}") diff --git a/threadcomponents/reports/report_exporter.py b/threadcomponents/reports/report_exporter.py index 93ebcaf..3f7eb4e 100644 --- a/threadcomponents/reports/report_exporter.py +++ b/threadcomponents/reports/report_exporter.py @@ -59,8 +59,14 @@ async def check_request_for_export(self, request, action): async def afb_export(self, request): """Exports a report in an AFB format.""" - data = await AFBExporter().export() - return json.dumps(data, indent=4) + report = await self.check_request_for_export(request, "afb-export") + report_id = report[UID] + report_title = report[TITLE] + techniques = await self.data_svc.get_confirmed_techniques_for_afb_export(report_id) + + data = AFBExporter(report_title, techniques).export() + filename = f"{sanitise_filename(report_title)}.afb" + return filename, json.dumps(data, indent=4) async def nav_export(self, request): """Exports a report in a navigator-friendly format.""" diff --git a/threadcomponents/service/data_svc.py b/threadcomponents/service/data_svc.py index c83cf9e..53cb48e 100644 --- a/threadcomponents/service/data_svc.py +++ b/threadcomponents/service/data_svc.py @@ -10,6 +10,7 @@ from contextlib import suppress from copy import deepcopy from datetime import datetime +from threadcomponents.constants import TTP, IOC from urllib.parse import quote # Text to set on attack descriptions where this originally was not set @@ -637,6 +638,25 @@ async def get_unconfirmed_undated_attack_count(self, report_id="", return_detail return unconfirmed_by_sentence if return_detail else count + async def get_confirmed_techniques_for_afb_export(self, report_id): + # The SQL select union query to retrieve the confirmed techniques and IoCs for the afb export + distinct_clause = "DISTINCT ON (tid) " if self.dao.db.IS_POSTGRESQL else "" + select_query = ( + f"SELECT {distinct_clause}" + f"{self.dao.db_qparam} AS type, attack_uid, tid, name, text, NULL AS refanged_text " + "FROM ((report_sentences INNER JOIN report_sentence_hits " + "ON report_sentences.uid = report_sentence_hits.sentence_id) " + "LEFT JOIN attack_uids ON attack_uid = attack_uids.uid) " + f"WHERE report_sentence_hits.report_uid = {self.dao.db_qparam} " + f"AND report_sentence_hits.confirmed = {self.dao.db_true_val} " + f"UNION SELECT {self.dao.db_qparam} AS type, NULL, NULL, NULL, text, refanged_sentence_text " + "FROM (report_sentences INNER JOIN report_sentence_indicators_of_compromise " + "ON report_sentences.uid = report_sentence_indicators_of_compromise.sentence_id) " + f"WHERE report_sentence_indicators_of_compromise.report_id = {self.dao.db_qparam}" + ) + + return await self.dao.raw_select(select_query, parameters=tuple([TTP, report_id, IOC, report_id])) + async def get_confirmed_techniques_for_nav_export(self, report_id): # Ensure date fields are converted into strings start_date = self.dao.db.sql_date_field_to_str( From 9ed6709318d801e3869d8e0166afe12e491d9d3d Mon Sep 17 00:00:00 2001 From: jecarr Date: Fri, 25 Jul 2025 18:53:25 +0100 Subject: [PATCH 3/4] AFB export unit tests --- tests/test_afb_export.py | 237 +++++++++++++++++++++++++++++++++++++++ tests/thread_app_test.py | 25 ++++- 2 files changed, 260 insertions(+), 2 deletions(-) create mode 100644 tests/test_afb_export.py diff --git a/tests/test_afb_export.py b/tests/test_afb_export.py new file mode 100644 index 0000000..a15f719 --- /dev/null +++ b/tests/test_afb_export.py @@ -0,0 +1,237 @@ +import os + +from datetime import datetime +from tests.thread_app_test import ThreadAppTest +from threadcomponents.constants import UID as UID_KEY +from uuid import uuid4 + + +class TestAFBExport(ThreadAppTest): + """A test suite for checking AFB-export of reports.""" + + DB_TEST_FILE = os.path.join("tests", "threadtestafbexport.db") + + async def setUpAsync(self): + await super().setUpAsync() + + self.report_id, self.report_title = str(uuid4()), "Blitzball: A Guide" + await self.submit_test_report( + dict( + uid=self.report_id, + title=self.report_title, + url="lets.blitz", + date_written="2025-07-25", + ), + sentences=[ + ":~$ echo $JECHT_SHOT", + "For more test data, I need to say some random IP addresses.", + "2607[:]f8b0[:]400e[:]c01[::]8a", + "74[.]125[.]199[.]138", + ], + attacks_found=[ + [("d99999", "Drain"), ("f32451", "Firaga"), ("s12345", "Yevon")], + [("d99999", "Drain"), ("f12345", "Fire")], + [], + [], + ], + ) + + self.report_id2, self.report_title2 = str(uuid4()), "Chocobo Racing: A Guide" + await self.submit_test_report( + dict( + uid=self.report_id2, + title=self.report_title2, + url="kw.eh", + date_written="2025-07-25", + ), + sentences=[ + "Kweh.", + "Kweh!?", + ], + attacks_found=[ + [("f32451", "Firaga")], + [], + ], + post_confirm_attack=True, + confirm_attack="f32451", + ) + + await self.confirm_report_attacks() + await self.save_report_iocs() + + async def confirm_report_attacks(self): + """Confirms the attacks in the test-report.""" + await self.confirm_report_sentence_attacks(self.report_id, 0, ["d99999", "s12345"]) + await self.confirm_report_sentence_attacks(self.report_id, 1, ["f12345"]) + + async def save_report_iocs(self): + """Saves the IoCs in the test-report.""" + sentence = await self.db.get("report_sentences", equal=dict(report_uid=self.report_id, sen_index=0)) + await self.client.post( + "/rest", + json=dict( + index="add_indicator_of_compromise", + sentence_id=sentence[0][UID_KEY], + ioc_text="echo $JECHT_SHOT", + ), + ) + + for sentence_index in range(2, 4): + sentence = await self.db.get( + "report_sentences", + equal=dict(report_uid=self.report_id, sen_index=sentence_index), + ) + await self.client.post( + "/rest", + json=dict( + index="suggest_and_save_ioc", + sentence_id=sentence[0][UID_KEY], + ), + ) + + async def get_export_afb_response(self): + """Requests the AFB export and returns the response.""" + return await self.client.get("/export/afb/Blitzball%3A%20A%20Guide") + + async def get_attached_afb_as_json(self): + """Returns the exported AFB as json.""" + response = await self.get_export_afb_response() + return await response.json() + + @staticmethod + def get_entries_from_export(exported, id_value): + """Returns the entries from the exported data given an ID-type.""" + all_objects = exported["objects"] + filtered_list = [] + + for current in all_objects: + if current["id"] == id_value: + filtered_list.append(current) + + return filtered_list + + async def trigger_object_by_id_value_test(self, id_value, expected_properties): + """Given an ID-value in the objects-list, tests the objects for that ID-value are correct.""" + exported = await self.get_attached_afb_as_json() + + retrieved = self.get_entries_from_export(exported, id_value) + expected_count = len(expected_properties) + self.assertEqual(len(retrieved), expected_count, f"{expected_count} x {id_value} entries not set.") + + for current in retrieved: + current_props = dict(current["properties"]) + + for key, value in current_props.items(): + if isinstance(value, list): + current_props[key] = dict(value) + + self.assertTrue(current_props in expected_properties, f"{id_value} object is an unexpected entry.") + + async def test_response_headers(self): + """Tests the response headers reflect an attachment.""" + response = await self.get_export_afb_response() + + self.assertEqual(response.content_disposition.type, "attachment", "Response is not an attachment.") + self.assertEqual( + response.content_disposition.filename, + "Blitzball__A_Guide.afb", + "Attachment's filename is different than expected.", + ) + + async def test_schema(self): + """Tests the schema is correctly set in the export.""" + exported = await self.get_attached_afb_as_json() + self.assertEqual(exported["schema"], "attack_flow_v2", "Schema not correctly set.") + + async def test_camera_object(self): + """Tests the camera-object is correctly set in the export.""" + exported = await self.get_attached_afb_as_json() + camera = exported["camera"] + + camera_fields = sorted(camera.keys()) + self.assertEqual(camera_fields, ["k", "x", "y"], "Camera-object not correctly set.") + + numbers_provided = all([isinstance(_, int) or isinstance(_, float) for _ in camera.values()]) + self.assertTrue(numbers_provided, "Camera-values are not numbers.") + + async def test_flow_object(self): + """Tests the flow-object is correctly set in the export.""" + exported = await self.get_attached_afb_as_json() + + flow_objects = self.get_entries_from_export(exported, "flow") + self.assertEqual(len(flow_objects), 1, "1 x Flow entry not set.") + + properties = dict(flow_objects[0]["properties"]) + self.assertEqual(properties["name"], self.report_title, "Report title not set in export.") + + today = datetime.now() + exported_date = datetime.strptime(properties["created"], "%Y-%m-%dT%H:%M:%S.%fZ") + self.assertEqual(today.date(), exported_date.date(), "Created timestamp is not of today.") + + object_list = sorted(flow_objects[0]["objects"]) + declared_ids = [] + + for current in exported["objects"]: + if current["id"] != "flow": + declared_ids.append(current["instance"]) + + self.assertEqual(object_list, sorted(declared_ids), "Objects-list has incorrect IDs.") + + async def test_anchors_key_declared(self): + """Tests objects from the export have an anchor key.""" + exported = await self.get_attached_afb_as_json() + + for current in exported["objects"]: + if current["id"] != "flow": + self.assertTrue("anchors" in current, "Non-flow entry missing 'anchors' key.") + + async def test_layout_object(self): + """Tests the layout-object is correctly set in the export.""" + exported = await self.get_attached_afb_as_json() + + flow_object = self.get_entries_from_export(exported, "flow")[0] + object_list = sorted(flow_object["objects"]) + layout = exported["layout"] + + layout_keys = sorted(layout.keys()) + self.assertEqual(layout_keys, object_list, "Layout-object not correctly set.") + + layout_values = [ln for l_coords in layout.values() for ln in l_coords] + numbers_provided = all([isinstance(_, int) or isinstance(_, float) for _ in layout_values]) + self.assertTrue(numbers_provided, "Layout-values are not numbers.") + + async def test_ipv6_object_declared(self): + """Tests an IPv6 entry is correctly included in the export.""" + expected = dict(value="2607[:]f8b0[:]400e[:]c01[::]8a") + await self.trigger_object_by_id_value_test("ipv6_addr", [expected]) + + async def test_ipv4_object_declared(self): + """Tests an IPv4 entry is correctly included in the export.""" + expected = dict(value="74[.]125[.]199[.]138") + await self.trigger_object_by_id_value_test("ipv4_addr", [expected]) + + async def test_process_object_declared(self): + """Tests a process-entry is correctly included in the export.""" + expected = dict(command_line=":~$ echo $JECHT_SHOT") + await self.trigger_object_by_id_value_test("process", [expected]) + + async def test_malware_object_declared(self): + """Tests a malware-entry is correctly included in the export.""" + expected = dict(name="Yevon") + await self.trigger_object_by_id_value_test("malware", [expected]) + + async def test_action_objects_declared(self): + """Tests action-entries are correctly included in the export.""" + expected_1 = dict( + name="Drain", + technique_id="T1029", + technique_ref="d99999", + ttp=dict(technique="T1029"), + ) + expected_2 = dict( + name="Fire", + technique_id="T1562", + technique_ref="f12345", + ttp=dict(technique="T1562"), + ) + await self.trigger_object_by_id_value_test("action", [expected_1, expected_2]) diff --git a/tests/thread_app_test.py b/tests/thread_app_test.py index dc0afed..ea679c7 100644 --- a/tests/thread_app_test.py +++ b/tests/thread_app_test.py @@ -104,7 +104,7 @@ def setUpClass(cls): report_exporter = ReportExporter(services=services_with_limit) cls.web_api_with_limit = WebAPI(services=services_with_limit, report_exporter=report_exporter) # Some test-attack data - cls.attacks = dict(d99999="Drain", f12345="Fire", f32451="Firaga", s00001="requiem") + cls.attacks = dict(d99999="Drain", f12345="Fire", f32451="Firaga", s00001="requiem", s12345="Yevon") @classmethod def tearDownClass(cls): @@ -122,10 +122,13 @@ async def setUpAsync(self): attack_2 = dict(uid="f32451", tid="T1562.004", name=self.attacks.get("f32451")) attack_3 = dict(uid="d99999", tid="T1029", name=self.attacks.get("d99999")) attack_4 = dict(uid="s00001", tid="T1485", name=self.attacks.get("s00001"), inactive=1) - for attack in [attack_1, attack_2, attack_3, attack_4]: + attack_5 = dict(uid="s12345", tid="S1030", name=self.attacks.get("s12345")) + + for attack in [attack_1, attack_2, attack_3, attack_4, attack_5]: # Ignoring Integrity Error in case other test case already has inserted this data (causing duplicate UIDs) with suppress(sqlite3.IntegrityError): await self.db.insert("attack_uids", attack) + # Insert some category, keyword & country data cat_1 = dict(uid="c010101", keyname="aerospace", name="Aerospace") cat_2 = dict(uid="c123456", keyname="music", name="Music") @@ -134,11 +137,13 @@ async def setUpAsync(self): group_2 = dict(uid="apt2", name="APT2") group_3 = dict(uid="apt3", name="APT3") self.data_svc.country_dict = dict(HB="Hobbiton", TA="Tatooine", WA="Wakanda") + for cat, group in [(cat_1, group_1), (cat_2, group_2), (cat_3, group_3)]: with suppress(sqlite3.IntegrityError): await self.db.insert("categories", cat) await self.db.insert("keywords", group) self.web_svc.categories_dict[cat["keyname"]] = dict(name=cat["name"], sub_categories=[]) + # Carry out pre-launch tasks except for prepare_queue(): replace the call of this to return (and do) nothing # We don't want multiple prepare_queue() calls so the queue does not accumulate between tests with patch.object(RestService, "prepare_queue", return_value=None): @@ -161,6 +166,7 @@ async def get_application(self): app.router.add_route("GET", self.web_svc.get_route(WebService.EDIT_KEY), self.web_api.edit) app.router.add_route("GET", self.web_svc.get_route(WebService.ABOUT_KEY), self.web_api.about) app.router.add_route("GET", self.web_svc.get_route(WebService.HOW_IT_WORKS_KEY), self.web_api.how_it_works) + app.router.add_route("GET", self.web_svc.get_route(WebService.EXPORT_AFB_KEY), self.web_api.afb_export) app.router.add_route("*", self.web_svc.get_route(WebService.REST_KEY), self.web_api.rest_api) # A different route for limit-testing app.router.add_route( @@ -245,6 +251,7 @@ async def submit_test_report( await self.db.insert("reports", report) # Mock the analysis of the report await self.rest_svc.start_analysis(criteria=report) + if post_confirm_attack: # Get the report sentences for this report db_sentences = await self.db.get("report_sentences", equal=dict(report_uid=report[UID_KEY])) @@ -259,6 +266,20 @@ async def submit_test_report( "/rest", json=dict(index="add_attack", sentence_id=sen_id, attack_uid=confirm_attack) ) + async def confirm_report_sentence_attacks(self, report_id, sentence_index, attack_list): + """Accepts a list of attacks for a given test-sentence index.""" + sentence = await self.db.get("report_sentences", equal=dict(report_uid=report_id, sen_index=sentence_index)) + + for accept in attack_list: + await self.client.post( + "/rest", + json=dict( + index="add_attack", + sentence_id=sentence[0][UID_KEY], + attack_uid=accept, + ), + ) + def mock_current_attack_data(self, attack_list=None): """Helper-method to mock the retrieval of the current Att%ck data by returning a specified attack-list.""" attack_list = attack_list or [] From 5740b2d97711873307479641b21217fa865d17b4 Mon Sep 17 00:00:00 2001 From: jecarr Date: Mon, 28 Jul 2025 10:13:50 +0100 Subject: [PATCH 4/4] Updated Spindle - Merged arachne-threat-intel/spindle#40 - Merged arachne-threat-intel/spindle#41 --- spindle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spindle b/spindle index cb5016a..5616218 160000 --- a/spindle +++ b/spindle @@ -1 +1 @@ -Subproject commit cb5016a7aeabc987d0669f81d5bd598c1adf39fc +Subproject commit 5616218ef615ba5ac1f7fb12adf452dfa7fad71c