diff --git a/plugins/experimental/ja4_fingerprint/plugin.cc b/plugins/experimental/ja4_fingerprint/plugin.cc index ada54ed1c98..16419d45668 100644 --- a/plugins/experimental/ja4_fingerprint/plugin.cc +++ b/plugins/experimental/ja4_fingerprint/plugin.cc @@ -31,11 +31,13 @@ #include #include +#include #include #include #include #include +#include #include #include #include @@ -79,8 +81,37 @@ constexpr int SSL_SUCCESS{1}; DbgCtl dbg_ctl{PLUGIN_NAME}; +int global_preserve_enabled{0}; + } // end anonymous namespace +static bool +read_config_option(int argc, char const *argv[], int &preserve) +{ + const struct option longopts[] = { + {"preserve", no_argument, &preserve, 1}, + {nullptr, 0, nullptr, 0} + }; + + optind = 0; + int opt{0}; + while ((opt = getopt_long(argc, const_cast(argv), "", longopts, nullptr)) >= 0) { + switch (opt) { + case '?': + Dbg(dbg_ctl, "Unrecognized command argument."); + case 0: + case -1: + break; + default: + Dbg(dbg_ctl, "Unexpected options error."); + return false; + } + } + + Dbg(dbg_ctl, "JA4 preserve is %s", (preserve == 1) ? "enabled" : "disabled"); + return true; +} + static int * get_user_arg_index() { @@ -112,12 +143,16 @@ make_word(unsigned char lowbyte, unsigned char highbyte) } void -TSPluginInit(int /* argc ATS_UNUSED */, char const ** /* argv ATS_UNUSED */) +TSPluginInit(int argc, char const **argv) { if (!register_plugin()) { TSError("[%s] Failed to register.", PLUGIN_NAME); return; } + if (!read_config_option(argc, argv, global_preserve_enabled)) { + TSError("[%s] Failed to parse options.", PLUGIN_NAME); + return; + } reserve_user_arg(); if (!create_log_file()) { TSError("[%s] Failed to create log.", PLUGIN_NAME); @@ -334,12 +369,36 @@ handle_read_request_hdr(TSCont cont, TSEvent event, void *edata) return TS_SUCCESS; } +// Check if a header field exists in the request. +static bool +header_exists(TSMBuffer bufp, TSMLoc hdr_loc, char const *field, int field_len) +{ + TSMLoc loc = TSMimeHdrFieldFind(bufp, hdr_loc, field, field_len); + if (loc != TS_NULL_MLOC) { + TSHandleMLocRelease(bufp, hdr_loc, loc); + return true; + } + return false; +} + void append_JA4_headers(TSCont /* cont ATS_UNUSED */, TSHttpTxn txnp, std::string const *fingerprint) { TSMBuffer bufp; TSMLoc hdr_loc; - if (TS_SUCCESS == TSHttpTxnClientReqGet(txnp, &bufp, &hdr_loc)) { + if (TS_SUCCESS != TSHttpTxnClientReqGet(txnp, &bufp, &hdr_loc)) { + Dbg(dbg_ctl, "Failed to get headers."); + return; + } + + // When preserve is enabled, check if ANY JA4 header exists. If so, skip + // adding ALL JA4 headers to avoid mismatched fingerprint data when requests + // traverse multiple proxies. + bool const ja4_header_exists = header_exists(bufp, hdr_loc, "ja4", 3) || + header_exists(bufp, hdr_loc, JA4_VIA_HEADER.data(), static_cast(JA4_VIA_HEADER.length())); + bool const skip_ja4_headers = global_preserve_enabled && ja4_header_exists; + + if (!skip_ja4_headers) { append_to_field(bufp, hdr_loc, "ja4", 3, fingerprint->data(), fingerprint->size()); TSMgmtString proxy_name = nullptr; @@ -351,9 +410,6 @@ append_JA4_headers(TSCont /* cont ATS_UNUSED */, TSHttpTxn txnp, std::string con append_to_field(bufp, hdr_loc, JA4_VIA_HEADER.data(), static_cast(JA4_VIA_HEADER.length()), proxy_name, static_cast(std::strlen(proxy_name))); TSfree(proxy_name); - - } else { - Dbg(dbg_ctl, "Failed to get headers."); } TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); diff --git a/plugins/ja3_fingerprint/ja3_fingerprint.cc b/plugins/ja3_fingerprint/ja3_fingerprint.cc index 5a9bb2ce768..4f9f3028cf7 100644 --- a/plugins/ja3_fingerprint/ja3_fingerprint.cc +++ b/plugins/ja3_fingerprint/ja3_fingerprint.cc @@ -194,6 +194,18 @@ append_to_field(TSMBuffer bufp, TSMLoc hdr_loc, const char *field, int field_len TSHandleMLocRelease(bufp, hdr_loc, target); } +// Check if a header field exists in the request. +static bool +header_exists(TSMBuffer bufp, TSMLoc hdr_loc, char const *field, int field_len) +{ + TSMLoc loc = TSMimeHdrFieldFind(bufp, hdr_loc, field, field_len); + if (loc != TS_NULL_MLOC) { + TSHandleMLocRelease(bufp, hdr_loc, loc); + return true; + } + return false; +} + static ja3_data * create_ja3_data(TSVConn const ssl_vc) { @@ -258,23 +270,31 @@ modify_ja3_headers(TSCont contp, TSHttpTxn txnp, ja3_data const *ja3_vconn_data) TSAssert(TS_SUCCESS == TSHttpTxnServerReqGet(txnp, &bufp, &hdr_loc)); } - TSMgmtString proxy_name = nullptr; - if (TS_SUCCESS != TSMgmtStringGet("proxy.config.proxy_name", &proxy_name)) { - TSError("[%s] Failed to get proxy name for %s, set 'proxy.config.proxy_name' in records.config", PLUGIN_NAME, - JA3_VIA_HEADER.data()); - proxy_name = TSstrdup("unknown"); - } - append_to_field(bufp, hdr_loc, JA3_VIA_HEADER.data(), static_cast(JA3_VIA_HEADER.length()), proxy_name, - static_cast(std::strlen(proxy_name)), preserve_flag); - TSfree(proxy_name); + // When preserve is enabled, check if ANY JA3 header exists. If so, skip + // adding ALL JA3 headers to avoid mismatched fingerprint data when requests + // traverse multiple proxies. + bool const ja3_header_exists = header_exists(bufp, hdr_loc, JA3_VIA_HEADER.data(), static_cast(JA3_VIA_HEADER.length())) || + header_exists(bufp, hdr_loc, "x-ja3-sig", 9) || header_exists(bufp, hdr_loc, "x-ja3-raw", 9); + bool const skip_ja3_headers = preserve_flag && ja3_header_exists; + + if (!skip_ja3_headers) { + TSMgmtString proxy_name = nullptr; + if (TS_SUCCESS != TSMgmtStringGet("proxy.config.proxy_name", &proxy_name)) { + TSError("[%s] Failed to get proxy name for %s, set 'proxy.config.proxy_name' in records.config", PLUGIN_NAME, + JA3_VIA_HEADER.data()); + proxy_name = TSstrdup("unknown"); + } + append_to_field(bufp, hdr_loc, JA3_VIA_HEADER.data(), static_cast(JA3_VIA_HEADER.length()), proxy_name, + static_cast(std::strlen(proxy_name)), false); - // Add JA3 md5 fingerprints - append_to_field(bufp, hdr_loc, "x-ja3-sig", 9, ja3_vconn_data->md5_string, 32, preserve_flag); + // Add JA3 md5 fingerprints. + append_to_field(bufp, hdr_loc, "x-ja3-sig", 9, ja3_vconn_data->md5_string, 32, false); - // If raw string is configured, added JA3 raw string to header as well - if (raw_flag) { - append_to_field(bufp, hdr_loc, "x-ja3-raw", 9, ja3_vconn_data->ja3_string.data(), ja3_vconn_data->ja3_string.size(), - preserve_flag); + // If raw string is configured, add JA3 raw string to header as well. + if (raw_flag) { + append_to_field(bufp, hdr_loc, "x-ja3-raw", 9, ja3_vconn_data->ja3_string.data(), ja3_vconn_data->ja3_string.size(), false); + } + TSfree(proxy_name); } TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); diff --git a/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py b/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py index e94c82196ab..8ec06814310 100644 --- a/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py +++ b/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint.test.py @@ -81,13 +81,18 @@ def _configure_server(self, tr: 'TestRun') -> None: self._server.Streams.All += Testers.ContainsExpression("https-request", "Verify the HTTPS request was received.") self._server.Streams.All += Testers.ContainsExpression("http2-request", "Verify the HTTP/2 request was received.") if not self._test_remap: - # Verify --preserve worked. - self._server.Streams.All += Testers.ContainsExpression("x-ja3-raw: .*,", "Verify the new raw header was added.") + # The first request has no existing JA3 headers, so headers are added. self._server.Streams.All += Testers.ContainsExpression( - "x-ja3-raw: first-signature", "Verify the already-existing raw header was preserved.") - self._server.Streams.All += Testers.ExcludesExpression( - "x-ja3-raw: first-signature;", "Verify no extra values were added due to preserve.") + "x-ja3-raw: .*,", "Verify the new raw header was added.", reflags=re.IGNORECASE) self._server.Streams.All += Testers.ContainsExpression("x-ja3-via: test.proxy.com", "The x-ja3-via string was added.") + # The second request has existing JA3 headers. With --preserve, + # no new JA3 headers are added (including x-ja3-sig). + self._server.Streams.All += Testers.ContainsExpression( + "x-ja3-raw: first-signature", "Verify the already-existing raw header was preserved.", reflags=re.IGNORECASE) + self._server.Streams.All += Testers.ExcludesExpression( + "x-ja3-sig:.*http2-request", + "Verify no JA3-Sig was added when other JA3 headers existed.", + reflags=re.IGNORECASE | re.DOTALL) def _configure_trafficserver(self) -> None: """Configure Traffic Server to be used in the test.""" @@ -189,8 +194,10 @@ def _verify_internal_headers(self) -> None: if self._modify_incoming: p.Streams.All += "modify-incoming-proxy.gold" + elif self._test_remap: + p.Streams.All += "modify-sent-proxy-remap.gold" else: - p.Streams.All += "modify-sent-proxy.gold" + p.Streams.All += "modify-sent-proxy-global.gold" JA3FingerprintTest(test_remap=False, modify_incoming=False) diff --git a/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint_global.replay.yaml b/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint_global.replay.yaml index bd42ddd9cc1..f0c14528716 100644 --- a/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint_global.replay.yaml +++ b/tests/gold_tests/pluginTest/ja3_fingerprint/ja3_fingerprint_global.replay.yaml @@ -86,13 +86,14 @@ sessions: content: size: 399 + # With --preserve and any JA3 header present, no new headers are added. proxy-request: headers: fields: - [ x-request, { value: 'http2-request', as: equal } ] - [ x-ja3-via, { value: 'first-via', as: equal } ] - - [ X-JA3-Sig, { as: present } ] - - [ X-JA3-Raw, { as: present } ] + - [ x-ja3-raw, { value: 'first-signature', as: equal } ] + - [ x-ja3-sig, { as: absent } ] server-response: headers: diff --git a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold index a8cd3a41bb7..480b97c39db 100644 --- a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold +++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-incoming-proxy.gold @@ -2,5 +2,6 @@ -- State Machine Id`` POST /some/path/http2`` `` -x-ja3-sig: `` +x-ja3-raw: first-signature`` +x-ja3-via: first-via`` `` diff --git a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy-global.gold b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy-global.gold new file mode 100644 index 00000000000..480b97c39db --- /dev/null +++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy-global.gold @@ -0,0 +1,7 @@ ++++++++++ Proxy's Request after hooks +++++++++ +-- State Machine Id`` +POST /some/path/http2`` +`` +x-ja3-raw: first-signature`` +x-ja3-via: first-via`` +`` diff --git a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy.gold b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy-remap.gold similarity index 89% rename from tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy.gold rename to tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy-remap.gold index a8cd3a41bb7..271809823d7 100644 --- a/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy.gold +++ b/tests/gold_tests/pluginTest/ja3_fingerprint/modify-sent-proxy-remap.gold @@ -3,4 +3,5 @@ POST /some/path/http2`` `` x-ja3-sig: `` +x-ja3-via: `` `` diff --git a/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.replay.yaml b/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.replay.yaml index f22de8c76d9..5c0309f15a3 100644 --- a/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.replay.yaml +++ b/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.replay.yaml @@ -18,6 +18,8 @@ meta: version: "1.0" sessions: + +# Session 1: No pre-existing JA4 headers - new headers should be added. - protocol: - name: http version: 1 @@ -33,11 +35,12 @@ sessions: fields: - [ Connection, keep-alive ] - [ Content-Length, 0 ] + - [ uuid, no-existing-headers ] proxy-request: headers: fields: - - [ ja4, { as: contains } ] + - [ ja4, { as: present } ] - [ x-ja4-via, { value: 'test.proxy.com', as: equal } ] server-response: @@ -46,3 +49,70 @@ sessions: content: encoding: plain data: Yay! + +# Session 2: Pre-existing JA4 headers - with preserve, no new headers added. +- protocol: + - name: http + version: 1 + - name: tcp + - name: ip + + transactions: + - client-request: + method: "GET" + version: "1.1" + url: /resource-with-headers + headers: + fields: + - [ Connection, keep-alive ] + - [ Content-Length, 0 ] + - [ uuid, existing-ja4-headers ] + - [ ja4, upstream-fingerprint ] + - [ x-ja4-via, upstream.proxy.com ] + + # With --preserve and existing JA4 headers, no new headers should be added. + proxy-request: + headers: + fields: + - [ ja4, { value: 'upstream-fingerprint', as: equal } ] + - [ x-ja4-via, { value: 'upstream.proxy.com', as: equal } ] + + server-response: + status: 200 + reason: OK + content: + encoding: plain + data: Preserved! + +# Session 3: Only x-ja4-via exists - should still trigger preserve for all. +- protocol: + - name: http + version: 1 + - name: tcp + - name: ip + + transactions: + - client-request: + method: "GET" + version: "1.1" + url: /resource-via-only + headers: + fields: + - [ Connection, keep-alive ] + - [ Content-Length, 0 ] + - [ uuid, existing-via-only ] + - [ x-ja4-via, upstream.proxy.com ] + + # With --preserve and only x-ja4-via present, no JA4 headers should be added. + proxy-request: + headers: + fields: + - [ ja4, { as: absent } ] + - [ x-ja4-via, { value: 'upstream.proxy.com', as: equal } ] + + server-response: + status: 200 + reason: OK + content: + encoding: plain + data: Via only! diff --git a/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.test.py b/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.test.py index ea241579df0..2de2429faca 100644 --- a/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.test.py +++ b/tests/gold_tests/pluginTest/ja4_fingerprint/ja4_fingerprint.test.py @@ -38,13 +38,15 @@ class TestJA4Fingerprint: server_counter: int = 0 ts_counter: int = 0 - def __init__(self, name: str, /, autorun: bool) -> None: + def __init__(self, name: str, /, autorun: bool, use_preserve: bool = False) -> None: '''Initialize the test. :param name: The name of the test. + :param use_preserve: Whether to use the --preserve flag. ''' self.name = name self.autorun = autorun + self.use_preserve = use_preserve def _init_run(self) -> 'TestRun': '''Initialize processes for the test run.''' @@ -127,7 +129,10 @@ def _configure_traffic_server(self, server_one: 'Process'): ts.Disk.ssl_multicert_config.AddLine(f'dest_ip=* ssl_cert_name=server.pem ssl_key_name=server.key') - ts.Disk.plugin_config.AddLine(f'ja4_fingerprint.so') + plugin_args = 'ja4_fingerprint.so' + if self.use_preserve: + plugin_args += ' --preserve' + ts.Disk.plugin_config.AddLine(plugin_args) log_path = os.path.join(ts.Variables.LOGDIR, "ja4_fingerprint.log") ts.Disk.File(log_path, id='log_file') @@ -152,3 +157,43 @@ def test1(params: TestParams) -> None: client.Streams.stdout += Testers.ContainsExpression(r'Yay!', 'We should receive the expected body.') params['ts'].Disk.traffic_out.Content += Testers.ContainsExpression( r'JA4 fingerprint:', 'We should receive the expected log message.') + + +@TestJA4Fingerprint.runner('With --preserve, existing JA4 headers should be preserved.', use_preserve=True) +def test_preserve(params: TestParams) -> None: + '''Test that --preserve skips adding headers when JA4 headers exist.''' + tr = params['tr'] + client = tr.Processes.Default + server = params['server_one'] + + # Request 1: No existing JA4 headers - should add them. + tr.MakeCurlCommand('-k -v "https://localhost:{0}/resource"'.format(params['port_one']), ts=params['ts']) + client.ReturnCode = 0 + client.Streams.stdout += Testers.ContainsExpression(r'Yay!', 'First request should succeed.') + + # Request 2: With existing JA4 headers - should preserve them. + tr2 = Test.AddTestRun('Verify preserve skips adding JA4 headers when they exist.') + tr2.MakeCurlCommand( + '-k -v -H "ja4: upstream-fingerprint" -H "x-ja4-via: upstream.proxy.com" ' + '"https://localhost:{0}/resource-with-headers"'.format(params['port_one']), + ts=params['ts']) + tr2.Processes.Default.ReturnCode = 0 + tr2.Processes.Default.Streams.stdout += Testers.ContainsExpression(r'Preserved!', 'Second request should preserve headers.') + + # Request 3: With only x-ja4-via - should also trigger preserve. + tr3 = Test.AddTestRun('Verify preserve triggers when only x-ja4-via exists.') + tr3.MakeCurlCommand( + '-k -v -H "x-ja4-via: upstream.proxy.com" ' + '"https://localhost:{0}/resource-via-only"'.format(params['port_one']), + ts=params['ts']) + tr3.Processes.Default.ReturnCode = 0 + tr3.Processes.Default.Streams.stdout += Testers.ContainsExpression(r'Via only!', 'Third request should succeed.') + + # Verify the replay file's proxy-request validations pass (checked per-request). + # Also verify via Proxy Verifier's validation logs. + server.Streams.All += Testers.ContainsExpression( + r'Equals Success.*"/resource-with-headers".*ja4.*upstream-fingerprint', + 'Preserved ja4 header should match original value.', + reflags=re.DOTALL) + server.Streams.All += Testers.ContainsExpression( + r'Absence Success.*"/resource-via-only".*ja4', 'ja4 should be absent when only x-ja4-via exists.', reflags=re.DOTALL)