Fix zeek ja4ssh add tests

.github/workflows/zeek-test.yml

@@ -0,0 +1,36 @@
+name: Zeek Tests
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'zeek/**'
+      - '.github/workflows/zeek-test.yml'
+  pull_request:
+    paths:
+      - 'zeek/**'
+      - '.github/workflows/zeek-test.yml'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+    container:
+      image: zeek/zeek:8.0.0
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install dependencies
+        run: |
+          apt-get update
+          apt-get install -y python3-pip git
+          # btest is distributed as a python package
+          pip3 install btest --break-system-packages
+
+      - name: Run btest
+        run: |
+          cd zeek/tests
+          btest -d Scripts/

zeek/ja4ssh/main.zeek

@@ -66,6 +66,8 @@ function get_mode(vec: vector of count): count {
     if (freq > max) {
       max = freq;
       mode = idx;
+    } else if (freq == max && idx < mode) {
+      mode = idx;
     }
   }
 
@@ -117,16 +119,20 @@ event ConnThreshold::packets_threshold_crossed(c: connection, threshold: count,
     if (is_orig) {
         ConnThreshold::set_packets_threshold(c,threshold + 1,T);
 
-        if (rp$tcp$dl == 0) {
-          ++c$fp$ja4ssh$orig_ack;
+        if ((rp$tcp$dl == 0) ) {
+          if (rp$tcp$flags == 0x10) {
+            ++c$fp$ja4ssh$orig_ack;
+          }
         } else {
           c$fp$ja4ssh$orig_pack_len += rp$tcp$dl;
         }
     } else {
         ConnThreshold::set_packets_threshold(c,threshold + 1,F);
 
-        if (rp$tcp$dl == 0) {
-          ++c$fp$ja4ssh$resp_ack;
+        if ((rp$tcp$dl == 0) ) {
+          if (rp$tcp$flags == 0x10) {
+            ++c$fp$ja4ssh$resp_ack;
+          }
         } else {
           c$fp$ja4ssh$resp_pack_len += rp$tcp$dl;
         }

zeek/tests/Scripts/ja4-conn.zeek

@@ -0,0 +1,2 @@
+# @TEST-EXEC: zeek -C -r ${TRACES}/ipv6.pcapng ../../../__load__.zeek %INPUT
+# @TEST-EXEC: btest-diff conn.log

zeek/tests/Scripts/ja4-dhcp.zeek

@@ -0,0 +1,2 @@
+# @TEST-EXEC: zeek -C -r ${TRACES}/dhcp.pcapng ../../../__load__.zeek %INPUT
+# @TEST-EXEC: btest-diff ja4d.log

zeek/tests/Scripts/ja4-http1-with-cookies.zeek

@@ -0,0 +1,2 @@
+# @TEST-EXEC: zeek -C -r ${TRACES}/http1-with-cookies.pcapng ../../../__load__.zeek %INPUT
+# @TEST-EXEC: btest-diff http.log

zeek/tests/Scripts/ja4-ssh2.zeek

@@ -0,0 +1,2 @@
+# @TEST-EXEC: zeek -C -r ${TRACES}/ssh2.pcapng ../../../__load__.zeek %INPUT
+# @TEST-EXEC: btest-diff ja4ssh.log

zeek/tests/Scripts/ja4-tls-handshake.zeek

@@ -0,0 +1,2 @@
+# @TEST-EXEC: zeek -C -r ${TRACES}/tls-handshake.pcapng ../../../__load__.zeek %INPUT
+# @TEST-EXEC: btest-diff ssl.log

zeek/tests/Traces/Scripts.ja4-conn/conn.log

@@ -0,0 +1,9 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	local_resp	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents	ip_proto	ja4l	ja4ls	ja4t	ja4ts
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	bool	count	string	count	count	count	count	set[string]	count	string	string	string	string
+1593980608.621087	C73Zcd4XqUIenT3Fef	2001:4998:ef83:14:8000::100d	64034	2606:4700::6811:d209	443	tcp	ssl	0.075341	237	2810	S1	F	F	0	ShADad	4	501	5	3122	-	6	35_64	18862_59_14792	65535__0_0	65535__0_0

zeek/tests/Traces/Scripts.ja4-dhcp/ja4d.log

@@ -0,0 +1,12 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ja4d
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	ja4d	client_mac	requested_ip	vendor_class_id	hostname
+#types	time	string	addr	port	addr	port	string	string	addr	string	string
+1102274184.317453	C73Zcd4XqUIenT3Fef	0.0.0.0	68	255.255.255.255	67	disco0000in_61-55_1-3-6-42	00:0b:82:01:fc:42	0.0.0.0	(empty)	(empty)
+1102274184.317748	Cl5woUpqKxGTMghIc	192.168.0.10	68	192.168.0.1	67	offer0000nn_1-58-59-51-54_00	00:0b:82:01:fc:42	-	(empty)	(empty)
+1102274184.317453	C73Zcd4XqUIenT3Fef	0.0.0.0	68	255.255.255.255	67	reqst0000in_61-54-55_1-3-6-42	00:0b:82:01:fc:42	192.168.0.10	(empty)	(empty)
+1102274184.317748	Cl5woUpqKxGTMghIc	192.168.0.10	68	192.168.0.1	67	dpack0000nn_58-59-51-54-1_00	00:0b:82:01:fc:42	-	(empty)	(empty)

zeek/tests/Traces/Scripts.ja4-http1-with-cookies/http.log

@@ -0,0 +1,9 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	http
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	version	user_agent	origin	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	tags	username	password	proxied	orig_fuids	orig_filenames	orig_mime_types	resp_fuids	resp_filenames	resp_mime_types	ja4h
+#types	time	string	addr	port	addr	port	count	string	string	string	string	string	string	string	count	count	count	string	count	string	set[enum]	string	string	set[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	vector[string]	string
+1690483736.912889	C73Zcd4XqUIenT3Fef	127.0.0.1	61256	127.0.0.1	8000	1	GET	localhost:8000	/	https://fake.example	1.0	curl/8.1.2	-	0	1114	200	OK	-	-	(empty)	-	-	-	-	-	-	FvbTb11CCpKvM264U8	-	text/html	ge11cr04da00_8ddaef5d77af_280f366eaa04_c2fb0fe53442

zeek/tests/Traces/Scripts.ja4-ssh2/ja4ssh.log

@@ -0,0 +1,10 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ja4ssh
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	ja4ssh
+#types	time	string	addr	port	addr	port	string
+1688672868.776057	Chu4srlj1wba0b8Fk	172.16.225.48	57377	54.160.114.75	22	c36s36_c76s124_c74s5
+1688672868.776057	Chu4srlj1wba0b8Fk	172.16.225.48	57377	54.160.114.75	22	c36s52_c42s76_c51s2

zeek/tests/Traces/Scripts.ja4-tls-handshake/ssl.log

@@ -0,0 +1,28 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssl
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	version	cipher	curve	server_name	resumed	last_alert	next_protocol	established	ssl_history	cert_chain_fps	client_cert_chain_fps	sni_matches_cert	ja4	ja4s
+#types	time	string	addr	port	addr	port	string	string	string	string	bool	string	string	bool	string	vector[string]	vector[string]	bool	string	string
+1675707161.245230	CGBsqN2uXmDTPVzDEl	192.168.1.168	52166	31.13.66.35	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	www.facebook.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_a56c5b993250
+1675707152.011673	CErRvO2gbIalp3tOc5	192.168.1.168	55945	142.251.16.95	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	www.googleapis.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707155.389827	ChIaVxLkUs61KxAz8	192.168.1.168	55906	142.251.16.100	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	ogs.google.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707155.300029	C2FwSz1eKJKOYWy4K3	192.168.1.168	60486	142.251.111.101	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	history.google.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707155.391962	C50XAHll41sdt2ft7	192.168.1.168	61836	142.251.111.101	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	aa.google.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707153.800775	CfIcQD2Ei8f1el2PJf	192.168.1.168	49333	142.251.163.94	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	update.googleapis.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707163.847611	CxgSZ03njQU3vGwSdi	192.168.1.168	50053	142.251.163.95	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	optimizationguide-pa.googleapis.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707161.164729	CKWJ9iIqGVdwNGbvc	192.168.1.168	51782	142.251.163.95	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	safebrowsing.googleapis.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707163.742632	CIbjXu4xSHdPAYPjA9	192.168.1.168	55309	142.251.163.95	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	content-autofill.googleapis.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707154.505201	CnWgof2NddogII3Yia	192.168.1.168	57662	142.251.163.102	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	encrypted-tbn0.gstatic.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707171.303486	C5rYFO1qSQXNJm9dEa	192.168.1.168	57188	142.251.163.132	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	lh5.googleusercontent.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707158.762270	CXRgQm10C8g8cktdTe	192.168.1.168	51905	142.251.163.138	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	encrypted-tbn0.gstatic.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707151.936380	Cl5woUpqKxGTMghIc	192.168.1.168	59102	142.251.163.147	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	www.google.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707161.165864	Co1c5e3TdbKAJB3ldg	192.168.1.168	54980	157.240.229.1	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	static.xx.fbcdn.net	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_a56c5b993250
+1675707161.269152	COaaiw3h19kSgb1BO6	192.168.1.168	59444	157.240.229.1	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	static.xx.fbcdn.net	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_a56c5b993250
+1675707161.165564	COQYXVPH0daDqWk2	192.168.1.168	61730	157.240.229.1	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	scontent-iad3-2.xx.fbcdn.net	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_a56c5b993250
+1675707163.575152	CVIuI3mAyVewLEUf	192.168.1.168	60374	157.240.229.2	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	video-iad3-2.xx.fbcdn.net	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_a56c5b993250
+1675707155.429145	CYq93o1jzMmpzZkM6d	192.168.1.168	59221	172.253.63.95	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	content-autofill.googleapis.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707151.936708	CN3K5p4CkjEhKRbQSg	192.168.1.168	57816	172.253.122.84	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	accounts.google.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581
+1675707155.341160	CMGtLttXbs1Yadtzc	192.168.1.168	60845	172.253.122.94	443	TLSv13	TLS_AES_128_GCM_SHA256	x25519	www.gstatic.com	F	-	-	F	Cs	-	-	-	q13d0310h3_55b375c5d22e_cd85d2d88918	q130200_1301_234ea6891581

zeek/tests/btest.cfg

@@ -0,0 +1,16 @@
+[btest]
+TestDirs=Scripts
+TmpDir=%(testbase)s/.tmp
+BaselineDir=%(testbase)s/Traces
+IgnoreDirs=.svn CVS .git .tmp
+IgnoreFiles=*.tmp *.swp #* *.trace .DS_Store
+
+[environment]
+ZEEK_SEED_FILE=%(testbase)s/random.seed
+TZ=UTC
+TEST_DIFF_CANONIFIER=awk '{ if ($1 == "#open" || $1 == "#close") next; print $0 }'
+LC_ALL=C
+TRACES=%(testbase)s/../../pcap
+BROPATH=`zeek-config --zeekpath`:%(testbase)s/..
+ZEEKPATH=`zeek-config --zeekpath`:%(testbase)s/..
+ZEEK_PLUGIN_PATH=`zeek-config --plugin_dir`

zeek/tests/random.seed

@@ -0,0 +1,21 @@
+1907645451
+2542257355
+2690548683
+2465255630
+3074807866
+4232382387
+2004422659
+2917066712
+2706037395
+2004411388
+2230090280
+4142033565
+1001624462
+1613373517
+3901537915
+1566829475
+1593044761
+3743307041
+2274259402
+2761772015
+2243476871