statusCodeCoverage.py

import json
from pathlib import Path
from collections import defaultdict
import argparse
import re
#HTTP Methoden definieren
HTTP_METHODS = {
    "get",
    "post",
    "put",
    "delete",
    "patch",
    "head",
    "options"
    }

# Parser für Argumente: Pfad zur Swagger.json und generated_tests/{aktueller Test}
parser = argparse.ArgumentParser(
    description="Calculate status code coverage based on Swagger and generated EvoMaster tests."
)

parser.add_argument(
    "--swagger",
    required=True,
    type=Path,
    help="Pfad zur swagger.json Datei"
    )

parser.add_argument(
    "--tests",
    required=True,
    type=Path,
    help="Path to the generated EvoMaster test folder"
)
parser.add_argument(
    "--debug",
    action="store_true",
    help="To Print more information about the found status codes"
)
args = parser.parse_args()

# Pfade zur swagger.json und zu den generierten Testdaten
swagger_path = args.swagger


test_files = list(args.tests.rglob("coveredTargets.txt"))

def normalize_path(path):
    """
    Entfernt das Präfix /api/v1, damit Swagger- und
    EvoMaster-Pfade dasselbe Format verwenden.
    """

    path = path.strip()

    if path.startswith("/api/v1/"):
        return path[len("/api/v1"):]

    if path == "/api/v1":
        return "/"

    return path

def get_defined_status_codes():
    """
    Funktion zum Auslesen der definierten Status Codes in der OAS JSON-Datei
    """
    with open(swagger_path, "r", encoding="utf-8") as f:
        swagger = json.load(f)
    # zum speichern der codes
    result = {}
    # durch die JSON iterieren
    for path, path_item in swagger.get("paths", {}).items():
        for method, operation in path_item.items():
            method_lowercase = method.lower()
            if method_lowercase not in HTTP_METHODS:
                continue
            responses = operation.get("responses", {})
            codes = set()
            
            for code in responses.keys():
                code = str(code)
                if re.fullmatch(r"\d{3}", code):
                    codes.add(code)

            result[(method_lowercase.upper(), normalize_path(path))] = {
                "defined_codes": codes
            }

    return result
            


def get_evomaster_status_codes():
    """
    Funktion zum Auslesen der von EvoMaster "gefundenen" Status Codes durch den Test
    """
    observed = defaultdict(set)
    # Regex: # (401) GET:/api/v1/repos/{owner}/{repo}/pulls
    call_pattern = re.compile(
        r"^([1-5]\d{2}):"
        r"(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS):"
        r"(.+)$",
        re.IGNORECASE,
    )

    for file in test_files:
        lines = file.read_text(
            encoding="utf-8",
            errors="ignore"
        ).splitlines()

        for line in lines:
            match = call_pattern.fullmatch(line.strip())

            if not match:
                continue

            status_code = match.group(1)
            method = match.group(2).upper()
            path = match.group(3).strip()
            path = normalize_path(path)
            
            observed[(method, path)].add(status_code)

    return observed

def calculate_coverage(defined, evomDisc):
    """
    Berechnet die Status-Code-Coverage aus der OAS und den entdeckten Status-Codes

    defined:
    status_code:
    """
    defined_combinations = set()
    for (method, path), information in defined.items():
        for status_code in information["defined_codes"]:
            defined_combinations.add(
                (method, path, status_code)
            )
    observed_combinations = set()
    for (method, path), status_codes in evomDisc.items():
        for status_code in status_codes:
            observed_combinations.add(
                (method, path, status_code)
            )
    all_possible_combinations = (
        defined_combinations | observed_combinations
    )
    coverage = (
        len(observed_combinations) / len(all_possible_combinations)
    )

    return coverage
# For test purposes:
def print_status_code_examples(data, name, is_defined=False, limit=10):
    """
    Gruppiert die Kombinationen nach Statuscode und zeigt pro
    Statuscode maximal 'limit' Beispiele.

    data:
        defined oder observed

    is_defined:
        True  -> Struktur von get_defined_status_codes()
        False -> Struktur von get_evomaster_status_codes()
    """

    grouped_codes = defaultdict(list)

    for (method, path), information in data.items():

        if is_defined:
            status_codes = information["defined_codes"]
        else:
            status_codes = information

        for status_code in status_codes:
            grouped_codes[status_code].append(
                (method, path)
            )

    print()
    print(f"=== {name} status codes ===")

    for status_code in sorted(
        grouped_codes.keys(),
        key=int
    ):
        combinations = sorted(
            grouped_codes[status_code]
        )

        print()
        print(
            f"Statuscode {status_code}: "
            f"{len(combinations)} Kombinationen"
        )

        for method, path in combinations[:limit]:
            print(
                f"  {method:7} {path}"
            )

defined = get_defined_status_codes() 
observed =get_evomaster_status_codes()
print("-----------------------------")
if(args.debug):
    print_status_code_examples(
        defined,
        "Defined",
        is_defined=True,
        limit=10
    ) 
    print_status_code_examples(
        observed,
        "Observed",
        is_defined=False,
        limit=10
    )
print("----------------------------------")
coverage = calculate_coverage(defined, observed)
print(f"[+] Coverage: {coverage * 100:.2f}%")