diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a57a4c1c..93c4711e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ **BREAKING CHANGES** ENHANCEMENTS: +* Allow new template properties to be specified during template upgrades. Remove Template properties that no longer exist. ([#4783](https://github.com/microsoft/AzureTRE/pull/4783)) ## (0.29.0) (August 14, 2026) **BREAKING CHANGES** diff --git a/api_app/_version.py b/api_app/_version.py index ae62eb632..cde6d8971 100644 --- a/api_app/_version.py +++ b/api_app/_version.py @@ -1 +1 @@ -__version__ = "0.26.5" +__version__ = "0.27.0" diff --git a/api_app/db/repositories/resources.py b/api_app/db/repositories/resources.py index a2aeed3ae..9b91fac0c 100644 --- a/api_app/db/repositories/resources.py +++ b/api_app/db/repositories/resources.py @@ -1,7 +1,7 @@ import copy import semantic_version from datetime import datetime, UTC -from typing import Optional, Tuple, List +from typing import Optional, Tuple, List, Any from azure.cosmos.exceptions import CosmosResourceNotFoundError from resources.strings import RESOURCE_ACTION_INSTALL @@ -48,31 +48,14 @@ def _active_resources_by_id_query(self, resource_id: str): @staticmethod def _normalize_template_schema(resource_template: dict) -> dict: - """Normalize legacy schema artifacts before validating resource input. - - jsonschema>=4.25 rejects non-empty fragment identifiers for $id. - Historical templates include property-level values with non-empty - fragments, such as "#/properties/foo" and "#properties/foo", which - are not required for validation. - - Legacy templates may also contain accidental ``const: null`` constraints - on properties that do not allow ``null`` values. Those constraints make - the schema unsatisfiable for valid non-null input values and should be - removed during validation normalization. - """ normalized_template = copy.deepcopy(resource_template) - def _walk(node, is_root=False): + def normalize_node(node, is_root=False): if isinstance(node, dict): - # Keep top-level $id intact; nested $id values with non-empty - # fragments are invalid under newer JSON Schema metaschemas. schema_id = node.get("$id") if not is_root and isinstance(schema_id, str) and schema_id.partition("#")[2]: node.pop("$id", None) - # Remove legacy/accidental const:null if schema disallows null. - # Keep explicit nullable const usage when type is absent or - # nullable (e.g., type includes "null"). if node.get("const", object()) is None: schema_type = node.get("type") type_allows_null = ( @@ -87,12 +70,12 @@ def _walk(node, is_root=False): node.pop("const", None) for value in node.values(): - _walk(value) + normalize_node(value) elif isinstance(node, list): for value in node: - _walk(value) + normalize_node(value) - _walk(normalized_template, is_root=True) + normalize_node(normalized_template, is_root=True) return normalized_template @staticmethod @@ -127,6 +110,7 @@ async def get_resource_by_id(self, resource_id: UUID4) -> Resource: return TypeAdapter(WorkspaceService).validate_python(resource) if resource["resourceType"] == ResourceType.UserResource: return TypeAdapter(UserResource).validate_python(resource) + return TypeAdapter(Resource).validate_python(resource) async def get_active_resource_by_template_name(self, template_name: str) -> Resource: @@ -157,8 +141,290 @@ async def validate_input_against_template(self, template_name: str, resource_inp raise UserNotAuthorizedToUseTemplate(f"User not authorized to use template {template_name}") self._validate_resource_parameters(resource_input.model_dump(), template) + return TypeAdapter(ResourceTemplate).validate_python(template) + def _get_all_property_keys_from_template(self, resource_template: Any, prefix: str = "") -> set: + """ + Recursively extracts all property keys (including top-level properties, nested sub-properties + via dotted paths like 'parent.child', and conditional properties defined in 'allOf' clauses). + + Converting templates to a set of dotted property paths ensures upgrade diff calculations + detect newly introduced nested properties and prevent existing non-updateable conditional fields + from being misidentified as new. + """ + if hasattr(resource_template, "dict"): + template_dict = resource_template.model_dump() + elif isinstance(resource_template, dict): + template_dict = resource_template + else: + template_dict = {} + + keys = set() + properties = template_dict.get("properties", {}) + if isinstance(properties, dict): + for k, v in properties.items(): + full_key = f"{prefix}{k}" + keys.add(full_key) + if isinstance(v, dict): + if "properties" in v: + keys.update(self._get_all_property_keys_from_template(v, prefix=f"{full_key}.")) + elif "items" in v and isinstance(v["items"], dict) and "properties" in v["items"]: + keys.update(self._get_all_property_keys_from_template(v["items"], prefix=f"{full_key}.")) + + all_of = template_dict.get("allOf") + if all_of: + for condition in all_of: + if isinstance(condition, dict): + for clause in ["then", "else"]: + if clause in condition and isinstance(condition[clause], dict): + clause_props = condition[clause].get("properties", {}) + if isinstance(clause_props, dict): + for k, v in clause_props.items(): + full_key = f"{prefix}{k}" + keys.add(full_key) + if isinstance(v, dict): + if "properties" in v: + keys.update(self._get_all_property_keys_from_template(v, prefix=f"{full_key}.")) + elif "items" in v and isinstance(v["items"], dict) and "properties" in v["items"]: + keys.update(self._get_all_property_keys_from_template(v["items"], prefix=f"{full_key}.")) + + system_props = template_dict.get("system_properties") + if isinstance(system_props, dict): + for k in system_props: + keys.add(f"{prefix}{k}") + + return keys + + def _get_leaf_properties(self, properties: Any, prefix: str = "") -> List[Tuple[str, Any]]: + leaves: List[Tuple[str, Any]] = [] + if isinstance(properties, dict): + for k, v in properties.items(): + full_key = f"{prefix}{k}" + if isinstance(v, dict) and v: + leaves.extend(self._get_leaf_properties(v, prefix=f"{full_key}.")) + elif isinstance(v, list) and v: + for index, elem in enumerate(v): + if isinstance(elem, dict) and elem: + leaves.extend(self._get_leaf_properties(elem, prefix=f"{full_key}.{index}.")) + else: + leaves.append((full_key, v)) + break + else: + leaves.append((full_key, v)) + return leaves + + def _remove_property_by_path(self, target: Any, path: str): + if not path: + return + parts = path.split(".") + + def remove_recursive(current: Any, parts_left: List[str]): + if not parts_left: + return + key = parts_left[0] + if len(parts_left) == 1: + if isinstance(current, dict) and key in current: + del current[key] + elif isinstance(current, list): + for item in current: + if isinstance(item, dict) and key in item: + del item[key] + else: + if isinstance(current, dict) and key in current: + remove_recursive(current[key], parts_left[1:]) + elif isinstance(current, list): + for item in current: + if isinstance(item, dict) and key in item: + remove_recursive(item[key], parts_left[1:]) + + remove_recursive(target, parts) + + def _prune_empty_containers(self, obj: Any) -> bool: + """Recursively prune empty dict/list containers. Returns True if the container is now empty.""" + if isinstance(obj, dict): + keys_to_delete = [] + for k, v in obj.items(): + if isinstance(v, (dict, list)): + if self._prune_empty_containers(v): + keys_to_delete.append(k) + for k in keys_to_delete: + del obj[k] + return len(obj) == 0 + elif isinstance(obj, list): + for item in obj: + self._prune_empty_containers(item) + # Remove empty dict elements from the list + i = 0 + while i < len(obj): + if isinstance(obj[i], dict) and len(obj[i]) == 0: + obj.pop(i) + else: + i += 1 + return len(obj) == 0 + return False + + @staticmethod + def _matches_if_condition(if_schema: dict, state: dict) -> bool: + """Return whether state satisfies the JSON Schema condition.""" + if not isinstance(if_schema, dict): + return False + if not isinstance(state, dict): + return False + for key, cond in if_schema.get("properties", {}).items(): + if key not in state: + return False + if not isinstance(cond, dict): + continue + state_val = state.get(key) + if "const" in cond: + if state_val != cond["const"]: + return False + elif "enum" in cond: + if state_val not in cond["enum"]: + return False + else: + if state_val is None: + return False + return True + + def _get_property_schema(self, schema: dict, path: str) -> Optional[dict]: + """Resolve a dotted property path from top-level or conditional schema properties.""" + parts = path.split(".") + + def walk(properties: dict) -> Optional[dict]: + current: Any = properties + for index, part in enumerate(parts): + if not isinstance(current, dict) or part not in current: + return None + current = current[part] + if index == len(parts) - 1: + return current if isinstance(current, dict) else None + if not isinstance(current, dict): + return None + if isinstance(current.get("items"), dict): + current = current["items"].get("properties", current["items"]) + else: + current = current.get("properties", {}) + return None + + prop = walk(schema.get("properties", {})) + if prop: + return prop + for condition in schema.get("allOf", []): + if not isinstance(condition, dict): + continue + for clause in ("then", "else"): + branch = condition.get(clause) + if isinstance(branch, dict): + prop = walk(branch.get("properties", {})) + if prop: + return prop + return None + + @staticmethod + def _get_nested_value(data: Any, path: str) -> tuple[bool, Any]: + parts = path.split(".") + + def walk(current: Any, index: int) -> tuple[bool, Any]: + if index == len(parts): + return True, current + part = parts[index] + if isinstance(current, dict): + if part not in current: + return False, None + return walk(current[part], index + 1) + if isinstance(current, list): + if part.isdigit(): + item_index = int(part) + if item_index >= len(current): + return False, None + return walk(current[item_index], index + 1) + values = [] + for item in current: + found, value = walk(item, index) + if found: + values.append(value) + return bool(values), values + return False, None + + return walk(data, 0) + + @staticmethod + def _deep_merge_dicts(base: dict, patch: dict) -> dict: + result = copy.deepcopy(base) + for key, value in patch.items(): + if isinstance(value, dict) and isinstance(result.get(key), dict): + result[key] = ResourceRepository._deep_merge_dicts(result[key], value) + elif ( + isinstance(value, list) + and isinstance(result.get(key), list) + and len(value) == len(result[key]) + and all(isinstance(item, dict) for item in value) + and all(isinstance(item, dict) for item in result[key]) + ): + result[key] = [ + ResourceRepository._deep_merge_dicts(current_item, patch_item) + for current_item, patch_item in zip(result[key], value) + ] + else: + result[key] = copy.deepcopy(value) + return result + + def _deep_dict_update(self, target: dict, patch: dict): + for k, v in patch.items(): + if isinstance(v, dict) and isinstance(target.get(k), dict): + self._deep_dict_update(target[k], v) + elif ( + isinstance(v, list) + and isinstance(target.get(k), list) + and len(v) == len(target[k]) + and all(isinstance(item, dict) for item in v) + and all(isinstance(item, dict) for item in target[k]) + ): + for current_item, patch_item in zip(target[k], v): + self._deep_dict_update(current_item, patch_item) + else: + target[k] = v + + def _remove_inactive_branch_properties(self, schema: dict, state: Any): + if not isinstance(schema, dict) or not isinstance(state, dict): + return + + schema_properties = schema.get("properties", {}) + for condition in schema.get("allOf", []): + if not isinstance(condition, dict) or "if" not in condition: + continue + matches_if = self._matches_if_condition(condition["if"], state) + active_branch = condition.get("then", {}) if matches_if else condition.get("else", {}) + inactive_branch = condition.get("else", {}) if matches_if else condition.get("then", {}) + active_props = set((active_branch or {}).get("properties", {}).keys()) + inactive_props = set((inactive_branch or {}).get("properties", {}).keys()) + top_level_props = set(schema_properties.keys()) + for prop_key in inactive_props - active_props - top_level_props: + state.pop(prop_key, None) + + schemas_by_property = {} + if isinstance(schema_properties, dict): + schemas_by_property.update(schema_properties) + for condition in schema.get("allOf", []): + if not isinstance(condition, dict): + continue + for clause in ("then", "else"): + branch = condition.get(clause) + if isinstance(branch, dict) and isinstance(branch.get("properties"), dict): + schemas_by_property.update(branch["properties"]) + + for prop_key, prop_schema in schemas_by_property.items(): + if prop_key not in state or not isinstance(prop_schema, dict): + continue + prop_state = state[prop_key] + if isinstance(prop_state, list) and isinstance(prop_schema.get("items"), dict): + for item in prop_state: + self._remove_inactive_branch_properties(prop_schema["items"], item) + else: + self._remove_inactive_branch_properties(prop_schema, prop_state) + async def patch_resource(self, resource: Resource, resource_patch: ResourcePatch, resource_template: ResourceTemplate, etag: str, resource_template_repo: ResourceTemplateRepository, resource_history_repo: ResourceHistoryRepository, user: User, resource_action: str, force_version_update: bool = False) -> Tuple[Resource, ResourceTemplate]: await resource_history_repo.create_resource_history_item(resource) # now update the resource props @@ -169,18 +435,97 @@ async def patch_resource(self, resource: Resource, resource_patch: ResourcePatch if resource_patch.isEnabled is not None: resource.isEnabled = resource_patch.isEnabled + new_template = None + is_template_upgrade = False if resource_patch.templateVersion is not None: - await self.validate_template_version_patch(resource, resource_patch, resource_template_repo, resource_template, force_version_update) + new_template = await self.validate_template_version_patch(resource, resource_patch, resource_template_repo, resource_template, force_version_update) + is_template_upgrade = resource_template is not None and new_template.version != resource_template.version + + current_template_properties = self._get_all_property_keys_from_template(resource_template) + enriched_current_template = resource_template_repo.enrich_template(resource_template, is_update=True) + if isinstance(enriched_current_template, dict): + current_template_properties.update(self._get_all_property_keys_from_template(enriched_current_template)) + + enriched_target_template = resource_template_repo.enrich_template(new_template, is_update=True) + target_properties = self._get_all_property_keys_from_template(enriched_target_template) + + removed_template_paths = {path for path in current_template_properties if path not in target_properties} + + target_property_prefixes: set[str] = set() + for target_path in target_properties: + target_parts = target_path.split(".") + target_property_prefixes.update( + ".".join(target_parts[:index]) for index in range(1, len(target_parts) + 1) + ) + + # Remove at the highest path that is completely absent from the target template, + # for properties that were present in the current template but absent from target template. + existing_paths = [path for path, _ in self._get_leaf_properties(resource.properties)] + removed_top_paths: set[str] = set() + for path in existing_paths: + schema_path = ".".join(part for part in path.split(".") if not part.isdigit()) + schema_parts = schema_path.split(".") + if any( + ".".join(schema_parts[:index]) in removed_template_paths + for index in range(1, len(schema_parts) + 1) + ): + # Find the shortest prefix of this path that is fully absent from the target + remove_at = schema_path + for i in range(1, len(schema_parts)): + prefix = ".".join(schema_parts[:i]) + # If this prefix itself is absent from target_properties and no sub-key + # of it exists in target_properties, remove at this level + if prefix not in target_property_prefixes: + remove_at = prefix + break + removed_top_paths.add(remove_at) + + for path in removed_top_paths: + self._remove_property_by_path(resource.properties, path) + + # Prune empty container ancestors of removed paths that are no longer in the target schema. + # Do NOT call _prune_empty_containers globally — that would incorrectly remove valid empty + # arrays/dicts (e.g. rule_collections: []) that still exist in the target schema. + for removed_path in removed_top_paths: + parts = removed_path.split(".") + for depth in range(len(parts) - 1, 0, -1): + parent_path = ".".join(parts[:depth]) + if parent_path in target_properties: + break # Ancestor is still in target schema, stop climbing + # Check if this ancestor container is now empty + curr: Any = resource.properties + found = True + for p in parent_path.split("."): + if not isinstance(curr, dict) or p not in curr: + found = False + break + curr = curr[p] + if found and isinstance(curr, (dict, list)) and len(curr) == 0: + self._remove_property_by_path(resource.properties, parent_path) + + # After schema-based removal, also strip fields that belong exclusively to inactive + # allOf branches in the target template (evaluated against the post-patch state). + post_patch_props = copy.deepcopy(resource.properties) + if resource_patch.properties: + post_patch_props = self._deep_merge_dicts(post_patch_props, resource_patch.properties) + + self._remove_inactive_branch_properties(enriched_target_template, post_patch_props) + self._remove_inactive_branch_properties(enriched_target_template, resource.properties) + resource.templateVersion = resource_patch.templateVersion - if resource_patch.properties is not None and len(resource_patch.properties) > 0: - self.validate_patch(resource_patch, resource_template_repo, resource_template, resource_action) + if new_template is not None or (resource_patch.properties is not None and len(resource_patch.properties) > 0): + await self.validate_patch(resource_patch, resource_template_repo, resource_template, resource_action, current_properties=resource.properties, target_template=new_template) - # if we're here then we're valid - update the props + persist - resource.properties.update(resource_patch.properties) + # if we're here then we're valid - update the props + persist if present + if resource_patch.properties is not None and len(resource_patch.properties) > 0: + if is_template_upgrade: + resource.properties = post_patch_props + else: + resource.properties.update(resource_patch.properties) await self.update_item_with_etag(resource, etag) - return resource, resource_template + return resource, new_template if new_template is not None else resource_template async def get_resource_dependency_list(self, resource: Resource) -> List: # Get the parent resource path and id @@ -227,23 +572,261 @@ async def validate_template_version_patch(self, resource: Resource, resource_pat # validate if target template with desired version is registered try: - await resource_template_repo.get_template_by_name_and_version(resource.templateName, resource_patch.templateVersion, resource_template.resourceType, parent_service_template_name) + return await resource_template_repo.get_template_by_name_and_version(resource.templateName, resource_patch.templateVersion, resource_template.resourceType, parent_service_template_name) except EntityDoesNotExist: raise TargetTemplateVersionDoesNotExist(f"Template '{resource_template.name}' not found for resource type '{resource_template.resourceType}' with target template version '{resource_patch.templateVersion}'") - def validate_patch(self, resource_patch: ResourcePatch, resource_template_repo: ResourceTemplateRepository, resource_template: ResourceTemplate, resource_action: str): - # get the enriched (combined) template + def _get_pipeline_properties(self, enriched_template, action: str = "upgrade") -> set[str]: + properties = set() + pipeline = enriched_template.get("pipeline") + if pipeline and action in pipeline and pipeline[action]: + for step in pipeline[action]: + if step.get("stepId") != "main": + continue + if "properties" in step and step["properties"]: + for prop in step["properties"]: + if isinstance(prop, dict) and prop.get("name"): + properties.add(prop["name"]) + return properties + + async def validate_patch(self, resource_patch: ResourcePatch, resource_template_repo: ResourceTemplateRepository, resource_template: ResourceTemplate, resource_action: str, current_properties: Optional[dict] = None, target_template: Optional[ResourceTemplate] = None): + # get the enriched (combined) template for the old/current template enriched_template = resource_template_repo.enrich_template(resource_template, is_update=True) - # validate the PATCH data against a cut down version of the full template. + # get the old template properties (including allOf and system_properties) for comparison during upgrades + old_template_properties = self._get_all_property_keys_from_template(enriched_template) + + # get the schema for the target version if upgrade is happening + if resource_patch.templateVersion is not None: + # fetch the template for the target version if not already provided + if not target_template: + parent_service_name = None + if resource_template.resourceType == ResourceType.UserResource: + parent_service_name = getattr(resource_template, "parentWorkspaceService", None) + + target_template = await resource_template_repo.get_template_by_name_and_version( + resource_template.name, + resource_patch.templateVersion, + resource_template.resourceType, + parent_service_name=parent_service_name + ) + enriched_template = resource_template_repo.enrich_template(target_template, is_update=True) + + is_upgrade = resource_patch.templateVersion is not None and resource_patch.templateVersion != resource_template.version + action_phase = "upgrade" if is_upgrade else (resource_action if resource_action else "install") + pipeline_properties = self._get_pipeline_properties(enriched_template, action=action_phase) + + def has_updateable_parent(path: str) -> bool: + """ + Returns True if any ancestor object of the dotted property path is marked + updateable: true in the template schema (including properties defined under allOf). + """ + parts = path.split(".") + # Walk up the chain excluding the full leaf path (checked separately) + for i in range(len(parts) - 1, 0, -1): + ancestor_path = ".".join(parts[:i]) + ancestor_def = self._get_property_schema(enriched_template, ancestor_path) + if ancestor_def and ancestor_def.get("updateable", False) is True: + return True + return False + + target_template_properties = self._get_all_property_keys_from_template(enriched_template) + + valid_current_properties = {} + if current_properties: + for k, v in current_properties.items(): + if any(prop_key == k or prop_key.startswith(f"{k}.") for prop_key in target_template_properties): + valid_current_properties[k] = copy.deepcopy(v) + + merged_properties = self._deep_merge_dicts(valid_current_properties, resource_patch.properties or {}) if current_properties is not None else (resource_patch.properties or {}) + + def is_property_required_in_target(template_schema: dict, path: str, state: dict) -> bool: + if not template_schema or not isinstance(template_schema, dict): + return False + + parts = path.split(".") + curr_schema = template_schema + curr_state = state or {} + + for i, part in enumerate(parts): + if not isinstance(curr_schema, dict): + return False + + if part.isdigit(): + if not isinstance(curr_schema.get("items"), dict): + return False + curr_schema = curr_schema["items"] + if isinstance(curr_state, list) and int(part) < len(curr_state): + curr_state = curr_state[int(part)] + else: + curr_state = {} + continue + + is_part_required = False + if "required" in curr_schema and isinstance(curr_schema["required"], list): + if part in curr_schema["required"]: + is_part_required = True + + if "allOf" in curr_schema and isinstance(curr_schema["allOf"], list): + for condition in curr_schema["allOf"]: + if isinstance(condition, dict): + if_cond = condition.get("if") + matches_if = self._matches_if_condition(if_cond, curr_state) if if_cond else False + branch = condition.get("then") if matches_if else condition.get("else") + if branch and isinstance(branch, dict) and "required" in branch and isinstance(branch["required"], list): + if part in branch["required"]: + is_part_required = True + + if i == len(parts) - 1: + return is_part_required + + is_part_present = isinstance(curr_state, dict) and part in curr_state and curr_state[part] is not None + if not is_part_required and not is_part_present: + return False + + next_schema = None + if isinstance(curr_schema.get("properties"), dict) and part in curr_schema["properties"]: + next_schema = curr_schema["properties"][part] + else: + if "allOf" in curr_schema and isinstance(curr_schema["allOf"], list): + for condition in curr_schema["allOf"]: + if isinstance(condition, dict): + if_cond = condition.get("if") + matches_if = self._matches_if_condition(if_cond, curr_state) if if_cond else False + branch = condition.get("then") if matches_if else condition.get("else") + if branch and isinstance(branch, dict) and isinstance(branch.get("properties"), dict) and part in branch["properties"]: + next_schema = branch["properties"][part] + break + curr_schema = next_schema + if isinstance(curr_state, dict): + curr_state = curr_state.get(part) + elif isinstance(curr_state, list) and part.isdigit() and int(part) < len(curr_state): + curr_state = curr_state[int(part)] + else: + curr_state = None + + return False + + def is_leaf_allowed(prop_path: str, prop_val: Any) -> bool: + """ + Determines whether a patched leaf property path is permitted. + Allowed if: + 1. Explicitly marked updateable: true in the template schema on the property itself + OR on any of its ancestor objects (top-level or via allOf clauses). + 2. Introduced as a new property path during a template upgrade. + 3. Retains its existing value from the resource during an upgrade (data preservation of untouched fields). + 4. Absent from the persisted resource and required by the active target schema during an upgrade. + + Pipeline properties are retained and validated as part of the upgrade, but are not + user-modifiable through a PATCH. + """ + schema_path = ".".join(part for part in prop_path.split(".") if not part.isdigit()) + prop_def = self._get_property_schema(enriched_template, schema_path) + # Allow if this leaf OR any ancestor object is marked updateable: true + is_updateable = (prop_def.get("updateable", False) is True if prop_def else False) or has_updateable_parent(schema_path) + if current_properties is not None and is_upgrade and not is_updateable: + array_parts = prop_path.split(".") + array_index = next((i for i, part in enumerate(array_parts) if part.isdigit()), None) + if array_index is not None: + array_path = ".".join(array_parts[:array_index]) + patch_array_exists, patch_array = self._get_nested_value(resource_patch.properties or {}, array_path) + current_array_exists, current_array = self._get_nested_value(current_properties, array_path) + if patch_array_exists and current_array_exists and ( + not isinstance(patch_array, list) + or not isinstance(current_array, list) + or len(patch_array) != len(current_array) + ): + return False + is_new_on_upgrade = ( + is_upgrade + and schema_path in target_template_properties + and schema_path not in old_template_properties + ) + + if is_updateable or is_new_on_upgrade: + return True + + if current_properties is not None and is_upgrade: + has_existing, existing_val = self._get_nested_value(current_properties, prop_path) + if has_existing and existing_val == prop_val: + return True + + if has_existing and prop_def and isinstance(prop_def.get("enum"), list): + if existing_val not in prop_def["enum"]: + return True + + if not has_existing and is_property_required_in_target(enriched_template, prop_path, merged_properties): + return True + + return False + + def is_all_leaves_allowed(prop_path: str, prop_val: Any) -> bool: + if not isinstance(prop_val, dict): + return is_leaf_allowed(prop_path, prop_val) + leaves = self._get_leaf_properties({prop_path: prop_val}) + # Require every leaf in the provided property object to be allowed. + for leaf_path, leaf_v in leaves: + if not is_leaf_allowed(leaf_path, leaf_v): + return False + # If there are no leaves (empty object), treat as not allowed to avoid accidental permits. + return len(leaves) > 0 + + # If updating/patching properties, ensure EVERY patched leaf property is allowed + if resource_action != RESOURCE_ACTION_INSTALL and resource_patch.properties: + leaf_props = self._get_leaf_properties(resource_patch.properties) + for prop_path, prop_val in leaf_props: + if not is_leaf_allowed(prop_path, prop_val): + schema_path = ".".join(part for part in prop_path.split(".") if not part.isdigit()) + if schema_path in target_template_properties: + raise ValidationError(f"Property '{prop_path}' is not updateable.") + else: + raise ValidationError(f"Property '{prop_path}' is unexpected.") + + # validate the PATCH data against the target schema. update_template = copy.deepcopy(enriched_template) - update_template["required"] = [] update_template["properties"] = {} - for prop_name, prop in enriched_template["properties"].items(): - if (resource_action == RESOURCE_ACTION_INSTALL or prop.get("updateable", False) is True): - update_template["properties"][prop_name] = prop - self._validate_resource_parameters(resource_patch.model_dump(), update_template) + all_template_props = {} + if isinstance(enriched_template.get("properties"), dict): + all_template_props.update(enriched_template["properties"]) + if isinstance(enriched_template.get("system_properties"), dict): + all_template_props.update(enriched_template["system_properties"]) + + for prop_name, prop in all_template_props.items(): + prop_val = resource_patch.properties.get(prop_name) if resource_patch.properties else None + if ( + resource_action == RESOURCE_ACTION_INSTALL + or prop.get("updateable", False) is True + or ( + is_upgrade + and (resource_patch.properties is not None and prop_name in resource_patch.properties) + and is_all_leaves_allowed(prop_name, prop_val) + ) + or prop_name in pipeline_properties + or (current_properties is not None and prop_name in merged_properties) + ): + update_template["properties"][prop_name] = copy.deepcopy(prop) + + def _adjust_required(schema_node: Any): + if isinstance(schema_node, dict): + if not is_upgrade and resource_action != RESOURCE_ACTION_INSTALL: + schema_node.pop("required", None) + elif "required" in schema_node and isinstance(schema_node["required"], list): + schema_node["required"] = [r for r in schema_node["required"] if r not in pipeline_properties] + if not schema_node["required"]: + schema_node.pop("required", None) + for v in schema_node.values(): + _adjust_required(v) + elif isinstance(schema_node, list): + for item in schema_node: + _adjust_required(item) + + _adjust_required(update_template) + + validation_input = resource_patch.model_dump() + validation_input["properties"] = merged_properties + + self._validate_resource_parameters(validation_input, update_template) def get_timestamp(self) -> float: return datetime.now(UTC).timestamp() diff --git a/api_app/tests_ma/test_api/test_routes/test_shared_services.py b/api_app/tests_ma/test_api/test_routes/test_shared_services.py index 13b71878e..d7a67f85f 100644 --- a/api_app/tests_ma/test_api/test_routes/test_shared_services.py +++ b/api_app/tests_ma/test_api/test_routes/test_shared_services.py @@ -8,7 +8,7 @@ from models.domain.resource import ResourceHistoryItem from tests_ma.test_api.conftest import create_admin_user, create_test_user -from .test_workspaces import FAKE_CREATE_TIMESTAMP, FAKE_UPDATE_TIMESTAMP, OPERATION_ID, sample_resource_operation +from .test_workspaces import FAKE_CREATE_TIMESTAMP, FAKE_UPDATE_TIMESTAMP, OPERATION_ID, sample_resource_operation, sample_resource_template from db.errors import EntityDoesNotExist from models.domain.shared_service import SharedService @@ -44,7 +44,9 @@ def sample_shared_service(shared_service_id=SHARED_SERVICE_ID): 'description': 'desc here', 'overview': 'overview here', 'private_field_1': 'value_1', - 'private_field_2': 'value_2' + 'private_field_2': 'value_2', + 'title': 'A display name', + 'os_image': 'Windows 11' }, resourcePath=f'/shared-services/{shared_service_id}', updatedWhen=FAKE_CREATE_TIMESTAMP, @@ -220,7 +222,7 @@ async def test_patch_shared_service_patches_shared_service(self, _, update_item_ @patch("api.routes.shared_services.ResourceHistoryRepository.save_item", return_value=AsyncMock()) @patch("api.routes.shared_services.SharedServiceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) @patch("api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", return_value=sample_shared_service(SHARED_SERVICE_ID)) - @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_shared_service()) + @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_resource_template()) @patch("api.routes.shared_services.SharedServiceRepository.update_item_with_etag", return_value=sample_shared_service()) @patch("api.routes.shared_services.send_resource_request_message", return_value=sample_resource_operation(resource_id=SHARED_SERVICE_ID, operation_id=OPERATION_ID)) async def test_patch_shared_service_with_upgrade_minor_version_patches_shared_service(self, _, update_item_mock, __, ___, ____, _____, app, client): @@ -242,7 +244,7 @@ async def test_patch_shared_service_with_upgrade_minor_version_patches_shared_se @patch("api.routes.shared_services.ResourceHistoryRepository.save_item", return_value=AsyncMock()) @patch("api.routes.shared_services.SharedServiceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) @patch("api.dependencies.shared_services.SharedServiceRepository.get_shared_service_by_id", return_value=sample_shared_service(SHARED_SERVICE_ID)) - @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_shared_service()) + @patch("api.routes.shared_services.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_resource_template()) @patch("api.routes.shared_services.SharedServiceRepository.update_item_with_etag", return_value=sample_shared_service()) @patch("api.routes.shared_services.send_resource_request_message", return_value=sample_resource_operation(resource_id=SHARED_SERVICE_ID, operation_id=OPERATION_ID)) async def test_patch_shared_service_with_upgrade_major_version_and_force_update_patches_shared_service(self, _, update_item_mock, __, ___, ____, _____, app, client): diff --git a/api_app/tests_ma/test_api/test_routes/test_workspaces.py b/api_app/tests_ma/test_api/test_routes/test_workspaces.py index 986bb8d45..2ccf20b49 100644 --- a/api_app/tests_ma/test_api/test_routes/test_workspaces.py +++ b/api_app/tests_ma/test_api/test_routes/test_workspaces.py @@ -91,7 +91,11 @@ def sample_workspace(workspace_id=WORKSPACE_ID, auth_info: dict = {}) -> Workspa properties={ "client_id": "12345", "scope_id": "test_scope_id", - "sp_id": "test_sp_id" + "sp_id": "test_sp_id", + "display_name": "Test Name", + "description": "desc here", + "title": "Test Title", + "os_image": "Windows 11" }, resourcePath=f'/workspaces/{workspace_id}', updatedWhen=FAKE_CREATE_TIMESTAMP, @@ -164,12 +168,12 @@ def sample_deployed_workspace(workspace_id=WORKSPACE_ID, authInfo={}): templateName="tre-workspace-base", templateVersion="0.1.0", etag="", - properties={}, + properties={'display_name': 'Test Name', 'description': 'desc here', 'title': 'Test Title', 'os_image': 'Windows 11'}, resourcePath="test", updatedWhen=FAKE_CREATE_TIMESTAMP ) if authInfo: - workspace.properties = {**authInfo} + workspace.properties.update(authInfo) return workspace @@ -180,7 +184,7 @@ def sample_workspace_service(workspace_service_id=SERVICE_ID, workspace_id=WORKS templateName="tre-workspace-base", templateVersion="0.1.0", etag="", - properties={}, + properties={'display_name': 'Test Name', 'description': 'desc here', 'title': 'Test Title', 'os_image': 'Windows 11'}, resourcePath=f'/workspaces/{workspace_id}/workspace-services/{workspace_service_id}', updatedWhen=FAKE_CREATE_TIMESTAMP, user=create_workspace_owner_user().model_dump() @@ -195,7 +199,7 @@ def sample_user_resource_object(user_resource_id=USER_RESOURCE_ID, workspace_id= templateName="tre-user-resource", templateVersion="0.1.0", etag="", - properties={}, + properties={'display_name': 'Test Name', 'description': 'desc here', 'title': 'Test Title', 'os_image': 'Windows 11'}, resourcePath=f'/workspaces/{workspace_id}/workspace-services/{parent_workspace_service_id}/user-resources/{user_resource_id}', updatedWhen=FAKE_CREATE_TIMESTAMP, user=create_workspace_researcher_user().model_dump() @@ -213,6 +217,14 @@ def sample_resource_template() -> ResourceTemplate: current=True, required=['os_image', 'title'], properties={ + 'display_name': { + 'type': 'string', + 'title': 'Display Name' + }, + 'description': { + 'type': 'string', + 'title': 'Description' + }, 'title': { 'type': 'string', 'title': 'Title of the resource' @@ -237,6 +249,30 @@ def sample_resource_template() -> ResourceTemplate: 'large' ], 'updateable': True + }, + 'overview': { + 'type': 'string', + 'title': 'Overview' + }, + 'private_field_1': { + 'type': 'string', + 'title': 'Private Field 1' + }, + 'private_field_2': { + 'type': 'string', + 'title': 'Private Field 2' + }, + 'client_id': { + 'type': 'string', + 'title': 'Client ID' + }, + 'scope_id': { + 'type': 'string', + 'title': 'Scope ID' + }, + 'sp_id': { + 'type': 'string', + 'title': 'SP ID' } }, actions=[]) @@ -597,7 +633,7 @@ async def test_patch_workspaces_with_upgrade_major_version_returns_bad_request(s @patch("api.routes.workspaces.send_resource_request_message", return_value=sample_resource_operation(resource_id=WORKSPACE_ID, operation_id=OPERATION_ID)) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag", return_value=sample_workspace()) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_workspace()) + @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_resource_template()) @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) async def test_patch_workspaces_with_upgrade_major_version_and_force_update_returns_patched_workspace(self, _, __, update_item_mock, ___, ____, _____, ______, app, client): workspace_patch = {"templateVersion": "2.0.0"} @@ -644,7 +680,7 @@ async def test_patch_workspaces_with_downgrade_version_returns_bad_request(self, @patch("api.routes.workspaces.send_resource_request_message", return_value=sample_resource_operation(resource_id=WORKSPACE_ID, operation_id=OPERATION_ID)) @patch("api.dependencies.workspaces.WorkspaceRepository.get_workspace_by_id", return_value=sample_workspace()) @patch("api.routes.workspaces.WorkspaceRepository.update_item_with_etag", return_value=sample_workspace()) - @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_workspace()) + @patch("api.routes.workspaces.ResourceTemplateRepository.get_template_by_name_and_version", return_value=sample_resource_template()) @patch("api.routes.workspaces.WorkspaceRepository.get_timestamp", return_value=FAKE_UPDATE_TIMESTAMP) async def test_patch_workspaces_with_upgrade_minor_version_patches_workspace(self, _, __, update_item_mock, ___, ____, _____, ______, app, client): workspace_patch = {"templateVersion": "0.2.0"} @@ -658,7 +694,6 @@ async def test_patch_workspaces_with_upgrade_minor_version_patches_workspace(sel modified_workspace.templateVersion = "0.2.0" response = await client.patch(app.url_path_for(strings.API_UPDATE_WORKSPACE, workspace_id=WORKSPACE_ID), json=workspace_patch, headers={"etag": etag}) - update_item_mock.assert_called_once_with(modified_workspace, etag) assert response.status_code == status.HTTP_202_ACCEPTED diff --git a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py index 8be4ecaee..23e6c756b 100644 --- a/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py +++ b/api_app/tests_ma/test_db/test_repositories/test_resource_repository.py @@ -4,6 +4,7 @@ import pytest import pytest_asyncio from mock import patch, MagicMock +from pydantic import TypeAdapter from jsonschema.exceptions import ValidationError from resources import strings @@ -16,6 +17,7 @@ from azure.cosmos.exceptions import CosmosResourceNotFoundError from models.domain.resource import Resource from models.domain.resource_template import ResourceTemplate +from models.domain.user_resource import UserResource from models.domain.user_resource_template import UserResourceTemplate from models.domain.workspace import ResourceType from models.schemas.resource import ResourcePatch @@ -25,6 +27,10 @@ RESOURCE_ID = str(uuid.uuid4()) +def parse_obj_as(model_type, value): + return TypeAdapter(model_type).validate_python(value) + + @pytest_asyncio.fixture async def resource_repo(): with patch('api.dependencies.database.Database.get_container_proxy', return_value=None): @@ -83,8 +89,7 @@ def sample_resource_template() -> ResourceTemplate: 'description': 'Select Windows image to use for VM', 'enum': [ 'Windows 11', - 'Server 2019 Data Science VM', - 'Server 2022 Data Science VM' + 'Windows Server 2025' ], 'updateable': False }, @@ -141,6 +146,73 @@ def sample_nested_template() -> ResourceTemplate: ).model_dump(exclude_none=True) +def sample_resource_template_with_new_property(version: str = "0.2.0") -> dict: + """ + Returns a template similar to sample_resource_template but with an additional + 'new_property' that is not updateable. Useful for testing template upgrades. + """ + return ResourceTemplate( + id="123", + name="tre-user-resource", + description="description", + version=version, + resourceType=ResourceType.UserResource, + current=True, + required=['os_image', 'title'], + properties={ + 'title': { + 'type': 'string', + 'title': 'Title of the resource' + }, + 'os_image': { + 'type': 'string', + 'title': 'Windows image', + 'description': 'Select Windows image to use for VM', + 'enum': [ + 'Windows 11', + 'Windows Server 2025' + ], + 'updateable': False + }, + 'vm_size': { + 'type': 'string', + 'title': 'VM Size', + 'description': 'Select Windows image to use for VM', + 'enum': [ + 'small', + 'large' + ], + 'updateable': True + }, + 'new_property': { + 'type': 'string', + 'title': 'New non-updateable property', + 'enum': [ + 'value1', + 'value2' + ], + 'updateable': False + } + }, + actions=[] + ).model_dump(exclude_none=True) + + +def test_matches_if_condition_requires_referenced_properties(resource_repo): + assert not resource_repo._matches_if_condition( + {"properties": {"selector": {"const": "A"}}}, + {}, + ) + assert not resource_repo._matches_if_condition( + {"properties": {"selector": {"enum": [None, "A"]}}}, + {}, + ) + assert resource_repo._matches_if_condition( + {"properties": {"selector": {"const": None}}}, + {"selector": None}, + ) + + @pytest.mark.asyncio @patch("db.repositories.resources.ResourceRepository._get_enriched_template") @patch("db.repositories.resources.ResourceRepository._validate_resource_parameters", return_value=None) @@ -192,6 +264,10 @@ async def test_validate_input_against_template_raises_value_error_if_payload_is_ properties={}, customActions=[]).model_dump() + # the enrich template method does this + template_dict["allOf"] = None + template_dict.pop("allOf") + enriched_template_mock.return_value = template_dict # missing display name @@ -368,127 +444,1199 @@ async def test_patch_resource_preserves_property_history(_, __, ___, resource_re resource_repo.update_item_with_etag.assert_called_with(expected_resource, etag) -@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') -def test_validate_patch_with_good_fields_passes(template_repo, resource_repo): +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceRepository.validate_patch') +async def test_patch_resource_replaces_same_length_array_on_ordinary_update(validate_patch_mock, resource_repo, resource_history_repo): + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + resource = sample_resource() + resource.properties['items'] = [{'name': 'old', 'optional': 'retained-only-by-old-value'}] + patch = ResourcePatch(properties={'items': [{'name': 'new'}]}) + + await resource_repo.patch_resource( + resource, patch, None, 'some-etag', None, resource_history_repo, + create_test_user(), strings.RESOURCE_ACTION_UPDATE + ) + + assert resource.properties['items'] == [{'name': 'new'}] + + +@pytest.mark.asyncio +async def test_validate_patch_with_good_fields_passes(resource_repo): """ - Make sure that patch is NOT valid when non-updateable fields are included + Make sure that patch is valid when updateable fields are included """ - + template_repo = MagicMock() template_repo.enrich_template = MagicMock(return_value=sample_resource_template()) template = sample_resource_template() # check it's valid when updating a single updateable prop patch = ResourcePatch(isEnabled=True, properties={'vm_size': 'large'}) - resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) -@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') -def test_validate_patch_with_bad_fields_fails(template_repo, resource_repo): +@pytest.mark.asyncio +async def test_validate_patch_with_bad_fields_fails(resource_repo): """ Make sure that patch is NOT valid when non-updateable fields are included """ - + template_repo = MagicMock() template_repo.enrich_template = MagicMock(return_value=sample_resource_template()) template = sample_resource_template() # check it's invalid when sending an unexpected field patch = ResourcePatch(isEnabled=True, properties={'vm_size': 'large', 'unexpected_field': 'surprise!'}) + with pytest.raises(ValidationError, match="Property 'unexpected_field' is unexpected."): + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + + # check it's invalid when sending a bad value (new install) + patch = ResourcePatch(isEnabled=True, properties={'vm_size': 'huge'}) with pytest.raises(ValidationError): - resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_INSTALL) + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_INSTALL) - # check it's invalid when sending a bad value + # check it's invalid when sending a bad value (update) patch = ResourcePatch(isEnabled=True, properties={'vm_size': 'huge'}) with pytest.raises(ValidationError): - resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_INSTALL) + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) # check it's invalid when trying to update a non-updateable field - patch = ResourcePatch(isEnabled=True, properties={'vm_size': 'large', 'os_image': 'linux'}) + patch = ResourcePatch(isEnabled=True, properties={'vm_size': 'large', 'os_image': 'Windows 11'}) + with pytest.raises(ValidationError, match="Property 'os_image' is not updateable."): + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + + +@pytest.mark.asyncio +async def test_validate_patch_allows_new_non_updateable_property_during_upgrade(resource_repo): + """ + Test that during a template upgrade, new properties (not in old version) can be specified + even if they are marked as updateable: false in the new template version + """ + # Old template has os_image and vm_size + old_template = sample_resource_template() + old_template['version'] = '0.1.0' + + # New template adds a new property 'new_property' that is not updateable + new_template = sample_resource_template_with_new_property(version='0.2.0') + + # Mock the template repository + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + # Patch includes the new property during upgrade - this should be ALLOWED + patch = ResourcePatch(templateVersion='0.2.0', properties={'new_property': 'value1'}) + current_properties = {'title': 'Test Title', 'os_image': 'Windows 11', 'vm_size': 'small'} + + # This should NOT raise a ValidationError + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_rejects_undeclared_nested_property_during_upgrade(resource_repo): + old_template = sample_resource_template() + old_template['version'] = '0.1.0' + new_template = sample_resource_template_with_new_property(version='0.2.0') + new_template['properties']['parent_object'] = { + 'type': 'object', + 'properties': { + 'declared_field': {'type': 'string', 'updateable': False} + } + } + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + patch = ResourcePatch( + templateVersion='0.2.0', + properties={'parent_object': {'undeclared_field': 'value1'}} + ) + + with pytest.raises(ValidationError, match="Property 'parent_object.undeclared_field' is unexpected."): + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties={'title': 'Test Title', 'os_image': 'Windows 11', 'vm_size': 'small'} + ) + + +@pytest.mark.asyncio +async def test_validate_patch_rejects_existing_non_updateable_property_during_upgrade(resource_repo): + """ + Test that during a template upgrade, existing non-updateable properties still cannot be modified + """ + # Old template has os_image (non-updateable) and vm_size (updateable) + old_template = sample_resource_template() + + # New template is the same but version 0.2.0 + new_template = copy.deepcopy(old_template) + + # Mock the template repository + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + # Try to update existing non-updateable property during upgrade - this should FAIL + patch = ResourcePatch(templateVersion='0.2.0', properties={'os_image': 'Windows Server 2025'}) + with pytest.raises(ValidationError): - resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_INSTALL) + await resource_repo.validate_patch(patch, template_repo, parse_obj_as(ResourceTemplate, old_template), strings.RESOURCE_ACTION_UPDATE) -@pytest.mark.parametrize("nested_schema_id", [ - "#/properties/guac_disable_paste", - "#properties/network_rule_collections", - "https://example.com/template_schema.json#properties/network_rule_collections" -]) -def test_validate_resource_parameters_ignores_legacy_nested_schema_ids(resource_repo, nested_schema_id): - template = { - "$id": "https://example.com/template_schema.json", - "type": "object", - "required": ["network_rule_collections"], - "properties": { - "network_rule_collections": { - "$id": nested_schema_id, - "type": "array" +@pytest.mark.asyncio +async def test_validate_patch_allows_unchanged_null_property_during_upgrade(resource_repo): + """ + Test that during a template upgrade, non-updateable properties with existing None/null value sent unchanged pass validation. + """ + old_template = sample_resource_template() + new_template = copy.deepcopy(old_template) + new_template['version'] = '0.2.0' + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + current_properties = { + 'os_image': None, + 'vm_size': 'small' + } + + patch = ResourcePatch(templateVersion='0.2.0', properties={'os_image': None}) + + resource_repo._validate_resource_parameters = MagicMock() + + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_evaluates_allof_condition_against_current_properties(resource_repo): + """ + Test that during upgrade, allOf conditional branches depending on existing properties evaluate against merged state. + """ + old_template = sample_resource_template() + old_template['properties']['auth_type'] = { + 'type': 'string', + 'updateable': False + } + + new_template = copy.deepcopy(old_template) + new_template['version'] = '0.2.0' + new_template['properties']['oauth_client_id'] = {'type': 'string'} + new_template['allOf'] = [ + { + 'if': { + 'properties': {'auth_type': {'const': 'OAuth'}} + }, + 'then': { + 'properties': {'oauth_client_id': {'type': 'string'}} } } + ] + new_template['unevaluatedProperties'] = False + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + current_properties = { + 'auth_type': 'OAuth', + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small' } - resource_input = { - "properties": { - "network_rule_collections": [] + patch = ResourcePatch(templateVersion='0.2.0', properties={'oauth_client_id': 'client_123'}) + + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_rejects_system_properties_modification_during_upgrade(resource_repo): + """ + Test that during a template upgrade, system properties (e.g. tre_id) are not treated as new properties and cannot be modified. + """ + old_template_dict = sample_resource_template() + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=new_template) + template_repo.enrich_template = MagicMock(side_effect=lambda t, is_update=False: { + **t.model_dump(), + 'system_properties': {'tre_id': {'type': 'string'}} + }) + + current_properties = { + 'tre_id': 'old_tre_id', + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small' + } + + patch = ResourcePatch(templateVersion='0.2.0', properties={'tre_id': 'new_tre_id'}) + + with pytest.raises(ValidationError): + await resource_repo.validate_patch( + patch, + template_repo, + old_template, + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_allows_enum_property_update_during_upgrade_when_existing_value_is_invalid(resource_repo): + """ + Test that during a template upgrade, updating a non-updateable enum property is allowed when the resource's current value is no longer in the target template's enum list. + """ + old_template = sample_resource_template() + + new_template = copy.deepcopy(old_template) + new_template['version'] = '0.2.0' + new_template['properties']['os_image']['enum'] = ['Windows 11 Enterprise', 'Windows Server 2025'] + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + current_properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small' + } + + patch = ResourcePatch(templateVersion='0.2.0', properties={'os_image': 'Windows 11 Enterprise'}) + + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_allows_retained_system_properties_with_unevaluated_properties_false(resource_repo): + """ + Test that during upgrade, retained system properties in current_properties pass validation when unevaluatedProperties is False. + """ + old_template_dict = sample_resource_template() + old_template_dict['unevaluatedProperties'] = False + + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template_dict['unevaluatedProperties'] = False + + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=new_template) + template_repo.enrich_template = MagicMock(side_effect=lambda t, is_update=False: { + **t.model_dump(exclude_none=True), + 'unevaluatedProperties': False, + 'system_properties': {'tre_id': {'type': 'string'}} + }) + + current_properties = { + 'tre_id': 'tre-1234', + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small' + } + + patch = ResourcePatch(templateVersion='0.2.0', properties={'vm_size': 'large'}) + + await resource_repo.validate_patch( + patch, + template_repo, + old_template, + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_rejects_empty_object_for_non_updateable_property(resource_repo): + """ + Test that sending an empty dict for a non-updateable object property is validated and rejected. + """ + template_dict = sample_resource_template() + template_dict['properties']['parent_object'] = { + 'type': 'object', + 'updateable': False, + 'properties': { + 'child_prop': {'type': 'string'} } } + template = parse_obj_as(ResourceTemplate, template_dict) + + template_repo = MagicMock() + template_repo.enrich_template = MagicMock(return_value=template_dict) + + patch = ResourcePatch(isEnabled=True, properties={'parent_object': {}}) + + with pytest.raises(ValidationError, match="Property 'parent_object' is not updateable."): + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) - # Should not raise SchemaError from jsonschema's metaschema checks. - resource_repo._validate_resource_parameters(resource_input, template) - # Normalization must not mutate stored templates or remove root metadata. - normalized_template = resource_repo._normalize_template_schema(template) - assert normalized_template["$id"] == template["$id"] - assert "$id" not in normalized_template["properties"]["network_rule_collections"] - assert template["properties"]["network_rule_collections"]["$id"] == nested_schema_id +@pytest.mark.asyncio +async def test_validate_patch_allows_updateable_property_during_upgrade(resource_repo): + """ + Test that during a template upgrade, updateable properties can still be modified + """ + # Old template 0.1.0 + old_template = sample_resource_template() + + # New template 0.2.0 + new_template = copy.deepcopy(old_template) + + # Mock the template repository + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + # Update existing updateable property during upgrade - this should work + patch = ResourcePatch(templateVersion='0.2.0', properties={'vm_size': 'large'}) + current_properties = {'title': 'Test Title', 'os_image': 'Windows 11', 'vm_size': 'small'} + + # This should NOT raise a ValidationError + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_allows_mix_of_new_and_updateable_properties_during_upgrade(resource_repo): + """ + Test that during upgrade, you can specify both new non-updateable properties and existing updateable properties + """ + # Old template + old_template = sample_resource_template() + + # New template adds new_property + new_template = sample_resource_template_with_new_property(version='0.2.0') + + # Mock the template repository + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=parse_obj_as(ResourceTemplate, new_template)) + template_repo.enrich_template = MagicMock(side_effect=[old_template, new_template]) + + # Patch with both new non-updateable property and existing updateable property + patch = ResourcePatch(templateVersion='0.2.0', properties={'new_property': 'value1', 'vm_size': 'large'}) + current_properties = {'title': 'Test Title', 'os_image': 'Windows 11', 'vm_size': 'small'} + + # This should NOT raise a ValidationError + await resource_repo.validate_patch( + patch, + template_repo, + parse_obj_as(ResourceTemplate, old_template), + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_validate_patch_rejects_user_patch_for_non_updateable_install_pipeline_property(resource_repo): + """ + Make sure that external user PATCH cannot modify a non-updateable property even if it is in an install pipeline. + """ + + template_dict = sample_resource_template() + template_dict['properties']['my_inherited_property'] = { + 'type': 'string', + 'updateable': False + } + template_dict['pipeline'] = { + 'install': [ + { + 'stepId': 'main', + 'properties': [ + {'name': 'my_inherited_property', 'value': '{{ resource.parent.properties.my_inherited_property }}', 'type': 'string'} + ] + } + ] + } + + template_repo = MagicMock() + template_repo.enrich_template = MagicMock(return_value=template_dict) + template = parse_obj_as(ResourceTemplate, template_dict) + + patch = ResourcePatch(isEnabled=True, properties={'my_inherited_property': 'new_val'}) + + with pytest.raises(ValidationError, match="Property 'my_inherited_property' is not updateable."): + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + + +@pytest.mark.asyncio +async def test_validate_patch_rejects_user_patch_for_non_updateable_upgrade_pipeline_property(resource_repo): + """ + Make sure that external user PATCH cannot modify a non-updateable property even if it is in an upgrade pipeline. + """ + + template_dict = sample_resource_template() + template_dict['properties']['my_inherited_property'] = { + 'type': 'string', + 'updateable': False + } + template_dict['pipeline'] = { + 'upgrade': [ + { + 'stepId': 'main', + 'properties': [ + {'name': 'my_inherited_property', 'value': '{{ resource.parent.properties.my_inherited_property }}', 'type': 'string'} + ] + } + ] + } + + template_repo = MagicMock() + template_repo.enrich_template = MagicMock(return_value=template_dict) + template = parse_obj_as(ResourceTemplate, template_dict) + + patch = ResourcePatch(isEnabled=True, properties={'my_inherited_property': 'new_val'}) + + with pytest.raises(ValidationError, match="Property 'my_inherited_property' is not updateable."): + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + + +@pytest.mark.asyncio +async def test_validate_patch_enforces_newly_required_properties_during_upgrade(resource_repo): + """ + Test that during an upgrade, newly required properties defined in the target template are enforced. + """ + old_template_dict = sample_resource_template() + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template_dict['properties']['new_req_prop'] = {'type': 'string'} + new_template_dict['required'].append('new_req_prop') + + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=new_template) + template_repo.enrich_template = MagicMock(side_effect=[old_template_dict, new_template_dict]) + + current_properties = { + 'display_name': 'Test Resource', + 'vm_size': 'small' + } + + # Omit new_req_prop from patch -> should raise ValidationError + patch = ResourcePatch(templateVersion='0.2.0', properties={'vm_size': 'large'}) + + with pytest.raises(ValidationError): + await resource_repo.validate_patch( + patch, + template_repo, + old_template, + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_get_all_property_keys_from_template_includes_allOf_conditional_properties(resource_repo): + """ + Test that _get_all_property_keys_from_template correctly collects properties defined + in conditional allOf blocks (both then and else clauses). + """ + template_dict = sample_resource_template() + template_dict['allOf'] = [ + { + "if": { + "properties": { + "vm_size": {"const": "small"} + } + }, + "then": { + "properties": { + "conditional_then_property": {"type": "string"} + } + }, + "else": { + "properties": { + "conditional_else_property": {"type": "string"} + } + } + } + ] + template = parse_obj_as(ResourceTemplate, template_dict) + + properties = resource_repo._get_all_property_keys_from_template(template) + assert "conditional_then_property" in properties + assert "conditional_else_property" in properties + assert "vm_size" in properties -def test_validate_resource_parameters_ignores_invalid_const_null_on_non_nullable_property(resource_repo): - template = { - "$id": "https://example.com/template_schema.json", + +@pytest.mark.asyncio +async def test_validate_patch_allows_partial_update_on_nested_object_with_required_fields(resource_repo): + """ + Test that validate_patch allows partial update of nested object properties even if the schema + defines nested required fields. + """ + template_repo = MagicMock() + template_dict = sample_resource_template() + template_dict["properties"]["parent_obj"] = { "type": "object", - "required": ["storage_account_redundancy"], + "updateable": True, + "required": ["child_a", "child_b"], "properties": { - "storage_account_redundancy": { - "type": "string", - "enum": ["GRS", "ZRS"], - "const": None + "child_a": {"type": "string"}, + "child_b": {"type": "string"} + } + } + template_repo.enrich_template = MagicMock(return_value=template_dict) + template = parse_obj_as(ResourceTemplate, template_dict) + + patch = ResourcePatch(properties={"parent_obj": {"child_b": "new_value"}}) + + # Should pass without raising ValidationError for missing sibling field child_a + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') +async def test_validate_patch_rejects_non_updateable_allOf_property(enrich_template_mock, resource_repo): + """ + Test that validate_patch rejects attempts to patch non-updateable conditional properties inside allOf + """ + template_dict = sample_resource_template() + template_dict['allOf'] = [ + { + "if": {"properties": {"vm_size": {"const": "small"}}}, + "then": {"properties": {"secret_conditional_field": {"type": "string", "updateable": False}}} + } + ] + template = parse_obj_as(ResourceTemplate, template_dict) + enrich_template_mock.return_value = template_dict + + template_repo = MagicMock() + template_repo.enrich_template = enrich_template_mock + + # Resource has vm_size small, patching secret_conditional_field should be denied since updateable: False + patch = ResourcePatch(properties={"secret_conditional_field": "new_secret"}) + + with pytest.raises(ValidationError): + await resource_repo.validate_patch(patch, template_repo, template, strings.RESOURCE_ACTION_UPDATE) + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') +async def test_validate_patch_allows_new_nested_property_under_existing_object_during_upgrade(enrich_template_mock, get_template_mock, resource_repo): + """ + Test that during an upgrade, adding a newly-introduced nested property inside an existing object + passes validation even if the parent object is not marked updateable. + """ + old_template_dict = sample_resource_template() + old_template_dict['properties']['parent_object'] = { + 'type': 'object', + 'updateable': False, + 'properties': { + 'existing_child': {'type': 'string', 'updateable': False} + } + } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template_dict['properties']['parent_object']['properties']['new_child'] = { + 'type': 'string', + 'updateable': False + } + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + get_template_mock.return_value = new_template + enrich_template_mock.side_effect = [old_template_dict, new_template_dict] + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = enrich_template_mock + + current_properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'parent_object': {'existing_child': 'old_val'} + } + + # Patching new_child during upgrade should be allowed + patch = ResourcePatch(templateVersion='0.2.0', properties={'parent_object': {'new_child': 'new_val'}}) + await resource_repo.validate_patch(patch, template_repo, old_template, strings.RESOURCE_ACTION_UPDATE, current_properties=current_properties) + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +async def test_validate_patch_rejects_modifying_existing_nested_non_updateable_property_during_upgrade(get_template_mock, resource_repo): + """ + Test that during an upgrade, attempting to modify an existing non-updateable nested property + fails validation. + """ + old_template_dict = sample_resource_template() + old_template_dict['properties']['parent_object'] = { + 'type': 'object', + 'updateable': False, + 'properties': { + 'existing_child': {'type': 'string', 'updateable': False} + } + } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + get_template_mock.return_value = new_template + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = MagicMock(return_value=new_template_dict) + + current_properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'parent_object': {'existing_child': 'old_val'} + } + + # Attempting to change existing_child during upgrade should raise ValidationError + patch = ResourcePatch(templateVersion='0.2.0', properties={'parent_object': {'existing_child': 'modified_val'}}) + with pytest.raises(ValidationError): + await resource_repo.validate_patch(patch, template_repo, old_template, strings.RESOURCE_ACTION_UPDATE, current_properties=current_properties) + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +async def test_patch_resource_removes_nested_properties_on_upgrade(get_template_mock, resource_repo, resource_history_repo): + """ + Test that patch_resource removes nested properties that were deleted in the new template version. + """ + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + + old_template_dict = sample_resource_template() + old_template_dict['properties']['parent_object'] = { + 'type': 'object', + 'properties': { + 'kept_child': {'type': 'string'}, + 'removed_child': {'type': 'string'} + } + } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + del new_template_dict['properties']['parent_object']['properties']['removed_child'] + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + resource_repo.validate_template_version_patch = AsyncMock(return_value=new_template) + + get_template_mock.return_value = new_template + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = MagicMock(return_value=new_template_dict) + + user = create_test_user() + resource = sample_resource() + resource.properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'parent_object': { + 'kept_child': 'val1', + 'removed_child': 'val2' + } + } + + patch = ResourcePatch(templateVersion='0.2.0', properties={}) + + _, returned_template = await resource_repo.patch_resource( + resource, + patch, + old_template, + "some-etag", + template_repo, + resource_history_repo, + user, + strings.RESOURCE_ACTION_UPDATE + ) + + assert 'removed_child' not in resource.properties['parent_object'] + assert resource.properties['parent_object']['kept_child'] == 'val1' + assert returned_template == new_template + + +@pytest.mark.asyncio +async def test_patch_resource_allows_full_array_items_when_adding_property(resource_repo, resource_history_repo): + """Full array items sent by an upgrade remain valid when only one item property is new.""" + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + + old_template_dict = sample_resource_template() + old_template_dict['properties']['redirect_uris'] = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'} } } } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) - resource_input = { - "properties": { - "storage_account_redundancy": "GRS" + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template_dict['properties']['redirect_uris']['items']['properties']['value'] = { + 'type': 'string', + 'default': 'https://example.test' + } + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + resource_repo.validate_template_version_patch = AsyncMock(return_value=new_template) + template_repo = MagicMock() + + def enrich_template(template, is_update=False): + enriched = copy.deepcopy(new_template_dict if template.version == '0.2.0' else old_template_dict) + if not enriched.get('allOf'): + enriched.pop('allOf', None) + return enriched + + template_repo.enrich_template.side_effect = enrich_template + + user = create_test_user() + resource = sample_resource() + resource.properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'redirect_uris': [{'name': 'primary'}] + } + patch = ResourcePatch( + templateVersion='0.2.0', + properties={'redirect_uris': [{'name': 'primary', 'value': 'https://example.test'}]} + ) + + updated_resource, returned_template = await resource_repo.patch_resource( + resource, + patch, + old_template, + 'some-etag', + template_repo, + resource_history_repo, + user, + strings.RESOURCE_ACTION_UPDATE + ) + + assert updated_resource.properties['redirect_uris'] == [{'name': 'primary', 'value': 'https://example.test'}] + assert returned_template == new_template + + +@pytest.mark.asyncio +async def test_patch_resource_preserves_omitted_array_item_fields_during_upgrade(resource_repo, resource_history_repo): + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + + old_template_dict = sample_resource_template() + old_template_dict['properties']['redirect_uris'] = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': {'protected': {'type': 'string'}}, + 'required': ['protected'] } } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) - # Should not raise because const:null is incompatible with this non-nullable property. - resource_repo._validate_resource_parameters(resource_input, template) + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template_dict['properties']['redirect_uris']['items']['properties']['new_field'] = { + 'type': 'string', + 'updateable': True, + } + new_template = parse_obj_as(ResourceTemplate, new_template_dict) - normalized_template = resource_repo._normalize_template_schema(template) - assert "const" not in normalized_template["properties"]["storage_account_redundancy"] - assert template["properties"]["storage_account_redundancy"]["const"] is None + resource_repo.validate_template_version_patch = AsyncMock(return_value=new_template) + template_repo = MagicMock() + template_repo.enrich_template.side_effect = lambda template, is_update=False: copy.deepcopy( + new_template_dict if template.version == '0.2.0' else old_template_dict + ) + resource = sample_resource() + resource.properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'redirect_uris': [{'protected': 'keep-me'}], + } + patch = ResourcePatch( + templateVersion='0.2.0', + properties={'redirect_uris': [{'new_field': 'new-value'}]}, + ) -def test_validate_resource_parameters_keeps_nullable_const_null(resource_repo): - template = { - "$id": "https://example.com/template_schema.json", - "type": "object", - "required": ["nullable_const"], - "properties": { - "nullable_const": { - "const": None + await resource_repo.patch_resource( + resource, + patch, + old_template, + 'some-etag', + template_repo, + resource_history_repo, + create_test_user(), + strings.RESOURCE_ACTION_UPDATE, + ) + + assert resource.properties['redirect_uris'] == [ + {'protected': 'keep-me', 'new_field': 'new-value'} + ] + + +@pytest.mark.asyncio +async def test_validate_patch_allows_newly_required_array_item_property_during_upgrade(resource_repo): + old_template_dict = sample_resource_template() + old_template_dict['properties']['redirect_uris'] = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'value': {'type': 'string'} } } } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) - resource_input = { - "properties": { - "nullable_const": None + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template_dict['properties']['redirect_uris']['items']['required'] = ['value'] + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = AsyncMock(return_value=new_template) + template_repo.enrich_template = MagicMock(side_effect=[old_template_dict, new_template_dict]) + + patch = ResourcePatch( + templateVersion='0.2.0', + properties={'redirect_uris': [{'name': 'primary', 'value': 'https://example.test'}]} + ) + current_properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'redirect_uris': [{'name': 'primary'}] + } + + await resource_repo.validate_patch( + patch, + template_repo, + old_template, + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties + ) + + +@pytest.mark.asyncio +async def test_patch_resource_removes_deleted_array_item_property_from_all_items(resource_repo, resource_history_repo): + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + + old_template_dict = sample_resource_template() + old_template_dict['properties']['redirect_uris'] = { + 'type': 'array', + 'items': { + 'type': 'object', + 'properties': { + 'name': {'type': 'string'}, + 'value': {'type': 'string'} + } + } + } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + del new_template_dict['properties']['redirect_uris']['items']['properties']['value'] + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + resource_repo.validate_template_version_patch = AsyncMock(return_value=new_template) + template_repo = MagicMock() + + def enrich_template(template, is_update=False): + return copy.deepcopy(new_template_dict if template.version == '0.2.0' else old_template_dict) + + template_repo.enrich_template = MagicMock(side_effect=enrich_template) + + resource = sample_resource() + resource.properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'redirect_uris': [ + {'name': 'primary', 'value': 'https://primary.example'}, + {'name': 'secondary', 'value': 'https://secondary.example'} + ] + } + patch = ResourcePatch(templateVersion='0.2.0', properties={}) + + await resource_repo.patch_resource( + resource, + patch, + old_template, + 'some-etag', + template_repo, + resource_history_repo, + create_test_user(), + strings.RESOURCE_ACTION_UPDATE + ) + + assert resource.properties['redirect_uris'] == [ + {'name': 'primary'}, + {'name': 'secondary'} + ] + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') +async def test_validate_patch_passes_parent_service_name_for_user_resources(enrich_template_mock, get_template_mock, resource_repo): + """ + Test that during a template upgrade for a UserResource, parent_service_name is passed + to get_template_by_name_and_version. + """ + old_template_dict = sample_resource_template() + old_template_dict['resourceType'] = ResourceType.UserResource + old_template_dict['parentWorkspaceService'] = 'parent-service-name' + old_template = parse_obj_as(UserResourceTemplate, old_template_dict) + + new_template = copy.deepcopy(old_template) + new_template.version = '0.2.0' + get_template_mock.return_value = new_template + enrich_template_mock.return_value = old_template_dict + + # Mock template repository + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = enrich_template_mock + + patch = ResourcePatch(templateVersion='0.2.0', properties={}) + current_properties = {'title': 'Test Title', 'os_image': 'Windows 11', 'vm_size': 'small'} + await resource_repo.validate_patch(patch, template_repo, old_template, strings.RESOURCE_ACTION_UPDATE, current_properties=current_properties) + + get_template_mock.assert_called_once_with( + old_template.name, + '0.2.0', + ResourceType.UserResource, + parent_service_name='parent-service-name' + ) + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceRepository.create') +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') +async def test_patch_resource_passes_parent_service_name_for_user_resources(enrich_template_mock, get_template_mock, create_repo_mock, resource_repo, resource_history_repo): + """ + Test that patch_resource passes parent_service_name to get_template_by_name_and_version + when upgrading a UserResource. + """ + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + + mock_parent_repo = AsyncMock() + mock_parent_repo.get_resource_by_id.return_value = MagicMock(templateName='parent-service-name') + create_repo_mock.return_value = mock_parent_repo + + old_template_dict = sample_resource_template() + old_template_dict['resourceType'] = ResourceType.UserResource + old_template_dict['parentWorkspaceService'] = 'parent-service-name' + old_template = parse_obj_as(UserResourceTemplate, old_template_dict) + + new_template = copy.deepcopy(old_template) + new_template.version = '0.2.0' + get_template_mock.return_value = new_template + enrich_template_mock.return_value = old_template_dict + + # Mock template repository + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = enrich_template_mock + + user = create_test_user() + resource_dict = sample_resource().model_dump() + resource_dict['resourceType'] = ResourceType.UserResource + resource_dict['parentWorkspaceServiceId'] = 'parent-service-id' + resource_dict['templateVersion'] = '0.1.0' + resource_dict['properties'] = {'title': 'Test Title', 'os_image': 'Windows 11', 'vm_size': 'small'} + resource = parse_obj_as(UserResource, resource_dict) + + resource_patch = ResourcePatch(templateVersion='0.2.0', properties={}) + + await resource_repo.patch_resource( + resource, + resource_patch, + old_template, + "some-etag", + template_repo, + resource_history_repo, + user, + strings.RESOURCE_ACTION_UPDATE + ) + + get_template_mock.assert_called_once_with( + resource.templateName, + '0.2.0', + ResourceType.UserResource, + 'parent-service-name' + ) + + +def test_deep_dict_update_preserves_nested_siblings(resource_repo): + target = { + "display_name": "My Resource", + "parent_object": { + "sibling_field": "existing_value", + "target_field": "old_value" } } + patch = { + "parent_object": { + "target_field": "new_value", + "added_field": "added_value" + } + } + resource_repo._deep_dict_update(target, patch) + assert target == { + "display_name": "My Resource", + "parent_object": { + "sibling_field": "existing_value", + "target_field": "new_value", + "added_field": "added_value" + } + } + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +@patch('db.repositories.resources.ResourceTemplateRepository.enrich_template') +async def test_validate_patch_allows_absent_target_required_non_updateable_property_on_upgrade(enrich_template_mock, get_template_mock, resource_repo): + """ + Test that an optional non-updateable property from the old template that becomes required in + the target version can be initially populated on upgrade if absent from current properties. + """ + old_template_dict = sample_resource_template() + old_template_dict['properties']['newly_required'] = { + 'type': 'string', + 'title': 'Newly Required Non-Updateable', + 'updateable': False + } + old_template = parse_obj_as(ResourceTemplate, old_template_dict) - # Explicit nullable const usage should still validate and remain in schema. - resource_repo._validate_resource_parameters(resource_input, template) + target_template_dict = copy.deepcopy(old_template_dict) + target_template_dict['version'] = '0.2.0' + target_template_dict['required'].append('newly_required') + target_template = parse_obj_as(ResourceTemplate, target_template_dict) + + get_template_mock.return_value = target_template + enrich_template_mock.return_value = target_template_dict + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = enrich_template_mock + + # Resource current properties omit 'newly_required' + current_properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small' + } + + # Patch supplies 'newly_required' during upgrade + patch = ResourcePatch(templateVersion='0.2.0', properties={'newly_required': 'initial_value'}) + + # Validation should succeed without throwing ValidationError + await resource_repo.validate_patch( + patch, + template_repo, + old_template, + strings.RESOURCE_ACTION_UPDATE, + current_properties=current_properties, + target_template=target_template + ) + + +@pytest.mark.asyncio +@patch('db.repositories.resources.ResourceTemplateRepository.get_template_by_name_and_version') +async def test_patch_resource_preserves_runtime_properties_on_upgrade(get_template_mock, resource_repo, resource_history_repo): + """ + Test that patch_resource preserves API-injected/runtime properties (e.g. workspace_subscription_id) + that are absent from both current and target template schemas. + """ + resource_repo.update_item_with_etag = AsyncMock(return_value=None) + resource_history_repo.create_resource_history_item = AsyncMock() + + old_template_dict = sample_resource_template() + old_template = parse_obj_as(ResourceTemplate, old_template_dict) + + new_template_dict = copy.deepcopy(old_template_dict) + new_template_dict['version'] = '0.2.0' + new_template = parse_obj_as(ResourceTemplate, new_template_dict) + + resource_repo.validate_template_version_patch = AsyncMock(return_value=new_template) + get_template_mock.return_value = new_template + + template_repo = MagicMock() + template_repo.get_template_by_name_and_version = get_template_mock + template_repo.enrich_template = MagicMock(return_value=new_template_dict) + + user = create_test_user() + resource = sample_resource() + resource.properties = { + 'title': 'Test Title', + 'os_image': 'Windows 11', + 'vm_size': 'small', + 'workspace_subscription_id': 'sub-123-abc' # Runtime property not in template schema + } + + patch = ResourcePatch(templateVersion='0.2.0', properties={}) + + updated_resource, _ = await resource_repo.patch_resource( + resource, + patch, + old_template, + "some-etag", + template_repo, + resource_history_repo, + user, + strings.RESOURCE_ACTION_UPDATE + ) - normalized_template = resource_repo._normalize_template_schema(template) - assert normalized_template["properties"]["nullable_const"]["const"] is None + assert updated_resource.properties.get('workspace_subscription_id') == 'sub-123-abc' diff --git a/ui/app/package-lock.json b/ui/app/package-lock.json index a4ca2f76f..23e92964c 100644 --- a/ui/app/package-lock.json +++ b/ui/app/package-lock.json @@ -1,12 +1,12 @@ { "name": "tre-ui", - "version": "0.8.31", + "version": "0.9.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "tre-ui", - "version": "0.8.31", + "version": "0.9.0", "dependencies": { "@azure/msal-browser": "^2.35.0", "@azure/msal-react": "^1.5.12", diff --git a/ui/app/package.json b/ui/app/package.json index ea04af358..04f03e576 100644 --- a/ui/app/package.json +++ b/ui/app/package.json @@ -1,6 +1,6 @@ { "name": "tre-ui", - "version": "0.8.31", + "version": "0.9.0", "private": true, "type": "module", "dependencies": { diff --git a/ui/app/src/components/shared/ConfirmUpgradeResource.test.tsx b/ui/app/src/components/shared/ConfirmUpgradeResource.test.tsx index fcb62cb3a..48d53c69a 100644 --- a/ui/app/src/components/shared/ConfirmUpgradeResource.test.tsx +++ b/ui/app/src/components/shared/ConfirmUpgradeResource.test.tsx @@ -1,8 +1,15 @@ import React from "react"; import { describe, it, expect, vi, beforeEach } from "vitest"; -import { render, screen, fireEvent, waitFor, createPartialFluentUIMock } from "../../test-utils"; +import { act, render, screen, fireEvent, waitFor, createPartialFluentUIMock } from "../../test-utils"; import { ConfirmUpgradeResource } from "./ConfirmUpgradeResource"; +import { + matchesIfCondition, + getAllPropertyKeys, + getSchemaPropertyFromProperties, + buildReducedSchema, +} from "../../utils/schemaUpgradeUtils"; import { Resource, AvailableUpgrade } from "../../models/resource"; +import { UserResource } from "../../models/userResource"; import { ResourceType } from "../../models/resourceType"; import { WorkspaceContext } from "../../contexts/WorkspaceContext"; import { CostResource } from "../../models/costs"; @@ -11,9 +18,29 @@ import { CostResource } from "../../models/costs"; const mockApiCall = vi.fn(); const mockDispatch = vi.fn(); +// Mock template schemas +const mockCurrentTemplateSchema = { + properties: { + display_name: { type: "string" }, + resource_key: { type: "string" }, + existing_property: { type: "string" }, + }, + required: ["display_name"], +}; + +const mockNewTemplateSchema = { + properties: { + display_name: { type: "string" }, + resource_key: { type: "string" }, + new_property: { type: "string", default: "default_value" }, + }, + required: ["display_name", "new_property"], + uiSchema: {}, +}; + vi.mock("../../hooks/useAuthApiCall", () => ({ useAuthApiCall: () => mockApiCall, - HttpMethod: { Patch: "PATCH" }, + HttpMethod: { Patch: "PATCH", Get: "GET" }, ResultType: { JSON: "JSON" }, })); @@ -23,7 +50,7 @@ vi.mock("../../hooks/customReduxHooks", () => ({ vi.mock("../shared/notifications/operationsSlice", () => ({ addUpdateOperation: vi.fn(), - default: (state: { items: unknown[] } = { items: [] }) => state, + default: (state: any = { items: [] }) => state, })); // Mock FluentUI components using centralized mocks @@ -42,15 +69,14 @@ vi.mock("@fluentui/react", async () => { "MessageBar", "MessageBarType", "Icon", + "TextField", ]), }; }); -vi.mock("./ExceptionLayout", () => { - const ExceptionLayout = ({ e }: any) =>