diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/en.json b/hat/assets/js/apps/Iaso/domains/app/translations/en.json index c26af2e845..0ab30e8494 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/en.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/en.json @@ -541,6 +541,7 @@ "iaso.instance.dialog.lockSuccess": "Lock added on the submission", "iaso.instance.dialog.unlockedWarning": "Are you sure you want to unlock the submission ?", "iaso.instance.disableColumnSelectionMessage": "Apply search to be able to select visible columns", + "iaso.instance.downloadAll": "Download all", "iaso.instance.duplicate": "Compare submissions", "iaso.instance.editGpsFromInstanceWarning": "This operation can still be undone", "iaso.instance.end": "End period", diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/es.json b/hat/assets/js/apps/Iaso/domains/app/translations/es.json index 0ea4c9bbc3..2f43435d4b 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/es.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/es.json @@ -418,6 +418,7 @@ "iaso.instance.dialog.lockSuccess": "Bloqueo agregado al envío", "iaso.instance.dialog.unlockedWarning": "¿Está seguro de que desea desbloquear el envío?", "iaso.instance.disableColumnSelectionMessage": "Aplique la búsqueda para poder seleccionar columnas visibles", + "iaso.instance.downloadAll": "Descargar todo", "iaso.instance.duplicate": "Comparar envíos", "iaso.instance.editGpsFromInstanceWarning": "Esta operación aún se puede deshacer", "iaso.instance.error": "No se puede encontrar un envío con este ID", diff --git a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json index ed4e1e5050..d9366ddfd6 100644 --- a/hat/assets/js/apps/Iaso/domains/app/translations/fr.json +++ b/hat/assets/js/apps/Iaso/domains/app/translations/fr.json @@ -542,6 +542,7 @@ "iaso.instance.dialog.lockSuccess": "Verrou placé sur la soumission", "iaso.instance.dialog.unlockedWarning": "Êtes-vous sûr de vouloir déverrouiller la soumission ?", "iaso.instance.disableColumnSelectionMessage": "Lancer la recherche pour pouvoir sélectionner les colonnes visibles", + "iaso.instance.downloadAll": "Tout télécharger", "iaso.instance.duplicate": "Comparer les soumissions", "iaso.instance.editGpsFromInstanceWarning": "Cette opération peut toujours être annulée.", "iaso.instance.end": "End period", diff --git a/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFilesListComponent.tsx b/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFilesListComponent.tsx index 22ddbe164d..d3fd1e43d9 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFilesListComponent.tsx +++ b/hat/assets/js/apps/Iaso/domains/instances/components/InstancesFilesListComponent.tsx @@ -1,6 +1,6 @@ import React, { FunctionComponent, useMemo, useState } from 'react'; -import { Box, Tab, Tabs } from '@mui/material'; +import { Box, Button, Tab, Tabs } from '@mui/material'; import { makeStyles } from '@mui/styles'; import { useSafeIntl } from 'bluesquare-components'; @@ -238,6 +238,12 @@ const InstancesFilesList: FunctionComponent = ({ } /> )} + + ); }; diff --git a/hat/assets/js/apps/Iaso/domains/instances/messages.ts b/hat/assets/js/apps/Iaso/domains/instances/messages.ts index d0b91fbea3..ee20c0278c 100644 --- a/hat/assets/js/apps/Iaso/domains/instances/messages.ts +++ b/hat/assets/js/apps/Iaso/domains/instances/messages.ts @@ -1006,6 +1006,10 @@ const MESSAGES = defineMessages({ id: 'iaso.instance.org_unit_status', defaultMessage: 'Org Unit Status', }, + downloadAll: { + id: 'iaso.instance.downloadAll', + defaultMessage: 'Download all', + }, }); export default MESSAGES; diff --git a/iaso/api/instances/views.py b/iaso/api/instances/views.py index 06b8913319..28622065b2 100644 --- a/iaso/api/instances/views.py +++ b/iaso/api/instances/views.py @@ -48,6 +48,7 @@ InstanceSerializer, UnlockSerializer, ) +from iaso.api.instances.zip import generate_zip from iaso.api.org_units import HasCreateOrgUnitPermission from iaso.api.permission_checks import AuthenticationEnforcedPermission from iaso.engine.validation_workflow import ValidationWorkflowEngine @@ -1019,6 +1020,20 @@ def instance_logs(self, request, pk=None, logId=None): log_dict["form_descriptor"] = instance.form_version.form_descriptor if instance.form_version else None return Response(log_dict) + @action(["GET"], detail=True) + def download_attachments(self, request, pk=None) -> StreamingHttpResponse: + instance = get_object_or_404( + Instance.objects.filter_for_user(request.user).prefetch_related("instancefile_set").filter(pk=pk) + ) + return StreamingHttpResponse( + streaming_content=generate_zip(instance), + headers={ + "Content-Type": "application/zip", + "Content-Disposition": f'attachment; filename="{instance.name}-{instance.id}.zip"', + "Access-Control-Expose-Headers": "Content-Disposition", + }, + ) + def find_entity(account: Account, entity_uuid: str, entity_type_id: Optional[int] = None) -> Entity: # In case of duplicate UUIDs in the database, only allow 1 non-deleted one. diff --git a/iaso/api/instances/zip.py b/iaso/api/instances/zip.py new file mode 100644 index 0000000000..eff8302cf9 --- /dev/null +++ b/iaso/api/instances/zip.py @@ -0,0 +1,42 @@ +from collections.abc import AsyncGenerator, AsyncIterable +from stat import S_IFREG + +from stream_zip import NO_COMPRESSION_64, AsyncMemberFile, async_stream_zip +from wrapt import sync_to_async + +from iaso.models import Instance, InstanceFile + + +async def generate_zip(instance: Instance) -> AsyncGenerator[bytes]: + """ + This function generates a zip and starts sending it to client chunk by + chunk as soon as it's being created. + + The zip file contains all the files attached to the instance + """ + + async def generate_files_to_add_in_zip() -> AsyncIterable[AsyncMemberFile]: + mode = S_IFREG | 0o600 + instance_files = await sync_to_async(list)(instance.instancefile_set.all()) + for instance_file in instance_files: + + async def file_content_iterator(file: InstanceFile) -> AsyncIterable[bytes]: + with file.file.open("rb") as f: + while True: + chunk: bytes = f.read(65536) # 65536 is the default int value for async_stream_zip's chunk_size + if not chunk: + break + yield chunk + + yield ( + instance_file.name, + instance_file.created_at, + mode, + NO_COMPRESSION_64, + file_content_iterator(instance_file), + ) + + zipped_chunks = async_stream_zip(generate_files_to_add_in_zip()) + + async for zipped_chunk in zipped_chunks: + yield zipped_chunk diff --git a/iaso/tests/api/instances/test_instances.py b/iaso/tests/api/instances/test_instances.py index e05f8f87f0..1d84295442 100644 --- a/iaso/tests/api/instances/test_instances.py +++ b/iaso/tests/api/instances/test_instances.py @@ -1,6 +1,9 @@ +import asyncio import datetime +import io import json import typing +import zipfile from decimal import Decimal from unittest import mock @@ -12,10 +15,12 @@ from django.contrib.gis.geos import Point from django.core.files import File from django.core.files.base import ContentFile +from django.core.files.uploadedfile import UploadedFile from django.db import connection from django.test.utils import CaptureQueriesContext from django.utils import timezone from django.utils.timezone import now +from prompt_toolkit.eventloop import async_generator from rest_framework import status from hat.api.export_utils import timestamp_to_utc_datetime @@ -3894,6 +3899,38 @@ def test_instances_list_fields_filter_order_combinations(self): for item in instances: self.assertEqual(set(item.keys()), case["expected_fields"]) + def test_download_attachments_not_authenticated(self): + instance = self.create_form_instance(form=self.form_1, project=self.project, org_unit=self.ou_top_1) + + response = self.client.get(f"/api/instances/{instance.pk}/download_attachments/") + self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_download_attachments(self): + self.client.force_authenticate(self.yoda) + instance = self.create_form_instance(form=self.form_1, project=self.project, org_unit=self.ou_top_1) + with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file: + m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test1.jpg") + m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test2.webp") + m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test3.webp") + m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test4.pdf") + + response = self.client.get(f"/api/instances/{instance.pk}/download_attachments/") + self.assertEqual(response.status_code, status.HTTP_200_OK) + + async def streaming_content_to_bytes(content: async_generator) -> io.BytesIO: + bytes = io.BytesIO() + async for x in content: + bytes.write(x) + return bytes + + content = asyncio.run(streaming_content_to_bytes(response.streaming_content)) + with zipfile.ZipFile(content, "a", zipfile.ZIP_DEFLATED, False) as zf: + self.assertEqual(4, len(zf.namelist())) + self.assertEqual("test1.jpg", zf.namelist()[0]) + self.assertEqual("test2.webp", zf.namelist()[1]) + self.assertEqual("test3.webp", zf.namelist()[2]) + self.assertEqual("test4.pdf", zf.namelist()[3]) + def assertInstanceListContainsStrictly(self, api_response, expected_instances): try: self.assertEqual(api_response.status_code, status.HTTP_200_OK) diff --git a/pyproject.toml b/pyproject.toml index f7fbaab36c..972f2c32d5 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,9 +87,11 @@ dependencies = [ "simpleeval==1.0.3", # https://github.com/danthedeckie/simpleeval/tags "snaql==0.4.2", # trypelim - https://github.com/zaiev/snaql/tags "sqlalchemy==2.0.23", # https://github.com/sqlalchemy/sqlalchemy/tags + "stream-zip==0.0.84", # https://stream-zip.docs.trade.gov.uk/ "tsp-solver2==0.4.1", # trypelim - https://github.com/dmishin/tsp-solver/tags "unidecode==1.3.7", # https://github.com/avian2/unidecode/tags "urllib3<2", + "wrapt>=2.2.1", "xlsxwriter==3.1.9", # https://github.com/jmcnamara/XlsxWriter/tags ] # ------------------------------------------------------------------------------ diff --git a/uv.lock b/uv.lock index 69fafe5d4b..d64f7b172d 100644 --- a/uv.lock +++ b/uv.lock @@ -1889,9 +1889,11 @@ dependencies = [ { name = "simpleeval" }, { name = "snaql" }, { name = "sqlalchemy" }, + { name = "stream-zip" }, { name = "tsp-solver2" }, { name = "unidecode" }, { name = "urllib3" }, + { name = "wrapt" }, { name = "xlsxwriter" }, ] @@ -2020,9 +2022,11 @@ requires-dist = [ { name = "simpleeval", specifier = "==1.0.3" }, { name = "snaql", specifier = "==0.4.2" }, { name = "sqlalchemy", specifier = "==2.0.23" }, + { name = "stream-zip", specifier = "==0.0.84" }, { name = "tsp-solver2", specifier = "==0.4.1" }, { name = "unidecode", specifier = "==1.3.7" }, { name = "urllib3", specifier = "<2" }, + { name = "wrapt", specifier = ">=2.2.1" }, { name = "xlsxwriter", specifier = "==3.1.9" }, ] @@ -3767,6 +3771,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] +[[package]] +name = "stream-zip" +version = "0.0.84" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycryptodome" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1b/f2/bc3d35c70735fa118eb3a49af8ce786863df87c910c14483daca5d056613/stream_zip-0.0.84.tar.gz", hash = "sha256:32ba07c3c7fe947c0224a9f8e4a228f14080e01b17d8ffc78c16a6043b1fba12", size = 10074, upload-time = "2026-02-04T08:37:41.417Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/8c/e11decad4ab23780503666170b09a8d0d07b15045b522b0ba94fc5df257f/stream_zip-0.0.84-py3-none-any.whl", hash = "sha256:4e3a68aba1aba643035c4b9e219a61de519752693ff727c3268581a1e70f1376", size = 9890, upload-time = "2026-02-04T08:37:39.862Z" }, +] + [[package]] name = "tblib" version = "3.0.0"