[NERISP2627-111] Add a way to download all instance files

hat/assets/js/apps/Iaso/domains/app/translations/en.json

@@ -541,6 +541,7 @@
     "iaso.instance.dialog.lockSuccess": "Lock added on the submission",
     "iaso.instance.dialog.unlockedWarning": "Are you sure you want to unlock the submission ?",
     "iaso.instance.disableColumnSelectionMessage": "Apply search to be able to select visible columns",
+    "iaso.instance.downloadAll": "Download all",
     "iaso.instance.duplicate": "Compare submissions",
     "iaso.instance.editGpsFromInstanceWarning": "This operation can still be undone",
     "iaso.instance.end": "End period",

hat/assets/js/apps/Iaso/domains/app/translations/es.json

@@ -418,6 +418,7 @@
     "iaso.instance.dialog.lockSuccess": "Bloqueo agregado al envío",
     "iaso.instance.dialog.unlockedWarning": "¿Está seguro de que desea desbloquear el envío?",
     "iaso.instance.disableColumnSelectionMessage": "Aplique la búsqueda para poder seleccionar columnas visibles",
+    "iaso.instance.downloadAll": "Descargar todo",
     "iaso.instance.duplicate": "Comparar envíos",
     "iaso.instance.editGpsFromInstanceWarning": "Esta operación aún se puede deshacer",
     "iaso.instance.error": "No se puede encontrar un envío con este ID",

hat/assets/js/apps/Iaso/domains/app/translations/fr.json

@@ -542,6 +542,7 @@
     "iaso.instance.dialog.lockSuccess": "Verrou placé sur la soumission",
     "iaso.instance.dialog.unlockedWarning": "Êtes-vous sûr de vouloir déverrouiller la soumission ?",
     "iaso.instance.disableColumnSelectionMessage": "Lancer la recherche pour pouvoir sélectionner les colonnes visibles",
+    "iaso.instance.downloadAll": "Tout télécharger",
     "iaso.instance.duplicate": "Comparer les soumissions",
     "iaso.instance.editGpsFromInstanceWarning": "Cette opération peut toujours être annulée.",
     "iaso.instance.end": "End period",

hat/assets/js/apps/Iaso/domains/instances/components/InstancesFilesListComponent.tsx

@@ -1,6 +1,6 @@
 import React, { FunctionComponent, useMemo, useState } from 'react';
 
-import { Box, Tab, Tabs } from '@mui/material';
+import { Box, Button, Tab, Tabs } from '@mui/material';
 import { makeStyles } from '@mui/styles';
 
 import { useSafeIntl } from 'bluesquare-components';
@@ -238,6 +238,12 @@ const InstancesFilesList: FunctionComponent<Props> = ({
                     }
                 />
             )}
+
+            <Button
+                href={`/api/instances/${currentInstance?.id}/download_attachments/`}
+            >
+                {formatMessage(MESSAGES.downloadAll)}
+            </Button>
         </section>
     );
 };

hat/assets/js/apps/Iaso/domains/instances/messages.ts

@@ -1006,6 +1006,10 @@ const MESSAGES = defineMessages({
         id: 'iaso.instance.org_unit_status',
         defaultMessage: 'Org Unit Status',
     },
+    downloadAll: {
+        id: 'iaso.instance.downloadAll',
+        defaultMessage: 'Download all',
+    },
 });
 
 export default MESSAGES;

iaso/api/instances/views.py

@@ -48,6 +48,7 @@
     InstanceSerializer,
     UnlockSerializer,
 )
+from iaso.api.instances.zip import generate_zip
 from iaso.api.org_units import HasCreateOrgUnitPermission
 from iaso.api.permission_checks import AuthenticationEnforcedPermission
 from iaso.engine.validation_workflow import ValidationWorkflowEngine
@@ -1019,6 +1020,20 @@ def instance_logs(self, request, pk=None, logId=None):
         log_dict["form_descriptor"] = instance.form_version.form_descriptor if instance.form_version else None
         return Response(log_dict)
 
+    @action(["GET"], detail=True)
+    def download_attachments(self, request, pk=None) -> StreamingHttpResponse:
+        instance = get_object_or_404(
+            Instance.objects.filter_for_user(request.user).prefetch_related("instancefile_set").filter(pk=pk)
+        )
+        return StreamingHttpResponse(
+            streaming_content=generate_zip(instance),
+            headers={
+                "Content-Type": "application/zip",
+                "Content-Disposition": f'attachment; filename="{instance.name}-{instance.id}.zip"',
+                "Access-Control-Expose-Headers": "Content-Disposition",
+            },
+        )
+
 
 def find_entity(account: Account, entity_uuid: str, entity_type_id: Optional[int] = None) -> Entity:
     # In case of duplicate UUIDs in the database, only allow 1 non-deleted one.

iaso/api/instances/zip.py

@@ -0,0 +1,42 @@
+from collections.abc import AsyncGenerator, AsyncIterable
+from stat import S_IFREG
+
+from stream_zip import NO_COMPRESSION_64, AsyncMemberFile, async_stream_zip
+from wrapt import sync_to_async
+
+from iaso.models import Instance, InstanceFile
+
+
+async def generate_zip(instance: Instance) -> AsyncGenerator[bytes]:
+    """
+    This function generates a zip and starts sending it to client chunk by
+    chunk as soon as it's being created.
+
+    The zip file contains all the files attached to the instance
+    """
+
+    async def generate_files_to_add_in_zip() -> AsyncIterable[AsyncMemberFile]:
+        mode = S_IFREG | 0o600
+        instance_files = await sync_to_async(list)(instance.instancefile_set.all())
+        for instance_file in instance_files:
+
+            async def file_content_iterator(file: InstanceFile) -> AsyncIterable[bytes]:
+                with file.file.open("rb") as f:
+                    while True:
+                        chunk: bytes = f.read(65536)  # 65536 is the default int value for async_stream_zip's chunk_size
+                        if not chunk:
+                            break
+                        yield chunk
+
+            yield (
+                instance_file.name,
+                instance_file.created_at,
+                mode,
+                NO_COMPRESSION_64,
+                file_content_iterator(instance_file),
+            )
+
+    zipped_chunks = async_stream_zip(generate_files_to_add_in_zip())
+
+    async for zipped_chunk in zipped_chunks:
+        yield zipped_chunk

iaso/tests/api/instances/test_instances.py

@@ -1,6 +1,9 @@
+import asyncio
 import datetime
+import io
 import json
 import typing
+import zipfile
 
 from decimal import Decimal
 from unittest import mock
@@ -12,10 +15,12 @@
 from django.contrib.gis.geos import Point
 from django.core.files import File
 from django.core.files.base import ContentFile
+from django.core.files.uploadedfile import UploadedFile
 from django.db import connection
 from django.test.utils import CaptureQueriesContext
 from django.utils import timezone
 from django.utils.timezone import now
+from prompt_toolkit.eventloop import async_generator
 from rest_framework import status
 
 from hat.api.export_utils import timestamp_to_utc_datetime
@@ -3894,6 +3899,38 @@ def test_instances_list_fields_filter_order_combinations(self):
                 for item in instances:
                     self.assertEqual(set(item.keys()), case["expected_fields"])
 
+    def test_download_attachments_not_authenticated(self):
+        instance = self.create_form_instance(form=self.form_1, project=self.project, org_unit=self.ou_top_1)
+
+        response = self.client.get(f"/api/instances/{instance.pk}/download_attachments/")
+        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)
+
+    def test_download_attachments(self):
+        self.client.force_authenticate(self.yoda)
+        instance = self.create_form_instance(form=self.form_1, project=self.project, org_unit=self.ou_top_1)
+        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file:
+            m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test1.jpg")
+            m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test2.webp")
+            m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test3.webp")
+            m.InstanceFile.objects.create(instance=instance, file=UploadedFile(xls_file), name="test4.pdf")
+
+        response = self.client.get(f"/api/instances/{instance.pk}/download_attachments/")
+        self.assertEqual(response.status_code, status.HTTP_200_OK)
+
+        async def streaming_content_to_bytes(content: async_generator) -> io.BytesIO:
+            bytes = io.BytesIO()
+            async for x in content:
+                bytes.write(x)
+            return bytes
+
+        content = asyncio.run(streaming_content_to_bytes(response.streaming_content))
+        with zipfile.ZipFile(content, "a", zipfile.ZIP_DEFLATED, False) as zf:
+            self.assertEqual(4, len(zf.namelist()))
+            self.assertEqual("test1.jpg", zf.namelist()[0])
+            self.assertEqual("test2.webp", zf.namelist()[1])
+            self.assertEqual("test3.webp", zf.namelist()[2])
+            self.assertEqual("test4.pdf", zf.namelist()[3])
+
     def assertInstanceListContainsStrictly(self, api_response, expected_instances):
         try:
             self.assertEqual(api_response.status_code, status.HTTP_200_OK)

pyproject.toml

@@ -87,9 +87,11 @@ dependencies = [
     "simpleeval==1.0.3", # https://github.com/danthedeckie/simpleeval/tags
     "snaql==0.4.2", # trypelim - https://github.com/zaiev/snaql/tags
     "sqlalchemy==2.0.23", # https://github.com/sqlalchemy/sqlalchemy/tags
+    "stream-zip==0.0.84", # https://stream-zip.docs.trade.gov.uk/
     "tsp-solver2==0.4.1", # trypelim - https://github.com/dmishin/tsp-solver/tags
     "unidecode==1.3.7", # https://github.com/avian2/unidecode/tags
     "urllib3<2",
+    "wrapt>=2.2.1",
     "xlsxwriter==3.1.9", # https://github.com/jmcnamara/XlsxWriter/tags
 ]
 # ------------------------------------------------------------------------------