API - add export task when data export is requested

fittrackee/config.py

@@ -69,6 +69,7 @@ class BaseConfig:
         'authorization_code': 864000,  # 10 days
     }
     OAUTH2_REFRESH_TOKEN_GENERATOR = True
+    DATA_EXPORT_EXPIRATION = 24  # hours
 
 
 class DevelopmentConfig(BaseConfig):

fittrackee/tests/users/test_auth_api.py

@@ -2829,9 +2829,11 @@ class TestUserPrivacyPolicyUpdate(ApiTestCaseMixin):
         assert response.status_code == 400
 
 
+@patch('fittrackee.users.auth.export_data')
 class TestPostUserDataExportRequest(ApiTestCaseMixin):
     def test_it_returns_data_export_info_when_no_ongoing_request_exists_for_user(  # noqa
         self,
+        export_data_mock: Mock,
         app: Flask,
         user_1: User,
         user_2: User,
@@ -2858,6 +2860,7 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
 
     def test_it_returns_error_if_ongoing_request_exist(
         self,
+        export_data_mock: Mock,
         app: Flask,
         user_1: User,
     ) -> None:
@@ -2875,8 +2878,9 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
 
         self.assert_400(response, "ongoing request exists")
 
-    def test_it_returns_error_if_existing_request_is_completed_for_less_than_24_hours(  # noqa
+    def test_it_returns_error_if_existing_request_has_not_expired(
         self,
+        export_data_mock: Mock,
         app: Flask,
         user_1: User,
     ) -> None:
@@ -2896,14 +2900,16 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
 
         self.assert_400(response, "completed request already exists")
 
-    def test_it_returns_new_request_if_existing_request_is_completed_for_more_than_more_24_hours(  # noqa
+    def test_it_returns_new_request_if_existing_request_has_expired(  # noqa
         self,
+        export_data_mock: Mock,
         app: Flask,
         user_1: User,
     ) -> None:
+        export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
         completed_export_request = UserDataExport(
             user_id=user_1.id,
-            created_at=datetime.utcnow() - timedelta(hours=24),
+            created_at=datetime.utcnow() - timedelta(hours=export_expiration),
         )
         db.session.add(completed_export_request)
         completed_export_request.completed = True
@@ -2927,6 +2933,85 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
         assert data["status"] == "success"
         assert data["request"] == jsonify_dict(data_export_request.serialize())
 
+    def test_it_calls_export_data_tasks_when_request_is_created(
+        self,
+        export_data_mock: Mock,
+        app: Flask,
+        user_1: User,
+    ) -> None:
+        client, auth_token = self.get_test_client_and_auth_token(
+            app, user_1.email
+        )
+
+        client.post(
+            '/api/auth/profile/export/request',
+            content_type='application/json',
+            headers=dict(Authorization=f'Bearer {auth_token}'),
+        )
+
+        data_export_request = UserDataExport.query.filter_by(
+            user_id=user_1.id
+        ).first()
+        export_data_mock.send.assert_called_once_with(
+            export_request_id=data_export_request.id
+        )
+
+    def test_it_does_not_calls_export_data_tasks_when_request_already_exists(
+        self,
+        export_data_mock: Mock,
+        app: Flask,
+        user_1: User,
+    ) -> None:
+        export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
+        completed_export_request = UserDataExport(
+            user_id=user_1.id,
+            created_at=datetime.utcnow() - timedelta(hours=export_expiration),
+        )
+        db.session.add(completed_export_request)
+        db.session.commit()
+        client, auth_token = self.get_test_client_and_auth_token(
+            app, user_1.email
+        )
+
+        client.post(
+            '/api/auth/profile/export/request',
+            content_type='application/json',
+            headers=dict(Authorization=f'Bearer {auth_token}'),
+        )
+
+        export_data_mock.send.assert_not_called()
+
+    def test_it_returns_new_request_if_previous_request_has_expired(
+        self,
+        export_data_mock: Mock,
+        app: Flask,
+        user_1: User,
+    ) -> None:
+        export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
+        completed_export_request = UserDataExport(
+            user_id=user_1.id,
+            created_at=datetime.utcnow() - timedelta(hours=export_expiration),
+        )
+        db.session.add(completed_export_request)
+        completed_export_request.completed = True
+        db.session.commit()
+        client, auth_token = self.get_test_client_and_auth_token(
+            app, user_1.email
+        )
+
+        client.post(
+            '/api/auth/profile/export/request',
+            content_type='application/json',
+            headers=dict(Authorization=f'Bearer {auth_token}'),
+        )
+
+        data_export_request = UserDataExport.query.filter_by(
+            user_id=user_1.id
+        ).first()
+        export_data_mock.send.assert_called_once_with(
+            export_request_id=data_export_request.id
+        )
+
 
 class TestGetUserDataExportRequest(ApiTestCaseMixin):
     def test_it_returns_none_if_no_request(
@@ -2956,9 +3041,10 @@ class TestGetUserDataExportRequest(ApiTestCaseMixin):
         user_1: User,
         user_2: User,
     ) -> None:
+        export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
         completed_export_request = UserDataExport(
             user_id=user_1.id,
-            created_at=datetime.utcnow() - timedelta(hours=24),
+            created_at=datetime.utcnow() - timedelta(hours=export_expiration),
         )
         db.session.add(completed_export_request)
         db.session.commit()

fittrackee/tests/users/test_users_export_data.py

@@ -4,12 +4,13 @@ from unittest.mock import Mock, call, patch
 
 from flask import Flask
 
-from fittrackee.users.export_data import UserDataExporter
+from fittrackee import db
-from fittrackee.users.models import User
+from fittrackee.users.export_data import UserDataExporter, export_user_data
+from fittrackee.users.models import User, UserDataExport
 from fittrackee.workouts.models import Sport, Workout
 
 from ..mixins import CallArgsMixin
-from ..utils import random_string
+from ..utils import random_int, random_string
 from ..workouts.utils import post_a_workout
 
 
@@ -333,3 +334,83 @@ class TestUserDataExporterArchive(CallArgsMixin):
         exporter.generate_archive()
 
         assert os.path.isfile(expected_path)
+
+
+@patch('fittrackee.users.export_data.appLog')
+@patch.object(UserDataExporter, 'generate_archive')
+class TestExportUserData:
+    def test_it_logs_error_if_not_request_for_given_id(
+        self,
+        generate_archive: Mock,
+        logger_mock: Mock,
+        app: Flask,
+    ) -> None:
+        request_id = random_int()
+
+        export_user_data(export_request_id=request_id)
+
+        logger_mock.error.assert_called_once_with(
+            f"No export to process for id '{request_id}'"
+        )
+
+    def test_it_logs_error_if_request_already_processed(
+        self,
+        generate_archive: Mock,
+        logger_mock: Mock,
+        app: Flask,
+        user_1: User,
+    ) -> None:
+        export_request = UserDataExport(user_id=user_1.id)
+        db.session.add(export_request)
+        export_request.completed = True
+        db.session.commit()
+
+        export_user_data(export_request_id=export_request.id)
+
+        logger_mock.info.assert_called_once_with(
+            f"Export id '{export_request.id}' already processed"
+        )
+
+    def test_it_update_export_request_when_export_is_successful(
+        self,
+        generate_archive_mock: Mock,
+        logger_mock: Mock,
+        app: Flask,
+        user_1: User,
+    ) -> None:
+        export_request = UserDataExport(user_id=user_1.id)
+        db.session.add(export_request)
+        db.session.commit()
+        archive_name = random_string()
+        generate_archive_mock.return_value = (random_string(), archive_name)
+        archive_size = random_int()
+
+        with patch(
+            'fittrackee.users.export_data.os.path.getsize',
+            return_value=archive_size,
+        ):
+            export_user_data(export_request_id=export_request.id)
+
+        assert export_request.completed is True
+        assert export_request.updated_at is not None
+        assert export_request.file_name == archive_name
+        assert export_request.file_size == archive_size
+
+    def test_it_update_export_request_when_export_fails(
+        self,
+        generate_archive_mock: Mock,
+        logger_mock: Mock,
+        app: Flask,
+        user_1: User,
+    ) -> None:
+        export_request = UserDataExport(user_id=user_1.id)
+        db.session.add(export_request)
+        db.session.commit()
+        generate_archive_mock.return_value = (None, None)
+
+        export_user_data(export_request_id=export_request.id)
+
+        assert export_request.completed is True
+        assert export_request.updated_at is not None
+        assert export_request.file_name is None
+        assert export_request.file_size is None

fittrackee/users/auth.py

@@ -34,6 +34,7 @@ from fittrackee.utils import get_readable_duration
 from fittrackee.workouts.models import Sport
 
 from .models import BlacklistedToken, User, UserDataExport, UserSportPreference
+from .tasks import export_data
 from .utils.controls import check_password, is_valid_email, register_controls
 from .utils.token import decode_user_token
 
@@ -1735,9 +1736,10 @@ def request_user_data_export(auth_user: User) -> Union[Dict, HttpResponse]:
         if not existing_export_request.completed:
             return InvalidPayloadErrorResponse("ongoing request exists")
 
-        if (
+        export_expiration = current_app.config["DATA_EXPORT_EXPIRATION"]
-            existing_export_request.created_at
+        if existing_export_request.created_at > (
-            > datetime.datetime.utcnow() - datetime.timedelta(hours=24)
+            datetime.datetime.utcnow()
+            - datetime.timedelta(hours=export_expiration)
         ):
             return InvalidPayloadErrorResponse(
                 "completed request already exists"
@@ -1750,6 +1752,9 @@ def request_user_data_export(auth_user: User) -> Union[Dict, HttpResponse]:
         export_request = UserDataExport(user_id=auth_user.id)
         db.session.add(export_request)
         db.session.commit()
+
+        export_data.send(export_request_id=export_request.id)
+
         return {"status": "success", "request": export_request.serialize()}
     except (exc.IntegrityError, exc.OperationalError, ValueError) as e:
         return handle_error_and_return_response(e, db=db)

fittrackee/users/export_data.py

@@ -1,12 +1,13 @@
 import json
 import os
 import secrets
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 from zipfile import ZipFile
 
+from fittrackee import appLog, db
 from fittrackee.files import get_absolute_file_path
 
-from .models import User
+from .models import User, UserDataExport
 
 
 class UserDataExporter:
@@ -50,7 +51,7 @@ class UserDataExporter:
             export_file.write(json_object)
         return file_path
 
-    def generate_archive(self) -> Optional[str]:
+    def generate_archive(self) -> Tuple[Optional[str], Optional[str]]:
         try:
             user_data_file_name = self.export_data(
                 self.get_user_info(), "user_data"
@@ -81,6 +82,30 @@ class UserDataExporter:
                         )
 
             file_exists = os.path.exists(zip_path)
-            return zip_file if file_exists else None
+            return (zip_path, zip_file) if file_exists else (None, None)
         except Exception:
-            return None
+            return None, None
+
+
+def export_user_data(export_request_id: int) -> None:
+    export_request = UserDataExport.query.filter_by(
+        id=export_request_id
+    ).first()
+
+    if not export_request:
+        appLog.error(f"No export to process for id '{export_request_id}'")
+        return
+
+    if export_request.completed:
+        appLog.info(f"Export id '{export_request_id}' already processed")
+        return
+
+    user = User.query.filter_by(id=export_request.user_id).first()
+    exporter = UserDataExporter(user)
+    archive_file_path, archive_file_name = exporter.generate_archive()
+
+    export_request.completed = True
+    if archive_file_name and archive_file_path:
+        export_request.file_name = archive_file_name
+        export_request.file_size = os.path.getsize(archive_file_path)
+    db.session.commit()

fittrackee/users/tasks.py

@@ -0,0 +1,7 @@
+from fittrackee import dramatiq
+from fittrackee.users.export_data import export_user_data
+
+
+@dramatiq.actor(queue_name='fittrackee_users_exports')
+def export_data(export_request_id: int) -> None:
+    export_user_data(export_request_id)