API - add commands to export and delete archives

fittrackee/tests/users/test_users_export_data.py

@@ -1,11 +1,18 @@
 import os
 import secrets
+from datetime import datetime, timedelta
+from typing import Optional, Tuple
 from unittest.mock import Mock, call, patch
 
 from flask import Flask
 
 from fittrackee import db
-from fittrackee.users.export_data import UserDataExporter, export_user_data
+from fittrackee.users.export_data import (
+    UserDataExporter,
+    clean_user_data_export,
+    export_user_data,
+    generate_user_data_archives,
+)
 from fittrackee.users.models import User, UserDataExport
 from fittrackee.workouts.models import Sport, Workout
 
@@ -465,3 +472,165 @@ class TestExportUserData:
                 'fittrackee_url': 'http://0.0.0.0:5000',
             },
         )
+
+
+class UserDataExportTestCase:
+    @staticmethod
+    def create_user_request(
+        user: User, days: int = 0, completed: bool = True
+    ) -> UserDataExport:
+        user_data_export = UserDataExport(
+            user_id=user.id,
+            created_at=datetime.now() - timedelta(days=days),
+        )
+        db.session.add(user_data_export)
+        user_data_export.completed = completed
+        db.session.commit()
+        return user_data_export
+
+    def generate_archive(
+        self, user: User
+    ) -> Tuple[UserDataExport, Optional[str]]:
+        user_data_export = self.create_user_request(user, days=7)
+        exporter = UserDataExporter(user)
+        archive_path, archive_file_name = exporter.generate_archive()
+        user_data_export.file_name = archive_file_name
+        user_data_export.file_size = random_int()
+        db.session.commit()
+        return user_data_export, archive_path
+
+
+class TestCleanUserDataExport(UserDataExportTestCase):
+    def test_it_returns_0_when_no_export_requests(self, app: Flask) -> None:
+        counts = clean_user_data_export(days=7)
+
+        assert counts["deleted_requests"] == 0
+
+    def test_it_returns_0_when_export_request_is_not_completed(
+        self, app: Flask, user_1: User
+    ) -> None:
+        self.create_user_request(user_1, days=7, completed=False)
+
+        counts = clean_user_data_export(days=7)
+
+        assert counts["deleted_requests"] == 0
+
+    def test_it_returns_0_when_export_request_created_less_than_given_days(
+        self, app: Flask, user_1: User
+    ) -> None:
+        self.create_user_request(user_1, days=1)
+
+        counts = clean_user_data_export(days=7)
+
+        assert counts["deleted_requests"] == 0
+
+    def test_it_returns_export_requests_created_more_than_given_days_count(
+        self, app: Flask, user_1: User, user_2: User
+    ) -> None:
+        self.create_user_request(user_1, days=7)
+        self.create_user_request(user_2, days=7)
+
+        counts = clean_user_data_export(days=7)
+
+        assert counts["deleted_requests"] == 2
+
+    def test_it_returns_counts(
+        self, app: Flask, user_1: User, user_2: User, user_3: User
+    ) -> None:
+        user_1_data_export, archive_path = self.generate_archive(user_1)
+        user_2_data_export, archive_path = self.generate_archive(user_2)
+        self.create_user_request(user_3, days=7)
+
+        counts = clean_user_data_export(days=7)
+
+        assert counts["deleted_requests"] == 3
+        assert counts["deleted_archives"] == 2
+        assert counts["freed_space"] == (
+            user_1_data_export.file_size + user_2_data_export.file_size
+        )
+
+    def test_it_deletes_archive(
+        self, app: Flask, user_1: User, user_2: User
+    ) -> None:
+        _, archive_path = self.generate_archive(user_1)
+
+        clean_user_data_export(days=7)
+
+        assert os.path.exists(archive_path) is False  # type: ignore
+
+    def test_it_deletes_requests(
+        self, app: Flask, user_1: User, user_2: User
+    ) -> None:
+        self.generate_archive(user_1)
+
+        clean_user_data_export(days=7)
+
+        assert (
+            UserDataExport.query.filter_by(user_id=user_1.id).first() is None
+        )
+
+
+class TestGenerateUsersArchives(UserDataExportTestCase):
+    def test_it_returns_0_when_no_request(self, app: Flask) -> None:
+        count = generate_user_data_archives(max_count=1)
+
+        assert count == 0
+
+    def test_it_returns_0_when_request_request_completed(
+        self, app: Flask, user_1: User
+    ) -> None:
+        self.create_user_request(user_1, completed=True)
+
+        count = generate_user_data_archives(max_count=1)
+
+        assert count == 0
+
+    def test_it_returns_count_when_archive_is_generated_user_archive(
+        self, app: Flask, user_1: User
+    ) -> None:
+        self.create_user_request(user_1, completed=False)
+
+        count = generate_user_data_archives(max_count=1)
+
+        assert count == 1
+
+    @patch.object(secrets, 'token_urlsafe')
+    def test_it_generates_user_archive(
+        self, secrets_mock: Mock, app: Flask, user_1: User
+    ) -> None:
+        token_urlsafe = random_string()
+        secrets_mock.return_value = token_urlsafe
+        archive_path = os.path.join(
+            app.config['UPLOAD_FOLDER'],
+            'exports',
+            str(user_1.id),
+            f"archive_{token_urlsafe}.zip",
+        )
+        self.create_user_request(user_1, completed=False)
+
+        generate_user_data_archives(max_count=1)
+
+        assert os.path.exists(archive_path) is True  # type: ignore
+
+    def test_it_generates_max_count_of_archives(
+        self, app: Flask, user_1: User, user_2: User, user_3: User
+    ) -> None:
+        self.create_user_request(user_3, completed=False)
+        self.create_user_request(user_1, completed=False)
+        self.create_user_request(user_2, completed=False)
+
+        count = generate_user_data_archives(max_count=2)
+
+        assert count == 2
+        assert (
+            UserDataExport.query.filter_by(user_id=user_1.id).first().completed
+            is True
+        )
+        assert (
+            UserDataExport.query.filter_by(user_id=user_2.id).first().completed
+            is False
+        )
+        assert (
+            UserDataExport.query.filter_by(user_id=user_3.id).first().completed
+            is True
+        )

fittrackee/users/commands.py

@@ -2,9 +2,14 @@ import logging
 from typing import Optional
 
 import click
+from humanize import naturalsize
 
 from fittrackee.cli.app import app
 from fittrackee.users.exceptions import UserNotFoundException
+from fittrackee.users.export_data import (
+    clean_user_data_export,
+    generate_user_data_archives,
+)
 from fittrackee.users.utils.admin import UserManagerService
 from fittrackee.users.utils.token import clean_blacklisted_tokens
 
@@ -80,3 +85,39 @@ def clean(
     with app.app_context():
         deleted_rows = clean_blacklisted_tokens(days)
         logger.info(f'Blacklisted tokens deleted: {deleted_rows}.')
+
+
+@users_cli.command('clean_archives')
+@click.option('--days', type=int, required=True, help='Number of days.')
+def clean_export_archives(
+    days: int,
+) -> None:
+    """
+    Clean user export archives created for more than provided number of days.
+    """
+    with app.app_context():
+        counts = clean_user_data_export(days)
+        logger.info(
+            f'Deleted data export requests: {counts["deleted_requests"]}.'
+        )
+        logger.info(f'Deleted archives: {counts["deleted_archives"]}.')
+        logger.info(f'Freed space: {naturalsize(counts["freed_space"])}.')
+
+
+@users_cli.command('export_archives')
+@click.option(
+    '--max',
+    type=int,
+    required=True,
+    help='Maximum number of archives to generate.',
+)
+def export_archives(
+    max: int,
+) -> None:
+    """
+    Export user data in zip archive if incomplete requests exist.
+    To use in case redis is not set.
+    """
+    with app.app_context():
+        count = generate_user_data_archives(max)
+        logger.info(f'Generated archives: {count}.')

fittrackee/users/export_data.py

@@ -1,6 +1,7 @@
 import json
 import os
 import secrets
+from datetime import datetime, timedelta
 from typing import Dict, List, Optional, Tuple, Union
 from zipfile import ZipFile
 
@@ -135,3 +136,48 @@ def export_user_data(export_request_id: int) -> None:
             db.session.commit()
     except Exception as e:
         appLog.error(f'Error when exporting user data: {str(e)}')
+
+
+def clean_user_data_export(days: int) -> Dict:
+    counts = {"deleted_requests": 0, "deleted_archives": 0, "freed_space": 0}
+    limit = datetime.now() - timedelta(days=days)
+    export_requests = UserDataExport.query.filter(
+        UserDataExport.created_at < limit,
+        UserDataExport.completed == True,  # noqa
+    ).all()
+
+    if not export_requests:
+        return counts
+
+    archive_directory = get_absolute_file_path("exports")
+    for request in export_requests:
+        if request.file_name:
+            archive_path = os.path.join(
+                archive_directory, f"{request.user_id}", request.file_name
+            )
+            if os.path.exists(archive_path):
+                counts["deleted_archives"] += 1
+                counts["freed_space"] += request.file_size
+        # Archive is deleted when row is deleted
+        db.session.delete(request)
+        counts["deleted_requests"] += 1
+
+    db.session.commit()
+    return counts
+
+
+def generate_user_data_archives(max_count: int) -> int:
+    count = 0
+    export_requests = (
+        db.session.query(UserDataExport)
+        .filter(UserDataExport.completed == False)  # noqa
+        .order_by(UserDataExport.created_at)
+        .limit(max_count)
+        .all()
+    )
+
+    for export_request in export_requests:
+        export_user_data(export_request.id)
+        count += 1
+
+    return count