Alexander/FitTrackee
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

API - add export task when data export is requested

Browse Source
This commit is contained in:
Sam 2023-03-01 16:30:44 +01:00
parent e4fc8849be
commit 90c85f921c
6 changed files with 220 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
fittrackee/config.py
View File

@ -69,6 +69,7 @@ class BaseConfig:
'authorization_code': 864000, # 10 days
}
OAUTH2_REFRESH_TOKEN_GENERATOR = True
DATA_EXPORT_EXPIRATION = 24 # hours
class DevelopmentConfig(BaseConfig):

94
fittrackee/tests/users/test_auth_api.py
View File

@ -2829,9 +2829,11 @@ class TestUserPrivacyPolicyUpdate(ApiTestCaseMixin):
assert response.status_code == 400
@patch('fittrackee.users.auth.export_data')
class TestPostUserDataExportRequest(ApiTestCaseMixin):
def test_it_returns_data_export_info_when_no_ongoing_request_exists_for_user( # noqa
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
user_2: User,
@ -2858,6 +2860,7 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
def test_it_returns_error_if_ongoing_request_exist(
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
) -> None:
@ -2875,8 +2878,9 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
self.assert_400(response, "ongoing request exists")
def test_it_returns_error_if_existing_request_is_completed_for_less_than_24_hours( # noqa
def test_it_returns_error_if_existing_request_has_not_expired(
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
) -> None:
@ -2896,14 +2900,16 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
self.assert_400(response, "completed request already exists")
def test_it_returns_new_request_if_existing_request_is_completed_for_more_than_more_24_hours( # noqa
def test_it_returns_new_request_if_existing_request_has_expired( # noqa
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
) -> None:
export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
completed_export_request = UserDataExport(
user_id=user_1.id,
created_at=datetime.utcnow() - timedelta(hours=24),
created_at=datetime.utcnow() - timedelta(hours=export_expiration),
)
db.session.add(completed_export_request)
completed_export_request.completed = True
@ -2927,6 +2933,85 @@ class TestPostUserDataExportRequest(ApiTestCaseMixin):
assert data["status"] == "success"
assert data["request"] == jsonify_dict(data_export_request.serialize())
def test_it_calls_export_data_tasks_when_request_is_created(
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
) -> None:
client, auth_token = self.get_test_client_and_auth_token(
app, user_1.email
)
client.post(
'/api/auth/profile/export/request',
content_type='application/json',
headers=dict(Authorization=f'Bearer {auth_token}'),
)
data_export_request = UserDataExport.query.filter_by(
user_id=user_1.id
).first()
export_data_mock.send.assert_called_once_with(
export_request_id=data_export_request.id
)
def test_it_does_not_calls_export_data_tasks_when_request_already_exists(
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
) -> None:
export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
completed_export_request = UserDataExport(
user_id=user_1.id,
created_at=datetime.utcnow() - timedelta(hours=export_expiration),
)
db.session.add(completed_export_request)
db.session.commit()
client, auth_token = self.get_test_client_and_auth_token(
app, user_1.email
)
client.post(
'/api/auth/profile/export/request',
content_type='application/json',
headers=dict(Authorization=f'Bearer {auth_token}'),
)
export_data_mock.send.assert_not_called()
def test_it_returns_new_request_if_previous_request_has_expired(
self,
export_data_mock: Mock,
app: Flask,
user_1: User,
) -> None:
export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
completed_export_request = UserDataExport(
user_id=user_1.id,
created_at=datetime.utcnow() - timedelta(hours=export_expiration),
)
db.session.add(completed_export_request)
completed_export_request.completed = True
db.session.commit()
client, auth_token = self.get_test_client_and_auth_token(
app, user_1.email
)
client.post(
'/api/auth/profile/export/request',
content_type='application/json',
headers=dict(Authorization=f'Bearer {auth_token}'),
)
data_export_request = UserDataExport.query.filter_by(
user_id=user_1.id
).first()
export_data_mock.send.assert_called_once_with(
export_request_id=data_export_request.id
)
class TestGetUserDataExportRequest(ApiTestCaseMixin):
def test_it_returns_none_if_no_request(
@ -2956,9 +3041,10 @@ class TestGetUserDataExportRequest(ApiTestCaseMixin):
user_1: User,
user_2: User,
) -> None:
export_expiration = app.config["DATA_EXPORT_EXPIRATION"]
completed_export_request = UserDataExport(
user_id=user_1.id,
created_at=datetime.utcnow() - timedelta(hours=24),
created_at=datetime.utcnow() - timedelta(hours=export_expiration),
)
db.session.add(completed_export_request)
db.session.commit()

87
fittrackee/tests/users/test_export_data.py → fittrackee/tests/users/test_users_export_data.py
View File

@ -4,12 +4,13 @@ from unittest.mock import Mock, call, patch
from flask import Flask
from fittrackee.users.export_data import UserDataExporter
from fittrackee.users.models import User
from fittrackee import db
from fittrackee.users.export_data import UserDataExporter, export_user_data
from fittrackee.users.models import User, UserDataExport
from fittrackee.workouts.models import Sport, Workout
from ..mixins import CallArgsMixin
from ..utils import random_string
from ..utils import random_int, random_string
from ..workouts.utils import post_a_workout
@ -333,3 +334,83 @@ class TestUserDataExporterArchive(CallArgsMixin):
exporter.generate_archive()
assert os.path.isfile(expected_path)
@patch('fittrackee.users.export_data.appLog')
@patch.object(UserDataExporter, 'generate_archive')
class TestExportUserData:
def test_it_logs_error_if_not_request_for_given_id(
self,
generate_archive: Mock,
logger_mock: Mock,
app: Flask,
) -> None:
request_id = random_int()
export_user_data(export_request_id=request_id)
logger_mock.error.assert_called_once_with(
f"No export to process for id '{request_id}'"
)
def test_it_logs_error_if_request_already_processed(
self,
generate_archive: Mock,
logger_mock: Mock,
app: Flask,
user_1: User,
) -> None:
export_request = UserDataExport(user_id=user_1.id)
db.session.add(export_request)
export_request.completed = True
db.session.commit()
export_user_data(export_request_id=export_request.id)
logger_mock.info.assert_called_once_with(
f"Export id '{export_request.id}' already processed"
)
def test_it_update_export_request_when_export_is_successful(
self,
generate_archive_mock: Mock,
logger_mock: Mock,
app: Flask,
user_1: User,
) -> None:
export_request = UserDataExport(user_id=user_1.id)
db.session.add(export_request)
db.session.commit()
archive_name = random_string()
generate_archive_mock.return_value = (random_string(), archive_name)
archive_size = random_int()
with patch(
'fittrackee.users.export_data.os.path.getsize',
return_value=archive_size,
):
export_user_data(export_request_id=export_request.id)
assert export_request.completed is True
assert export_request.updated_at is not None
assert export_request.file_name == archive_name
assert export_request.file_size == archive_size
def test_it_update_export_request_when_export_fails(
self,
generate_archive_mock: Mock,
logger_mock: Mock,
app: Flask,
user_1: User,
) -> None:
export_request = UserDataExport(user_id=user_1.id)
db.session.add(export_request)
db.session.commit()
generate_archive_mock.return_value = (None, None)
export_user_data(export_request_id=export_request.id)
assert export_request.completed is True
assert export_request.updated_at is not None
assert export_request.file_name is None
assert export_request.file_size is None

11
fittrackee/users/auth.py
View File

@ -34,6 +34,7 @@ from fittrackee.utils import get_readable_duration
from fittrackee.workouts.models import Sport
from .models import BlacklistedToken, User, UserDataExport, UserSportPreference
from .tasks import export_data
from .utils.controls import check_password, is_valid_email, register_controls
from .utils.token import decode_user_token
@ -1735,9 +1736,10 @@ def request_user_data_export(auth_user: User) -> Union[Dict, HttpResponse]:
if not existing_export_request.completed:
return InvalidPayloadErrorResponse("ongoing request exists")
if (
existing_export_request.created_at
> datetime.datetime.utcnow() - datetime.timedelta(hours=24)
export_expiration = current_app.config["DATA_EXPORT_EXPIRATION"]
if existing_export_request.created_at > (
datetime.datetime.utcnow()
- datetime.timedelta(hours=export_expiration)
):
return InvalidPayloadErrorResponse(
"completed request already exists"
@ -1750,6 +1752,9 @@ def request_user_data_export(auth_user: User) -> Union[Dict, HttpResponse]:
export_request = UserDataExport(user_id=auth_user.id)
db.session.add(export_request)
db.session.commit()
export_data.send(export_request_id=export_request.id)
return {"status": "success", "request": export_request.serialize()}
except (exc.IntegrityError, exc.OperationalError, ValueError) as e:
return handle_error_and_return_response(e, db=db)

35
fittrackee/users/export_data.py
View File

@ -1,12 +1,13 @@
import json
import os
import secrets
from typing import Dict, List, Optional, Union
from typing import Dict, List, Optional, Tuple, Union
from zipfile import ZipFile
from fittrackee import appLog, db
from fittrackee.files import get_absolute_file_path
from .models import User
from .models import User, UserDataExport
class UserDataExporter:
@ -50,7 +51,7 @@ class UserDataExporter:
export_file.write(json_object)
return file_path
def generate_archive(self) -> Optional[str]:
def generate_archive(self) -> Tuple[Optional[str], Optional[str]]:
try:
user_data_file_name = self.export_data(
self.get_user_info(), "user_data"
@ -81,6 +82,30 @@ class UserDataExporter:
)
file_exists = os.path.exists(zip_path)
return zip_file if file_exists else None
return (zip_path, zip_file) if file_exists else (None, None)
except Exception:
return None
return None, None
def export_user_data(export_request_id: int) -> None:
export_request = UserDataExport.query.filter_by(
id=export_request_id
).first()
if not export_request:
appLog.error(f"No export to process for id '{export_request_id}'")
return
if export_request.completed:
appLog.info(f"Export id '{export_request_id}' already processed")
return
user = User.query.filter_by(id=export_request.user_id).first()
exporter = UserDataExporter(user)
archive_file_path, archive_file_name = exporter.generate_archive()
export_request.completed = True
if archive_file_name and archive_file_path:
export_request.file_name = archive_file_name
export_request.file_size = os.path.getsize(archive_file_path)
db.session.commit()

7
fittrackee/users/tasks.py Normal file
View File

@ -0,0 +1,7 @@
from fittrackee import dramatiq
from fittrackee.users.export_data import export_user_data
@dramatiq.actor(queue_name='fittrackee_users_exports')
def export_data(export_request_id: int) -> None:
export_user_data(export_request_id)