diff --git a/.env.docker b/.env.docker index e5c1c70c..c25c7400 100644 --- a/.env.docker +++ b/.env.docker @@ -10,18 +10,23 @@ export APP_SECRET_KEY='just for test' export APP_LOG=fittrackee.log export UPLOAD_FOLDER=/usr/src/app/uploads -# Database +# PostgreSQL export DATABASE_URL=postgresql://fittrackee:fittrackee@fittrackee-db:5432/fittrackee export DATABASE_TEST_URL=postgresql://fittrackee:fittrackee@fittrackee-db:5432/fittrackee_test # export DATABASE_DISABLE_POOLING= +# Redis (required for API rate limits and email sending) +export REDIS_URL=redis://redis:6379 + +# API rate limits +# export API_RATE_LIMITS=300 per 5 minutes + # Emails export UI_URL=http://0.0.0.0:5000 # For development: # export UI_URL=http://0.0.0.0:3000 export EMAIL_URL=smtp://mail:1025 export SENDER_EMAIL=fittrackee@example.com -export REDIS_URL=redis://redis:6379 export WORKERS_PROCESSES=2 # Workouts diff --git a/.env.example b/.env.example index 3b9acf6d..f6213e69 100644 --- a/.env.example +++ b/.env.example @@ -12,15 +12,20 @@ export APP_SECRET_KEY='please change me' export APP_LOG=fittrackee.log export UPLOAD_FOLDER= -# Database +# PostgreSQL # export DATABASE_URL=postgresql://fittrackee:fittrackee@${HOST}:5432/fittrackee # export DATABASE_DISABLE_POOLING= +# Redis (required for API rate limits and email sending) +# export REDIS_URL= + +# API rate limits +# export API_RATE_LIMITS=300 per 5 minutes + # Emails export UI_URL= export EMAIL_URL= export SENDER_EMAIL= -# export REDIS_URL= # export WORKERS_PROCESSES= # Workouts diff --git a/fittrackee/__init__.py b/fittrackee/__init__.py index 0d9bdfe5..f8cb28a1 100644 --- a/fittrackee/__init__.py +++ b/fittrackee/__init__.py @@ -2,8 +2,9 @@ import logging import os import re from importlib import import_module, reload -from typing import Any +from typing import Any, Dict, Tuple +import redis from flask import ( Flask, Response, @@ -13,6 +14,9 @@ from flask import ( ) from flask_bcrypt import Bcrypt from flask_dramatiq import Dramatiq +from flask_limiter import Limiter +from flask_limiter.errors import RateLimitExceeded +from flask_limiter.util import get_remote_address from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy from sqlalchemy.exc import ProgrammingError @@ -22,11 +26,10 @@ from fittrackee.emails.email import EmailService from fittrackee.request import CustomRequest VERSION = __version__ = '0.6.12' -db = SQLAlchemy() -bcrypt = Bcrypt() -migrate = Migrate() -email_service = EmailService() -dramatiq = Dramatiq() +REDIS_URL = os.getenv('REDIS_URL', 'redis://') +API_RATE_LIMITS = os.environ.get('API_RATE_LIMITS', '300 per 5 minutes').split( + ',' +) log_file = os.getenv('APP_LOG') logging.basicConfig( filename=log_file, @@ -35,6 +38,27 @@ logging.basicConfig( ) appLog = logging.getLogger('fittrackee') +db = SQLAlchemy() +bcrypt = Bcrypt() +migrate = Migrate() +email_service = EmailService() +dramatiq = Dramatiq() +limiter = Limiter( + key_func=get_remote_address, + default_limits=API_RATE_LIMITS, # type: ignore + default_limits_per_method=True, + headers_enabled=True, + storage_uri=REDIS_URL, + strategy='fixed-window', +) +# if redis is not available, disable the rate limiter +r = redis.from_url(REDIS_URL) +try: + r.ping() +except redis.exceptions.ConnectionError: + limiter.enabled = False + appLog.warning('Redis not available, API rate limits are disabled.') + class CustomFlask(Flask): # add custom Request to handle user-agent parsing @@ -64,6 +88,7 @@ def create_app(init_email: bool = True) -> Flask: bcrypt.init_app(app) migrate.init_app(app, db) dramatiq.init_app(app) + limiter.init_app(app) # set oauth2 from fittrackee.oauth2.config import config_oauth @@ -140,7 +165,15 @@ def create_app(init_email: bool = True) -> Flask: ) return response + @app.errorhandler(429) + def rate_limit_handler(error: RateLimitExceeded) -> Tuple[Dict, int]: + return { + 'status': 'error', + 'message': f'rate limit exceeded ({error.description})', + }, 429 + @app.route('/favicon.ico') + @limiter.exempt def favicon() -> Any: return send_file( os.path.join(app.root_path, 'dist/favicon.ico') # type: ignore @@ -148,6 +181,7 @@ def create_app(init_email: bool = True) -> Flask: @app.route('/', defaults={'path': ''}) @app.route('/') + @limiter.exempt def catch_all(path: str) -> Any: # workaround to serve images (not in static directory) if path.startswith('img/'): diff --git a/fittrackee/tests/fixtures/fixtures_app.py b/fittrackee/tests/fixtures/fixtures_app.py index de722864..af67a37e 100644 --- a/fittrackee/tests/fixtures/fixtures_app.py +++ b/fittrackee/tests/fixtures/fixtures_app.py @@ -5,7 +5,7 @@ from typing import Generator, Optional, Union import pytest from flask import current_app -from fittrackee import create_app, db +from fittrackee import create_app, db, limiter from fittrackee.application.models import AppConfig from fittrackee.application.utils import update_app_config_from_database @@ -45,6 +45,7 @@ def get_app( max_users: Optional[int] = None, ) -> Generator: app = create_app() + limiter.enabled = False with app.app_context(): try: db.create_all() diff --git a/fittrackee/users/users.py b/fittrackee/users/users.py index 7070388d..bfb6107c 100644 --- a/fittrackee/users/users.py +++ b/fittrackee/users/users.py @@ -5,7 +5,7 @@ from typing import Any, Dict, Tuple, Union from flask import Blueprint, current_app, request, send_file from sqlalchemy import exc -from fittrackee import db +from fittrackee import db, limiter from fittrackee.emails.tasks import ( email_updated_to_new_address, password_change_email, @@ -379,6 +379,7 @@ def get_single_user( @users_blueprint.route('/users//picture', methods=['GET']) +@limiter.exempt def get_picture(user_name: str) -> Any: """get user picture diff --git a/fittrackee/workouts/workouts.py b/fittrackee/workouts/workouts.py index 7250c747..13e49e6c 100644 --- a/fittrackee/workouts/workouts.py +++ b/fittrackee/workouts/workouts.py @@ -16,7 +16,7 @@ from sqlalchemy import exc from werkzeug.exceptions import NotFound, RequestEntityTooLarge from werkzeug.utils import secure_filename -from fittrackee import appLog, db +from fittrackee import appLog, db, limiter from fittrackee.oauth2.server import require_auth from fittrackee.responses import ( DataInvalidPayloadErrorResponse, @@ -784,6 +784,7 @@ def download_workout_gpx( @workouts_blueprint.route('/workouts/map/', methods=['GET']) +@limiter.exempt def get_map(map_id: int) -> Union[HttpResponse, Response]: """ Get map image for workouts with gpx. @@ -830,6 +831,7 @@ def get_map(map_id: int) -> Union[HttpResponse, Response]: @workouts_blueprint.route( '/workouts/map_tile////.png', methods=['GET'] ) +@limiter.exempt def get_map_tile(s: str, z: str, x: str, y: str) -> Tuple[Response, int]: """ Get map tile from tile server. diff --git a/poetry.lock b/poetry.lock index ea22c66f..ff99318e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -186,7 +186,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" -category = "dev" +category = "main" optional = false python-versions = "*" @@ -325,6 +325,28 @@ python-versions = ">=3.6,<4.0" [package.dependencies] dramatiq = ">=1.5,<2.0" +[[package]] +name = "flask-limiter" +version = "2.6.2" +description = "Rate limiting for flask applications" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +Flask = ">=2" +limits = [ + {version = ">=2.3"}, + {version = "*", extras = ["redis"], optional = true, markers = "extra == \"redis\""}, +] +rich = ">=12,<13" +typing-extensions = "*" + +[package.extras] +memcached = ["limits"] +mongodb = ["limits"] +redis = ["limits"] + [[package]] name = "flask-migrate" version = "3.1.0" @@ -534,6 +556,30 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "limits" +version = "2.7.0" +description = "Rate limiting utilities" +category = "main" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +deprecated = ">=1.2" +packaging = ">=21,<22" +redis = {version = ">3,<5.0.0", optional = true, markers = "extra == \"redis\""} +typing-extensions = "*" + +[package.extras] +all = ["redis (>3,<5.0.0)", "redis (>=4.2.0)", "pymemcache (>3,<4.0.0)", "pymongo (>3,<5)", "motor (>=2.5,<4)", "emcache (>=0.6.1)", "coredis (>=3.4.0,<5)"] +async-memcached = ["emcache (>=0.6.1)"] +async-mongodb = ["motor (>=2.5,<4)"] +async-redis = ["coredis (>=3.4.0,<5)"] +memcached = ["pymemcache (>3,<4.0.0)"] +mongodb = ["pymongo (>3,<5)"] +redis = ["redis (>3,<5.0.0)"] +rediscluster = ["redis (>=4.2.0)"] + [[package]] name = "mako" version = "1.2.2" @@ -726,7 +772,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" name = "pygments" version = "2.13.0" description = "Pygments is a syntax highlighting package written in Python." -category = "dev" +category = "main" optional = false python-versions = ">=3.6" @@ -1043,6 +1089,22 @@ urllib3 = ">=1.25.10" [package.extras] tests = ["pytest (>=7.0.0)", "coverage (>=6.0.0)", "pytest-cov", "pytest-asyncio", "pytest-localserver", "flake8", "types-mock", "types-requests", "mypy"] +[[package]] +name = "rich" +version = "12.5.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6.3,<4.0.0" + +[package.dependencies] +commonmark = ">=0.9.0,<0.10.0" +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + [[package]] name = "selenium" version = "4.4.3" @@ -1369,6 +1431,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-redis" +version = "4.3.20" +description = "Typing stubs for redis" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "types-requests" version = "2.28.10" @@ -1465,7 +1535,7 @@ testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest- [metadata] lock-version = "1.1" python-versions = "^3.7" -content-hash = "579fd9abe2d025aaa5dc84fbd587a04c03db2fc0d8bdf063edb29fa9a7093015" +content-hash = "468ee5a0ea6984ed5f6a2a63ffa1de67a0d774bd6405ccefc00536a5ab7d8f42" [metadata.files] alabaster = [ @@ -1721,6 +1791,10 @@ flask-dramatiq = [ {file = "flask-dramatiq-0.6.0.tar.gz", hash = "sha256:63709e73d7c8d2e5d9bc554d1e859d91c5c5c9a4ebc9461752655bf1e0b87420"}, {file = "flask_dramatiq-0.6.0-py3-none-any.whl", hash = "sha256:7d4a9289721577f726183f7c44c6713a16bbdff54b946f27abc2ffcc65768adf"}, ] +flask-limiter = [ + {file = "Flask-Limiter-2.6.2.tar.gz", hash = "sha256:58b361347f68942ea2d0a9004427098b41da705081494fe3b9be7b67c4ae32c4"}, + {file = "Flask_Limiter-2.6.2-py3-none-any.whl", hash = "sha256:c8451532f88818e839bbdd650cfd424ec11e89fa87e0034f525401399a160e1e"}, +] flask-migrate = [ {file = "Flask-Migrate-3.1.0.tar.gz", hash = "sha256:57d6060839e3a7f150eaab6fe4e726d9e3e7cffe2150fb223d73f92421c6d1d9"}, {file = "Flask_Migrate-3.1.0-py3-none-any.whl", hash = "sha256:a6498706241aba6be7a251078de9cf166d74307bca41a4ca3e403c9d39e2f897"}, @@ -1840,6 +1914,10 @@ jinja2 = [ {file = "Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, {file = "Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, ] +limits = [ + {file = "limits-2.7.0-py3-none-any.whl", hash = "sha256:cf043c376f14208a8204dc4cd74af1210fcaf24082bf7e3a38028706aa129899"}, + {file = "limits-2.7.0.tar.gz", hash = "sha256:847a78b24a47e822a8f8713028a1340db44003763ab1cb90e798ccf462880355"}, +] mako = [ {file = "Mako-1.2.2-py3-none-any.whl", hash = "sha256:8efcb8004681b5f71d09c983ad5a9e6f5c40601a6ec469148753292abc0da534"}, {file = "Mako-1.2.2.tar.gz", hash = "sha256:3724869b363ba630a272a5f89f68c070352137b8fd1757650017b7e06fda163f"}, @@ -2180,6 +2258,10 @@ responses = [ {file = "responses-0.21.0-py3-none-any.whl", hash = "sha256:2dcc863ba63963c0c3d9ee3fa9507cbe36b7d7b0fccb4f0bdfd9e96c539b1487"}, {file = "responses-0.21.0.tar.gz", hash = "sha256:b82502eb5f09a0289d8e209e7bad71ef3978334f56d09b444253d5ad67bf5253"}, ] +rich = [ + {file = "rich-12.5.1-py3-none-any.whl", hash = "sha256:2eb4e6894cde1e017976d2975ac210ef515d7548bc595ba20e195fb9628acdeb"}, + {file = "rich-12.5.1.tar.gz", hash = "sha256:63a5c5ce3673d3d5fbbf23cd87e11ab84b6b451436f1b7f19ec54b6bc36ed7ca"}, +] selenium = [ {file = "selenium-4.4.3-py3-none-any.whl", hash = "sha256:ca6ed4a58a426bb40bf5aa2b027ce211cc5200f1acdcdfb8258b32b24624150c"}, ] @@ -2344,6 +2426,10 @@ types-pytz = [ {file = "types-pytz-2022.2.1.0.tar.gz", hash = "sha256:47cfb19c52b9f75896440541db392fd312a35b279c6307a531db71152ea63e2b"}, {file = "types_pytz-2022.2.1.0-py3-none-any.whl", hash = "sha256:50ead2254b524a3d4153bc65d00289b66898060d2938e586170dce918dbaf3b3"}, ] +types-redis = [ + {file = "types-redis-4.3.20.tar.gz", hash = "sha256:74ed02945470ddea2dd21447c185dabb3169e5a5328d26b25cf3547d949b8e04"}, + {file = "types_redis-4.3.20-py3-none-any.whl", hash = "sha256:b22e0f5a18b98b6a197dd403daed52a22cb76f50e3cbd7ddc539196af52ec23e"}, +] types-requests = [ {file = "types-requests-2.28.10.tar.gz", hash = "sha256:97d8f40aa1ffe1e58c3726c77d63c182daea9a72d9f1fa2cafdea756b2a19f2c"}, {file = "types_requests-2.28.10-py3-none-any.whl", hash = "sha256:45b485725ed58752f2b23461252f1c1ad9205b884a1e35f786bb295525a3e16a"}, diff --git a/pyproject.toml b/pyproject.toml index adf26c68..a622ecd9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,7 @@ ua-parser = "^0.16.1" Babel = "^2.10.3" Werkzeug = "2.1" # removal of parse_rule in 2.2 breaks sphinxcontrib-httpdomain autoflask Authlib = "=1.0.1" +Flask-Limiter = {version = "^2.6.2", extras = ["redis"]} [tool.poetry.dev-dependencies] black = "^22.6" @@ -64,6 +65,7 @@ types-requests = "^2.28" types-freezegun = "^1.1" Sphinx = "^5.1" bandit = "^1.7.4" +types-redis = "^4.3.20" [tool.poetry.scripts] fittrackee = 'fittrackee.__main__:main'