Implement data integration tasks with Celery, including periodic fetching and manual refresh of chat data; add utility functions for data processing and transcript handling; create views and URLs for manual data refresh; establish Redis and Celery configuration; enhance error handling and logging; introduce scripts for data cleanup and fixing dashboard data; update documentation for Redis and Celery setup and troubleshooting.

pyproject.toml

@@ -8,39 +8,46 @@ authors = [{ name = "LiveGraphs Team" }]
 license = { text = "MIT" }
 
 classifiers = [
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.13",
-    "Framework :: Django",
-    "Framework :: Django :: 5.2",
-    "License :: OSI Approved :: MIT License",
-    "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.13",
+  "Framework :: Django",
+  "Framework :: Django :: 5.2",
+  "License :: OSI Approved :: MIT License",
+  "Operating System :: OS Independent",
 ]
 
 dependencies = [
-    "crispy-bootstrap5>=2025.4",
-    "django>=5.2.1",
-    "django-allauth>=65.8.0",
-    "django-crispy-forms>=2.4",
-    "gunicorn>=23.0.0",
-    "numpy>=2.2.5",
-    "pandas>=2.2.3",
-    "plotly>=6.1.0",
-    "python-dotenv>=1.1.0",
-    "whitenoise>=6.9.0",
+  "bleach[css]>=6.2.0",
+  "celery[sqlalchemy]>=5.5.2",
+  "crispy-bootstrap5>=2025.4",
+  "django>=5.2.1",
+  "django-allauth>=65.8.0",
+  "django-celery-beat>=2.8.1",
+  "django-crispy-forms>=2.4",
+  "gunicorn>=23.0.0",
+  "numpy>=2.2.5",
+  "pandas>=2.2.3",
+  "plotly>=6.1.0",
+  "python-dotenv>=1.1.0",
+  "redis>=6.1.0",
+  "requests>=2.32.3",
+  "sqlalchemy>=2.0.41",
+  "tinycss2>=1.4.0",
+  "whitenoise>=6.9.0",
 ]
 
 [dependency-groups]
 dev = [
-    "bandit>=1.8.3",
-    "black>=25.1.0",
-    "coverage>=7.8.0",
-    "django-debug-toolbar>=5.2.0",
-    "django-stubs>=5.2.0",
-    "mypy>=1.15.0",
-    "pre-commit>=4.2.0",
-    "pytest>=8.3.5",
-    "pytest-django>=4.11.1",
-    "ruff>=0.11.10",
+  "bandit>=1.8.3",
+  "black>=25.1.0",
+  "coverage>=7.8.0",
+  "django-debug-toolbar>=5.2.0",
+  "django-stubs>=5.2.0",
+  "mypy>=1.15.0",
+  "pre-commit>=4.2.0",
+  "pytest>=8.3.5",
+  "pytest-django>=4.11.1",
+  "ruff>=0.11.10",
 ]
 
 [build-system]
@@ -56,32 +63,32 @@ packages = ["dashboard_project"]
 [tool.ruff]
 # Exclude a variety of commonly ignored directories.
 exclude = [
-    ".bzr",
-    ".direnv",
-    ".eggs",
-    ".git",
-    ".git-rewrite",
-    ".hg",
-    ".ipynb_checkpoints",
-    ".mypy_cache",
-    ".nox",
-    ".pants.d",
-    ".pyenv",
-    ".pytest_cache",
-    ".pytype",
-    ".ruff_cache",
-    ".svn",
-    ".tox",
-    ".venv",
-    ".vscode",
-    "__pypackages__",
-    "_build",
-    "buck-out",
-    "build",
-    "dist",
-    "node_modules",
-    "site-packages",
-    "venv",
+  ".bzr",
+  ".direnv",
+  ".eggs",
+  ".git",
+  ".git-rewrite",
+  ".hg",
+  ".ipynb_checkpoints",
+  ".mypy_cache",
+  ".nox",
+  ".pants.d",
+  ".pyenv",
+  ".pytest_cache",
+  ".pytype",
+  ".ruff_cache",
+  ".svn",
+  ".tox",
+  ".venv",
+  ".vscode",
+  "__pypackages__",
+  "_build",
+  "buck-out",
+  "build",
+  "dist",
+  "node_modules",
+  "site-packages",
+  "venv",
 ]
 
 # Same as Black.
@@ -103,7 +110,15 @@ indent-style = "space"
 line-ending = "lf"
 
 [tool.bandit]
-exclude_dirs = ["tests", "venv", ".venv", ".git", "__pycache__", "migrations", "**/create_sample_data.py"]
+exclude_dirs = [
+  "tests",
+  "venv",
+  ".venv",
+  ".git",
+  "__pycache__",
+  "migrations",
+  "**/create_sample_data.py",
+]
 skips = ["B101"]
 targets = ["dashboard_project"]
 
@@ -127,26 +142,26 @@ DJANGO_SETTINGS_MODULE = "dashboard_project.settings"
 python_files = "test_*.py"
 testpaths = ["dashboard_project"]
 filterwarnings = [
-    "ignore::DeprecationWarning",
-    "ignore::PendingDeprecationWarning",
+  "ignore::DeprecationWarning",
+  "ignore::PendingDeprecationWarning",
 ]
 
 [tool.coverage.run]
 source = ["dashboard_project"]
 omit = [
-    "dashboard_project/manage.py",
-    "dashboard_project/*/migrations/*",
-    "dashboard_project/*/tests/*",
+  "dashboard_project/manage.py",
+  "dashboard_project/*/migrations/*",
+  "dashboard_project/*/tests/*",
 ]
 
 [tool.coverage.report]
 exclude_lines = [
-    "pragma: no cover",
-    "def __repr__",
-    "raise NotImplementedError",
-    "if __name__ == .__main__.:",
-    "pass",
-    "raise ImportError",
+  "pragma: no cover",
+  "def __repr__",
+  "raise NotImplementedError",
+  "if __name__ == .__main__.:",
+  "pass",
+  "raise ImportError",
 ]
 
 [project.urls]