Implement data integration tasks with Celery, including periodic fetching and manual refresh of chat data; add utility functions for data processing and transcript handling; create views and URLs for manual data refresh; establish Redis and Celery configuration; enhance error handling and logging; introduce scripts for data cleanup and fixing dashboard data; update documentation for Redis and Celery setup and troubleshooting.

Makefile

@@ -1,4 +1,4 @@
-.PHONY: venv install install-dev lint test format clean run migrate makemigrations superuser setup-node
+.PHONY: venv install install-dev lint test format clean run migrate makemigrations superuser setup-node celery celery-beat docker-build docker-up docker-down reset-db setup-dev procfile
 
 # Create a virtual environment
 venv:
@@ -51,6 +51,14 @@ clean:
 run:
 	cd dashboard_project && uv run python manage.py runserver 8001
 
+# Run Celery worker for background tasks
+celery:
+	cd dashboard_project && uv run celery -A dashboard_project worker --loglevel=info
+
+# Run Celery Beat for scheduled tasks
+celery-beat:
+	cd dashboard_project && uv run celery -A dashboard_project beat --scheduler django_celery_beat.schedulers:DatabaseScheduler
+
 # Apply migrations
 migrate:
 	cd dashboard_project && uv run python manage.py migrate
@@ -69,9 +77,51 @@ lock:
 
 # Setup pre-commit hooks
 setup-pre-commit:
-	uv pip install pre-commit
 	pre-commit install
 
 # Run pre-commit on all files
 lint-all:
 	pre-commit run --all-files
+
+# Docker commands
+docker-build:
+	docker-compose build
+
+docker-up:
+	docker-compose up -d
+
+docker-down:
+	docker-compose down
+
+# Initialize or reset the database in development
+reset-db:
+	cd dashboard_project && uv run python manage.py flush --no-input
+	cd dashboard_project && uv run python manage.py migrate
+
+# Start a Redis server in development (if not installed, fallback to SQLite)
+run-redis:
+	redis-server || echo "Redis not installed, using SQLite fallback"
+
+# Start all development services (web, redis, celery, celery-beat)
+run-all:
+	make run-redis & \
+	make run & \
+	make celery & \
+	make celery-beat
+
+# Test Celery task
+test-celery:
+	cd dashboard_project && uv run python manage.py test_celery
+
+# Initialize data integration
+init-data-integration:
+	cd dashboard_project && uv run python manage.py create_default_datasource
+	cd dashboard_project && uv run python manage.py create_default_datasource
+	cd dashboard_project && uv run python manage.py test_celery
+
+# Setup development environment
+setup-dev: venv install-dev migrate create_default_datasource
+	@echo "Development environment setup complete"
+
+procfile:
+	foreman start