Implement data integration tasks with Celery, including periodic fetching and manual refresh of chat data; add utility functions for data processing and transcript handling; create views and URLs for manual data refresh; establish Redis and Celery configuration; enhance error handling and logging; introduce scripts for data cleanup and fixing dashboard data; update documentation for Redis and Celery setup and troubleshooting.

This commit is contained in:
2025-05-18 13:33:11 +00:00
parent e8f2d2adc2
commit 8bbbb109bd
63 changed files with 4601 additions and 164 deletions

View File

@ -5,18 +5,55 @@ from django.contrib import admin
from .models import ChatSession, Dashboard, DataSource
@admin.register(DataSource)
class DataSourceAdmin(admin.ModelAdmin):
list_display = ("name", "company", "uploaded_at", "get_session_count")
list_display = (
"name",
"company",
"uploaded_at",
"get_external_source",
"get_session_count",
)
list_filter = ("company", "uploaded_at")
search_fields = ("name", "description", "company__name")
ordering = ("-uploaded_at",)
readonly_fields = ("get_external_data_status",)
fieldsets = (
(None, {"fields": ("name", "description", "company")}),
(
"Data Source",
{
"fields": ("file", "external_source"),
"description": "Either upload a file OR select an external data source. Not both.",
},
),
(
"Stats",
{
"fields": ("get_external_data_status",),
},
),
)
@admin.display(description="Sessions")
def get_session_count(self, obj):
return obj.chat_sessions.count()
get_session_count.short_description = "Sessions"
@admin.display(description="External Source")
def get_external_source(self, obj):
if obj.external_source:
return obj.external_source.name
return "None"
@admin.display(description="External Data Status")
def get_external_data_status(self, obj):
if obj.external_source:
return f"Last synced: {obj.external_source.last_synced or 'Never'} | Status: {obj.external_source.get_status()}"
return "No external data source linked"
@admin.register(ChatSession)
class ChatSessionAdmin(admin.ModelAdmin):
list_display = (
"session_id",
@ -45,21 +82,18 @@ class ChatSessionAdmin(admin.ModelAdmin):
)
ordering = ("-start_time",)
@admin.display(
description="Company",
ordering="data_source__company__name",
)
def get_company(self, obj):
return obj.data_source.company.name
get_company.short_description = "Company"
get_company.admin_order_field = "data_source__company__name"
@admin.register(Dashboard)
class DashboardAdmin(admin.ModelAdmin):
list_display = ("name", "company", "created_at", "updated_at")
list_filter = ("company", "created_at")
search_fields = ("name", "description", "company__name")
filter_horizontal = ("data_sources",)
ordering = ("-updated_at",)
admin.site.register(DataSource, DataSourceAdmin)
admin.site.register(ChatSession, ChatSessionAdmin)
admin.site.register(Dashboard, DashboardAdmin)

View File

@ -6,3 +6,7 @@ from django.apps import AppConfig
class DashboardConfig(AppConfig):
default_auto_field = "django.db.models.BigAutoField"
name = "dashboard"
def ready(self):
# Import signals
pass

View File

@ -0,0 +1,173 @@
# dashboard/management/commands/create_test_data.py
import csv
import os
from datetime import datetime, timedelta
from dashboard.models import DataSource
from data_integration.models import ChatMessage, ChatSession, ExternalDataSource
from django.core.management.base import BaseCommand
from django.utils.timezone import make_aware
class Command(BaseCommand):
help = "Create test data for external data source and link it to a dashboard data source"
def add_arguments(self, parser):
parser.add_argument(
"--company-id",
type=int,
help="Company ID to associate with the data source",
required=True,
)
parser.add_argument(
"--sample-file",
type=str,
help="Path to sample CSV file",
default="examples/sample.csv",
)
def handle(self, *args, **options): # noqa: ARG002
company_id = options["company_id"]
sample_file = options["sample_file"]
# Check if sample file exists
project_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../.."))
sample_path = os.path.join(project_root, sample_file)
if not os.path.exists(sample_path):
self.stdout.write(self.style.ERROR(f"Sample file not found: {sample_path}"))
return
# Create or get external data source
ext_source, created = ExternalDataSource.objects.get_or_create(
name="Test External Source",
defaults={
"api_url": "https://example.com/api",
"is_active": True,
"sync_interval": 3600,
"last_synced": make_aware(datetime.now()),
},
)
if created:
self.stdout.write(
self.style.SUCCESS(f"Created external data source: {ext_source.name} (ID: {ext_source.id})")
)
else:
self.stdout.write(f"Using existing external data source: {ext_source.name} (ID: {ext_source.id})")
# Create or get dashboard data source linked to external source
dash_source, created = DataSource.objects.get_or_create(
name="Test Dashboard Source",
company_id=company_id,
external_source=ext_source,
defaults={"description": "Test data source linked to external API"},
)
if created:
self.stdout.write(
self.style.SUCCESS(f"Created dashboard data source: {dash_source.name} (ID: {dash_source.id})")
)
else:
self.stdout.write(f"Using existing dashboard data source: {dash_source.name} (ID: {dash_source.id})")
# Import test data from CSV
session_count = 0
message_count = 0
# First clear any existing sessions
existing_count = ChatSession.objects.filter().count()
if existing_count > 0:
self.stdout.write(f"Clearing {existing_count} existing chat sessions")
ChatSession.objects.all().delete()
# Parse sample CSV
with open(sample_path, "r") as f:
reader = csv.reader(f)
header = next(reader) # Skip header
for row in reader:
# Make sure row has enough elements
padded_row = row + [""] * (len(header) - len(row))
# Create a dict from the row
data = dict(zip(header, padded_row, strict=False))
# Create a chat session
try:
# Parse dates
try:
start_time = make_aware(datetime.strptime(data.get("start_time", ""), "%d.%m.%Y %H:%M:%S"))
except ValueError:
start_time = make_aware(datetime.now() - timedelta(hours=1))
try:
end_time = make_aware(datetime.strptime(data.get("end_time", ""), "%d.%m.%Y %H:%M:%S"))
except ValueError:
end_time = make_aware(datetime.now())
# Convert values to appropriate types
escalated = data.get("escalated", "").lower() == "true"
forwarded_hr = data.get("forwarded_hr", "").lower() == "true"
messages_sent = int(data.get("messages_sent", 0) or 0)
tokens = int(data.get("tokens", 0) or 0)
tokens_eur = float(data.get("tokens_eur", 0) or 0)
user_rating = int(data.get("user_rating", 0) or 0) if data.get("user_rating", "") else None
# Create session
session = ChatSession.objects.create(
session_id=data.get("session_id", f"test-{session_count}"),
start_time=start_time,
end_time=end_time,
ip_address=data.get("ip_address", "127.0.0.1"),
country=data.get("country", ""),
language=data.get("language", ""),
messages_sent=messages_sent,
sentiment=data.get("sentiment", ""),
escalated=escalated,
forwarded_hr=forwarded_hr,
full_transcript_url=data.get("full_transcript", ""),
avg_response_time=float(data.get("avg_response_time", 0) or 0),
tokens=tokens,
tokens_eur=tokens_eur,
category=data.get("category", ""),
initial_msg=data.get("initial_msg", ""),
user_rating=user_rating,
)
session_count += 1
# Create messages for this session
if data.get("initial_msg"):
# User message
ChatMessage.objects.create(
session=session,
sender="User",
message=data.get("initial_msg", ""),
timestamp=start_time,
)
message_count += 1
# Assistant response
ChatMessage.objects.create(
session=session,
sender="Assistant",
message=f"This is a test response to {data.get('initial_msg', '')}",
timestamp=start_time + timedelta(seconds=30),
)
message_count += 1
except Exception as e:
self.stdout.write(self.style.ERROR(f"Error creating session: {e}"))
self.stdout.write(self.style.SUCCESS(f"Created {session_count} chat sessions with {message_count} messages"))
# Run the sync command to copy data to dashboard
self.stdout.write("Syncing data to dashboard...")
from django.core.management import call_command
call_command("sync_external_data", source_id=ext_source.id)
self.stdout.write(self.style.SUCCESS("Done! Your dashboard should now show test data."))

View File

@ -0,0 +1,128 @@
# dashboard/management/commands/sync_external_data.py
import logging
from dashboard.models import ChatSession as DashboardChatSession
from dashboard.models import DataSource
from data_integration.models import ChatSession as ExternalChatSession
from django.core.management.base import BaseCommand
from django.db import transaction
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = "Synchronize data from external data sources to dashboard data sources"
def add_arguments(self, parser):
parser.add_argument(
"--source-id",
type=int,
help="Specific external data source ID to sync",
required=False,
)
parser.add_argument(
"--clear",
action="store_true",
help="Clear existing dashboard data before sync",
)
def handle(self, *args, **options): # noqa: ARG002
source_id = options.get("source_id")
clear_existing = options.get("clear", False)
# Get all datasources that have an external_source
if source_id:
data_sources = DataSource.objects.filter(external_source_id=source_id)
if not data_sources.exists():
self.stdout.write(
self.style.WARNING(f"No dashboard data sources linked to external source ID {source_id}")
)
return
else:
data_sources = DataSource.objects.exclude(external_source=None)
if not data_sources.exists():
self.stdout.write(self.style.WARNING("No dashboard data sources with external sources found"))
return
total_synced = 0
total_errors = 0
for data_source in data_sources:
self.stdout.write(f"Processing dashboard data source: {data_source.name} (ID: {data_source.id})")
if not data_source.external_source:
self.stdout.write(self.style.WARNING(f" - No external source linked to {data_source.name}"))
continue
# Get all external chat sessions for this source
external_sessions = ExternalChatSession.objects.all()
session_count = external_sessions.count()
if session_count == 0:
self.stdout.write(self.style.WARNING(" - No external sessions found"))
continue
self.stdout.write(f" - Found {session_count} external sessions")
# Clear existing data if requested
if clear_existing:
existing_count = DashboardChatSession.objects.filter(data_source=data_source).count()
if existing_count > 0:
self.stdout.write(f" - Clearing {existing_count} existing dashboard sessions")
DashboardChatSession.objects.filter(data_source=data_source).delete()
# Process each external session
synced_count = 0
error_count = 0
for ext_session in external_sessions:
try:
with transaction.atomic():
# Create or update dashboard chat session
(
dashboard_session,
created,
) = DashboardChatSession.objects.update_or_create(
data_source=data_source,
session_id=ext_session.session_id,
defaults={
"start_time": ext_session.start_time,
"end_time": ext_session.end_time,
"ip_address": ext_session.ip_address,
"country": ext_session.country or "",
"language": ext_session.language or "",
"messages_sent": ext_session.messages_sent or 0,
"sentiment": ext_session.sentiment or "",
"escalated": ext_session.escalated or False,
"forwarded_hr": ext_session.forwarded_hr or False,
"full_transcript": ext_session.full_transcript_url or "",
"avg_response_time": ext_session.avg_response_time,
"tokens": ext_session.tokens or 0,
"tokens_eur": ext_session.tokens_eur,
"category": ext_session.category or "",
"initial_msg": ext_session.initial_msg or "",
"user_rating": (
str(ext_session.user_rating) if ext_session.user_rating is not None else ""
),
},
)
synced_count += 1
action = "Created" if created else "Updated"
self.stdout.write(f" - {action} session: {dashboard_session.session_id}")
except Exception as e:
self.stdout.write(self.style.ERROR(f" - Error syncing session {ext_session.session_id}: {str(e)}"))
logger.error(
f"Error syncing session {ext_session.session_id}: {e}",
exc_info=True,
)
error_count += 1
self.stdout.write(self.style.SUCCESS(f" - Synced {synced_count} sessions with {error_count} errors"))
total_synced += synced_count
total_errors += error_count
self.stdout.write(
self.style.SUCCESS(f"Sync complete. Total: {total_synced} sessions synced, {total_errors} errors")
)

View File

@ -0,0 +1,110 @@
# Generated by Django 5.2.1 on 2025-05-16 21:25
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
initial = True
dependencies = [
("accounts", "0001_initial"),
]
operations = [
migrations.CreateModel(
name="DataSource",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=255)),
("description", models.TextField(blank=True)),
("file", models.FileField(upload_to="data_sources/")),
("uploaded_at", models.DateTimeField(auto_now_add=True)),
(
"company",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="data_sources",
to="accounts.company",
),
),
],
),
migrations.CreateModel(
name="Dashboard",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("name", models.CharField(max_length=255)),
("description", models.TextField(blank=True)),
("created_at", models.DateTimeField(auto_now_add=True)),
("updated_at", models.DateTimeField(auto_now=True)),
(
"company",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="dashboards",
to="accounts.company",
),
),
(
"data_sources",
models.ManyToManyField(related_name="dashboards", to="dashboard.datasource"),
),
],
),
migrations.CreateModel(
name="ChatSession",
fields=[
(
"id",
models.BigAutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("session_id", models.CharField(max_length=255)),
("start_time", models.DateTimeField(blank=True, null=True)),
("end_time", models.DateTimeField(blank=True, null=True)),
("ip_address", models.GenericIPAddressField(blank=True, null=True)),
("country", models.CharField(blank=True, max_length=100)),
("language", models.CharField(blank=True, max_length=50)),
("messages_sent", models.IntegerField(default=0)),
("sentiment", models.CharField(blank=True, max_length=50)),
("escalated", models.BooleanField(default=False)),
("forwarded_hr", models.BooleanField(default=False)),
("full_transcript", models.TextField(blank=True)),
("avg_response_time", models.FloatField(blank=True, null=True)),
("tokens", models.IntegerField(default=0)),
("tokens_eur", models.FloatField(blank=True, null=True)),
("category", models.CharField(blank=True, max_length=100)),
("initial_msg", models.TextField(blank=True)),
("user_rating", models.CharField(blank=True, max_length=50)),
(
"data_source",
models.ForeignKey(
on_delete=django.db.models.deletion.CASCADE,
related_name="chat_sessions",
to="dashboard.datasource",
),
),
],
),
]

View File

@ -0,0 +1,35 @@
# Generated by Django 5.2.1 on 2025-05-17 23:10
import django.db.models.deletion
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("dashboard", "0001_initial"),
("data_integration", "0002_externaldatasource_error_count_and_more"),
]
operations = [
migrations.AddField(
model_name="datasource",
name="external_source",
field=models.ForeignKey(
blank=True,
help_text="Link to an external data source",
null=True,
on_delete=django.db.models.deletion.SET_NULL,
to="data_integration.externaldatasource",
),
),
migrations.AlterField(
model_name="datasource",
name="file",
field=models.FileField(
blank=True,
help_text="Upload a CSV file or leave empty if using an external data source",
null=True,
upload_to="data_sources/",
),
),
]

View File

@ -0,0 +1,16 @@
# Generated by Django 5.2.1 on 2025-05-18 00:09
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("dashboard", "0002_datasource_external_source_alter_datasource_file"),
]
operations = [
migrations.AlterUniqueTogether(
name="chatsession",
unique_together={("session_id", "data_source")},
),
]

View File

@ -5,11 +5,23 @@ from django.db import models
class DataSource(models.Model):
"""Model for uploaded data sources (CSV files)"""
"""Model for data sources (CSV files or external API data)"""
name = models.CharField(max_length=255)
description = models.TextField(blank=True)
file = models.FileField(upload_to="data_sources/")
file = models.FileField(
upload_to="data_sources/",
blank=True,
null=True,
help_text="Upload a CSV file or leave empty if using an external data source",
)
external_source = models.ForeignKey(
"data_integration.ExternalDataSource",
on_delete=models.SET_NULL,
blank=True,
null=True,
help_text="Link to an external data source",
)
uploaded_at = models.DateTimeField(auto_now_add=True)
company = models.ForeignKey(Company, on_delete=models.CASCADE, related_name="data_sources")
@ -42,6 +54,9 @@ class ChatSession(models.Model):
def __str__(self):
return f"Session {self.session_id}"
class Meta:
unique_together = ("session_id", "data_source")
class Dashboard(models.Model):
"""Model for custom dashboards that can be created by users"""

View File

@ -0,0 +1,79 @@
# dashboard/signals.py
import logging
from dashboard.models import ChatSession as DashboardChatSession
from dashboard.models import DataSource
from data_integration.models import ChatSession as ExternalChatSession
from django.db.models.signals import post_save
from django.dispatch import receiver
logger = logging.getLogger(__name__)
@receiver(post_save, sender=ExternalChatSession)
def sync_external_session_to_dashboard(
sender, # noqa: ARG001
instance,
created,
**kwargs, # noqa: ARG001
):
"""
Signal handler to sync external chat sessions to dashboard chat sessions
whenever an external session is created or updated.
Args:
sender: The model class that sent the signal (unused but required by Django's signal interface)
instance: The ExternalChatSession instance that was saved
created: Boolean indicating if this is a new instance
**kwargs: Additional keyword arguments (unused but required by Django's signal interface)
"""
# Find all dashboard data sources that are linked to this external data source
# Since ExternalChatSession doesn't have a direct link to ExternalDataSource,
# we need to sync to all dashboard data sources with external sources
data_sources = DataSource.objects.exclude(external_source=None)
if not data_sources.exists():
logger.warning(f"No dashboard data sources with external sources found for session {instance.session_id}")
return
for data_source in data_sources:
try:
# Create or update dashboard chat session
dashboard_session, created = DashboardChatSession.objects.update_or_create(
data_source=data_source,
session_id=instance.session_id,
defaults={
"start_time": instance.start_time,
"end_time": instance.end_time,
"ip_address": instance.ip_address,
"country": instance.country or "",
"language": instance.language or "",
"messages_sent": instance.messages_sent or 0,
"sentiment": instance.sentiment or "",
"escalated": instance.escalated or False,
"forwarded_hr": instance.forwarded_hr or False,
"full_transcript": instance.full_transcript_url or "",
"avg_response_time": instance.avg_response_time,
"tokens": instance.tokens or 0,
"tokens_eur": instance.tokens_eur,
"category": instance.category or "",
"initial_msg": instance.initial_msg or "",
"user_rating": (str(instance.user_rating) if instance.user_rating is not None else ""),
},
)
if created:
logger.info(
f"Created dashboard session: {dashboard_session.session_id} for data source {data_source.name}"
)
else:
logger.info(
f"Updated dashboard session: {dashboard_session.session_id} for data source {data_source.name}"
)
except Exception as e:
logger.error(
f"Error syncing session {instance.session_id} to data source {data_source.name}: {e}",
exc_info=True,
)

View File

@ -42,4 +42,6 @@ urlpatterns = [
path("data-view/", views.data_view, name="data_view"),
# Export to CSV
path("export/csv/", views_export.export_chats_csv, name="export_chats_csv"),
# Export to JSON
path("export/json/", views_export.export_chats_json, name="export_chats_json"),
]

View File

@ -200,7 +200,12 @@ def chat_session_detail_view(request, session_id):
# Check if this is an AJAX navigation request
if is_ajax_navigation(request):
html_content = render_to_string("dashboard/chat_session_detail.html", context, request=request)
return JsonResponse({"html": html_content, "title": f"Chat Session {session_id} | Chat Analytics"})
return JsonResponse(
{
"html": html_content,
"title": f"Chat Session {session_id} | Chat Analytics",
}
)
return render(request, "dashboard/chat_session_detail.html", context)
@ -277,7 +282,12 @@ def edit_dashboard_view(request, dashboard_id):
# Check if this is an AJAX navigation request
if is_ajax_navigation(request):
html_content = render_to_string("dashboard/dashboard_form.html", context, request=request)
return JsonResponse({"html": html_content, "title": f"Edit Dashboard: {dashboard.name} | Chat Analytics"})
return JsonResponse(
{
"html": html_content,
"title": f"Edit Dashboard: {dashboard.name} | Chat Analytics",
}
)
return render(request, "dashboard/dashboard_form.html", context)

View File

@ -1,6 +1,7 @@
# dashboard/views_export.py
import csv
import json
from datetime import timedelta
from django.contrib.auth.decorators import login_required
@ -135,3 +136,115 @@ def export_chats_csv(request):
)
return response
@login_required
def export_chats_json(request):
"""Export chat sessions to JSON with filtering options"""
user = request.user
company = user.company
if not company:
return HttpResponse("You are not associated with any company.", status=403)
# Get and apply filters
data_source_id = request.GET.get("data_source_id")
dashboard_id = request.GET.get("dashboard_id")
view = request.GET.get("view", "all")
start_date = request.GET.get("start_date")
end_date = request.GET.get("end_date")
country = request.GET.get("country")
sentiment = request.GET.get("sentiment")
escalated = request.GET.get("escalated")
# Base queryset
sessions = ChatSession.objects.filter(data_source__company=company)
# Apply data source filter if selected
if data_source_id:
data_source = get_object_or_404(DataSource, id=data_source_id, company=company)
sessions = sessions.filter(data_source=data_source)
# Apply dashboard filter if selected
if dashboard_id:
dashboard = get_object_or_404(Dashboard, id=dashboard_id, company=company)
data_sources = dashboard.data_sources.all()
sessions = sessions.filter(data_source__in=data_sources)
# Apply view filter
if view == "recent":
seven_days_ago = timezone.now() - timedelta(days=7)
sessions = sessions.filter(start_time__gte=seven_days_ago)
elif view == "positive":
sessions = sessions.filter(Q(sentiment__icontains="positive"))
elif view == "negative":
sessions = sessions.filter(Q(sentiment__icontains="negative"))
elif view == "escalated":
sessions = sessions.filter(escalated=True)
# Apply additional filters
if start_date:
sessions = sessions.filter(start_time__date__gte=start_date)
if end_date:
sessions = sessions.filter(start_time__date__lte=end_date)
if country:
sessions = sessions.filter(country__icontains=country)
if sentiment:
sessions = sessions.filter(sentiment__icontains=sentiment)
if escalated:
escalated_val = escalated.lower() == "true"
sessions = sessions.filter(escalated=escalated_val)
# Order by most recent first
sessions = sessions.order_by("-start_time")
# Create the filename
filename = "chat_sessions"
if dashboard_id:
dashboard = Dashboard.objects.get(id=dashboard_id)
filename = f"{dashboard.name.replace(' ', '_').lower()}_chat_sessions"
elif data_source_id:
data_source = DataSource.objects.get(id=data_source_id)
filename = f"{data_source.name.replace(' ', '_').lower()}_chat_sessions"
# Prepare the data for JSON export using list comprehension
data = [
{
"session_id": session.session_id,
"start_time": (session.start_time.isoformat() if session.start_time else None),
"end_time": session.end_time.isoformat() if session.end_time else None,
"ip_address": session.ip_address,
"country": session.country,
"language": session.language,
"messages_sent": session.messages_sent,
"sentiment": session.sentiment,
"escalated": session.escalated,
"forwarded_hr": session.forwarded_hr,
"full_transcript": session.full_transcript,
"avg_response_time": session.avg_response_time,
"tokens": session.tokens,
"tokens_eur": session.tokens_eur,
"category": session.category,
"initial_msg": session.initial_msg,
"user_rating": session.user_rating,
}
for session in sessions
]
# Create the HttpResponse with JSON header
response = HttpResponse(content_type="application/json")
response["Content-Disposition"] = f'attachment; filename="{filename}.json"'
# Add company and timestamp to the exported JSON
current_time = timezone.now().isoformat()
export_data = {
"company": company.name,
"export_date": current_time,
"export_type": "chat_sessions",
"data": data,
}
# Write JSON data to the response
json.dump(export_data, response, indent=2)
return response