Implement data integration tasks with Celery, including periodic fetching and manual refresh of chat data; add utility functions for data processing and transcript handling; create views and URLs for manual data refresh; establish Redis and Celery configuration; enhance error handling and logging; introduce scripts for data cleanup and fixing dashboard data; update documentation for Redis and Celery setup and troubleshooting.

2026-01-16 10:32:12 +01:00 · 2025-05-18 13:33:11 +00:00
parent e8f2d2adc2
commit 8bbbb109bd
63 changed files with 4601 additions and 164 deletions
--- a/dashboard_project/data_integration/migrations/0001_initial.py
+++ b/dashboard_project/data_integration/migrations/0001_initial.py
@ -0,0 +1,99 @@
+# Generated by Django 5.2.1 on 2025-05-17 21:14
+
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    initial = True
+
+    dependencies = []
+
+    operations = [
+        migrations.CreateModel(
+            name="ChatSession",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("session_id", models.CharField(max_length=255, unique=True)),
+                ("start_time", models.DateTimeField()),
+                ("end_time", models.DateTimeField()),
+                ("ip_address", models.GenericIPAddressField(blank=True, null=True)),
+                ("country", models.CharField(blank=True, max_length=255, null=True)),
+                ("language", models.CharField(blank=True, max_length=255, null=True)),
+                ("messages_sent", models.IntegerField(blank=True, null=True)),
+                ("sentiment", models.CharField(blank=True, max_length=255, null=True)),
+                ("escalated", models.BooleanField(blank=True, null=True)),
+                ("forwarded_hr", models.BooleanField(blank=True, null=True)),
+                (
+                    "full_transcript_url",
+                    models.URLField(blank=True, max_length=1024, null=True),
+                ),
+                ("avg_response_time", models.FloatField(blank=True, null=True)),
+                ("tokens", models.IntegerField(blank=True, null=True)),
+                ("tokens_eur", models.FloatField(blank=True, null=True)),
+                ("category", models.CharField(blank=True, max_length=255, null=True)),
+                ("initial_msg", models.TextField(blank=True, null=True)),
+                ("user_rating", models.IntegerField(blank=True, null=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="ExternalDataSource",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("name", models.CharField(default="External API", max_length=255)),
+                ("api_url", models.URLField(default="https://proto.notso.ai/XY/chats")),
+                (
+                    "auth_username",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                (
+                    "auth_password",
+                    models.CharField(blank=True, max_length=255, null=True),
+                ),
+                ("last_synced", models.DateTimeField(blank=True, null=True)),
+                ("is_active", models.BooleanField(default=True)),
+            ],
+        ),
+        migrations.CreateModel(
+            name="ChatMessage",
+            fields=[
+                (
+                    "id",
+                    models.BigAutoField(
+                        auto_created=True,
+                        primary_key=True,
+                        serialize=False,
+                        verbose_name="ID",
+                    ),
+                ),
+                ("timestamp", models.DateTimeField(auto_now_add=True)),
+                ("sender", models.CharField(max_length=255)),
+                ("message", models.TextField()),
+                ("safe_html_message", models.TextField(blank=True, null=True)),
+                (
+                    "session",
+                    models.ForeignKey(
+                        on_delete=django.db.models.deletion.CASCADE,
+                        related_name="messages",
+                        to="data_integration.chatsession",
+                    ),
+                ),
+            ],
+        ),
+    ]