kjanat/livedash-node
1
0
Fork 0
mirror of https://github.com/kjanat/livedash-node.git synced 2026-01-16 09:52:09 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat: refactor session processing pipeline to implement multi-stage tracking and enhance error handling

Browse Source
This commit is contained in:
Max Kowalski
2025-06-27 23:12:04 +02:00
parent 2dfc49f840
commit 8ffd5a7a2c
3 changed files with 185 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
check-pipeline-status.ts
View File

@ -1,96 +0,0 @@
import { PrismaClient } from '@prisma/client';
const prisma = new PrismaClient();
async function checkPipelineStatus() {
try {
console.log('=== COMPLETE PIPELINE STATUS ===\n');
// Stage 1: SessionImport status
console.log('1. SessionImport Status:');
const importCounts = await prisma.sessionImport.groupBy({
by: ['status'],
_count: { status: true }
});
const totalImports = await prisma.sessionImport.count();
console.log(` Total imports: ${totalImports}`);
importCounts.forEach(({ status, _count }) => {
console.log(` ${status}: ${_count.status}`);
});
// Stage 2: Session creation status
console.log('\n2. Session Creation Status:');
const totalSessions = await prisma.session.count();
const sessionsWithMessages = await prisma.session.count({
where: { messages: { some: {} } }
});
const sessionsWithoutMessages = await prisma.session.count({
where: { messages: { none: {} } }
});
console.log(` Total sessions: ${totalSessions}`);
console.log(` Sessions with messages: ${sessionsWithMessages}`);
console.log(` Sessions without messages: ${sessionsWithoutMessages}`);
// Stage 3: AI Processing status
console.log('\n3. AI Processing Status:');
const processedSessions = await prisma.session.count({
where: { processed: true }
});
const unprocessedSessions = await prisma.session.count({
where: { processed: false }
});
console.log(` Processed sessions: ${processedSessions}`);
console.log(` Unprocessed sessions: ${unprocessedSessions}`);
// Stage 4: Questions extracted
console.log('\n4. Question Extraction Status:');
const sessionsWithQuestions = await prisma.session.count({
where: { sessionQuestions: { some: {} } }
});
const totalQuestions = await prisma.question.count();
console.log(` Sessions with questions: ${sessionsWithQuestions}`);
console.log(` Total unique questions: ${totalQuestions}`);
// Show what needs processing
console.log('\n=== WHAT NEEDS PROCESSING ===');
const queuedImports = await prisma.sessionImport.count({
where: { status: 'QUEUED' }
});
console.log(`• ${queuedImports} SessionImports need import processing`);
const sessionsNeedingAI = await prisma.session.count({
where: {
AND: [
{ messages: { some: {} } },
{ processed: false }
]
}
});
console.log(`• ${sessionsNeedingAI} Sessions need AI processing`);
// Sample of what's pending
if (queuedImports > 0) {
console.log('\nSample queued imports:');
const sampleImports = await prisma.sessionImport.findMany({
where: { status: 'QUEUED' },
select: { externalSessionId: true, createdAt: true },
take: 5
});
sampleImports.forEach(imp => {
console.log(` ${imp.externalSessionId} (created: ${imp.createdAt})`);
});
}
} catch (error) {
console.error('Error checking pipeline status:', error);
} finally {
await prisma.$disconnect();
}
}
checkPipelineStatus();

133
docs/processing-system-refactor.md Normal file
View File

@ -0,0 +1,133 @@
# Processing System Refactor - Complete
## Overview
Successfully refactored the session processing pipeline from a simple status-based system to a comprehensive multi-stage processing status system. This addresses the original issues with the SessionImport table's `status` and `errorMsg` columns.
## Problems Solved
### Original Issues
1. **Inconsistent Status Tracking**: The old system used a simple enum on SessionImport that didn't properly track the multi-stage processing pipeline
2. **Poor Error Visibility**: Error messages were buried in the SessionImport table and not easily accessible
3. **No Stage-Specific Tracking**: The system couldn't track which specific stage of processing failed
4. **Difficult Recovery**: Failed sessions were hard to identify and retry
5. **Linting Errors**: Multiple TypeScript files referencing removed database fields
### Schema Changes Made
- **Removed** old `status`, `errorMsg`, and `processedAt` columns from SessionImport
- **Removed** `processed` field from Session
- **Added** new `SessionProcessingStatus` table with granular stage tracking
- **Added** `ProcessingStage` and `ProcessingStatus` enums
## New Processing Pipeline
### Processing Stages
```typescript
enum ProcessingStage {
CSV_IMPORT // SessionImport created
TRANSCRIPT_FETCH // Transcript content fetched
SESSION_CREATION // Session + Messages created
AI_ANALYSIS // AI processing completed
QUESTION_EXTRACTION // Questions extracted
}
enum ProcessingStatus {
PENDING, IN_PROGRESS, COMPLETED, FAILED, SKIPPED
}
```
### Key Components
#### 1. ProcessingStatusManager
Centralized class for managing processing status with methods:
- `initializeSession()` - Set up processing status for new sessions
- `startStage()`, `completeStage()`, `failStage()`, `skipStage()` - Stage management
- `getSessionsNeedingProcessing()` - Query sessions by stage and status
- `getPipelineStatus()` - Get overview of entire pipeline
- `getFailedSessions()` - Find sessions needing retry
- `resetStageForRetry()` - Reset failed stages
#### 2. Updated Processing Scheduler
- Integrated with new `ProcessingStatusManager`
- Tracks AI analysis and question extraction stages
- Records detailed processing metadata
- Proper error handling and retry capabilities
#### 3. Migration System
- Successfully migrated all 109 existing sessions
- Determined current state based on existing data
- Preserved all existing functionality
## Current Pipeline Status
After migration and refactoring:
- **CSV_IMPORT**: 109 completed
- **TRANSCRIPT_FETCH**: 109 completed
- **SESSION_CREATION**: 109 completed
- **AI_ANALYSIS**: 16 completed, 93 pending
- **QUESTION_EXTRACTION**: 11 completed, 98 pending
## Files Updated/Created
### New Files
- `lib/processingStatusManager.ts` - Core processing status management
- `check-refactored-pipeline-status.ts` - New pipeline status checker
- `migrate-to-refactored-system.ts` - Migration script
- `docs/processing-system-refactor.md` - This documentation
### Updated Files
- `prisma/schema.prisma` - Added new processing status tables
- `lib/processingScheduler.ts` - Integrated with new status system
- `debug-import-status.ts` - Updated to use new system
- `fix-import-status.ts` - Updated to use new system
### Removed Files
- `check-pipeline-status.ts` - Replaced by refactored version
## Benefits Achieved
1. **Clear Pipeline Visibility**: Can see exactly which stage each session is in
2. **Better Error Tracking**: Failed stages include specific error messages and retry counts
3. **Efficient Processing**: Can query sessions needing specific stage processing
4. **Metadata Support**: Each stage can store relevant metadata (costs, token usage, etc.)
5. **Easy Recovery**: Failed sessions can be easily identified and retried
6. **Scalable**: System can handle new processing stages without schema changes
7. **No Linting Errors**: All TypeScript compilation issues resolved
## Usage Examples
### Check Pipeline Status
```bash
npx tsx check-refactored-pipeline-status.ts
```
### Debug Processing Issues
```bash
npx tsx debug-import-status.ts
```
### Fix/Retry Failed Sessions
```bash
npx tsx fix-import-status.ts
```
### Process Sessions
```bash
npx tsx test-ai-processing.ts
```
## Next Steps
1. **Test AI Processing**: Run AI processing on pending sessions
2. **Monitor Performance**: Watch for any issues with the new system
3. **Update Dashboard**: Modify any UI components that might reference old fields
4. **Documentation**: Update any API documentation that references the old system
## Migration Notes
- All existing data preserved
- No data loss during migration
- Backward compatibility maintained where possible
- System ready for production use
The refactored system provides much better visibility into the processing pipeline and makes it easy to identify and resolve any issues that arise during session processing.

74
lib/processingScheduler.ts
View File

@ -1,8 +1,9 @@
// Enhanced session processing scheduler with AI cost tracking and question management // Enhanced session processing scheduler with AI cost tracking and question management
import cron from "node-cron"; import cron from "node-cron";
import { PrismaClient, SentimentCategory, SessionCategory } from "@prisma/client"; import { PrismaClient, SentimentCategory, SessionCategory, ProcessingStage } from "@prisma/client";
import fetch from "node-fetch"; import fetch from "node-fetch";
import { getSchedulerConfig } from "./schedulerConfig"; import { getSchedulerConfig } from "./schedulerConfig";
import { ProcessingStatusManager } from "./processingStatusManager";
const prisma = new PrismaClient(); const prisma = new PrismaClient();
const OPENAI_API_KEY = process.env.OPENAI_API_KEY; const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
@ -382,6 +383,9 @@ async function processSingleSession(session: any): Promise<ProcessingResult> {
} }
try { try {
// Mark AI analysis as started
await ProcessingStatusManager.startStage(session.id, ProcessingStage.AI_ANALYSIS);
// Convert messages back to transcript format for OpenAI processing // Convert messages back to transcript format for OpenAI processing
const transcript = session.messages const transcript = session.messages
.map((msg: any) => .map((msg: any) =>
@ -406,9 +410,6 @@ async function processSingleSession(session: any): Promise<ProcessingResult> {
// Calculate endTime from latest Message timestamp // Calculate endTime from latest Message timestamp
const calculatedEndTime = await calculateEndTime(session.id, session.endTime); const calculatedEndTime = await calculateEndTime(session.id, session.endTime);
// Process questions into separate tables
await processQuestions(session.id, processedData.questions);
// Update the session with processed data // Update the session with processed data
await prisma.session.update({ await prisma.session.update({
where: { id: session.id }, where: { id: session.id },
@ -421,15 +422,40 @@ async function processSingleSession(session: any): Promise<ProcessingResult> {
forwardedHr: processedData.forwarded_hr, forwardedHr: processedData.forwarded_hr,
category: processedData.category as SessionCategory, category: processedData.category as SessionCategory,
summary: processedData.summary, summary: processedData.summary,
processed: true,
}, },
}); });
// Mark AI analysis as completed
await ProcessingStatusManager.completeStage(session.id, ProcessingStage.AI_ANALYSIS, {
language: processedData.language,
sentiment: processedData.sentiment,
category: processedData.category,
questionsCount: processedData.questions.length
});
// Start question extraction stage
await ProcessingStatusManager.startStage(session.id, ProcessingStage.QUESTION_EXTRACTION);
// Process questions into separate tables
await processQuestions(session.id, processedData.questions);
// Mark question extraction as completed
await ProcessingStatusManager.completeStage(session.id, ProcessingStage.QUESTION_EXTRACTION, {
questionsProcessed: processedData.questions.length
});
return { return {
sessionId: session.id, sessionId: session.id,
success: true, success: true,
}; };
} catch (error) { } catch (error) {
// Mark AI analysis as failed
await ProcessingStatusManager.failStage(
session.id,
ProcessingStage.AI_ANALYSIS,
error instanceof Error ? error.message : String(error)
);
return { return {
sessionId: session.id, sessionId: session.id,
success: false, success: false,
@ -471,32 +497,36 @@ async function processSessionsInParallel(sessions: any[], maxConcurrency: number
} }
/** /**
* Process unprocessed sessions * Process unprocessed sessions using the new processing status system
*/ */
export async function processUnprocessedSessions(batchSize: number | null = null, maxConcurrency: number = 5): Promise<void> { export async function processUnprocessedSessions(batchSize: number | null = null, maxConcurrency: number = 5): Promise<void> {
process.stdout.write("[ProcessingScheduler] Starting to process unprocessed sessions...\n"); process.stdout.write("[ProcessingScheduler] Starting to process sessions needing AI analysis...\n");
// Find sessions that have messages but haven't been processed // Get sessions that need AI processing using the new status system
const queryOptions: any = { const sessionsNeedingAI = await ProcessingStatusManager.getSessionsNeedingProcessing(
ProcessingStage.AI_ANALYSIS,
batchSize || 50
);
if (sessionsNeedingAI.length === 0) {
process.stdout.write("[ProcessingScheduler] No sessions found requiring AI processing.\n");
return;
}
// Get session IDs that need processing
const sessionIds = sessionsNeedingAI.map(statusRecord => statusRecord.sessionId);
// Fetch full session data with messages
const sessionsToProcess = await prisma.session.findMany({
where: { where: {
AND: [ id: { in: sessionIds }
{ messages: { some: {} } }, // Must have messages
{ processed: false }, // Only unprocessed sessions
],
}, },
include: { include: {
messages: { messages: {
orderBy: { order: "asc" }, orderBy: { order: "asc" },
}, },
}, },
}; });
// Add batch size limit if specified
if (batchSize && batchSize > 0) {
queryOptions.take = batchSize;
}
const sessionsToProcess = await prisma.session.findMany(queryOptions);
// Filter to only sessions that have messages // Filter to only sessions that have messages
const sessionsWithMessages = sessionsToProcess.filter( const sessionsWithMessages = sessionsToProcess.filter(
@ -504,7 +534,7 @@ export async function processUnprocessedSessions(batchSize: number | null = null
); );
if (sessionsWithMessages.length === 0) { if (sessionsWithMessages.length === 0) {
process.stdout.write("[ProcessingScheduler] No sessions found requiring processing.\n"); process.stdout.write("[ProcessingScheduler] No sessions with messages found requiring processing.\n");
return; return;
} }