From 50b230aa9b8e6d6ec28d547b0329daf20de2f380 Mon Sep 17 00:00:00 2001 From: Max Kowalski Date: Fri, 27 Jun 2025 16:55:25 +0200 Subject: [PATCH] feat: Implement configurable scheduler settings and enhance CSV import functionality --- .env.example | 7 + .env.local.example | 22 +++ lib/processingScheduler.ts | 53 +++---- lib/scheduler.ts | 16 ++- lib/schedulerConfig.ts | 82 +++++++++++ package.json | 9 +- pnpm-lock.yaml | 287 +++++++++++++++++++++++++++++++++++++ prisma/seed.ts | 2 +- sample-csv-import-file.csv | 6 + server.ts | 21 ++- 10 files changed, 457 insertions(+), 48 deletions(-) create mode 100644 .env.local.example create mode 100644 lib/schedulerConfig.ts create mode 100644 sample-csv-import-file.csv diff --git a/.env.example b/.env.example index 56ca41f..3b99ece 100644 --- a/.env.example +++ b/.env.example @@ -11,3 +11,10 @@ NODE_ENV=development OPENAI_API_KEY=your_openai_api_key_here # Database connection - already configured in your prisma/schema.prisma + +# Scheduler Configuration +SCHEDULER_ENABLED=false # Enable/disable all schedulers (false for dev, true for production) +CSV_IMPORT_INTERVAL=*/15 * * * * # Cron expression for CSV imports (every 15 minutes) +SESSION_PROCESSING_INTERVAL=0 * * * * # Cron expression for session processing (every hour) +SESSION_PROCESSING_BATCH_SIZE=0 # 0 = unlimited sessions, >0 = specific limit +SESSION_PROCESSING_CONCURRENCY=5 # How many sessions to process in parallel diff --git a/.env.local.example b/.env.local.example new file mode 100644 index 0000000..30fc75c --- /dev/null +++ b/.env.local.example @@ -0,0 +1,22 @@ +# Copy this file to .env.local and configure as needed + +# NextAuth.js configuration +NEXTAUTH_URL=http://localhost:3000 +NEXTAUTH_SECRET=your_secret_key_here +NODE_ENV=development + +# OpenAI API key for session processing +OPENAI_API_KEY=your_openai_api_key_here + +# Scheduler Configuration +SCHEDULER_ENABLED=true # Set to false to disable all schedulers during development +CSV_IMPORT_INTERVAL=*/15 * * * * # Every 15 minutes (cron format) +SESSION_PROCESSING_INTERVAL=0 * * * * # Every hour (cron format) +SESSION_PROCESSING_BATCH_SIZE=0 # 0 = process all sessions, >0 = limit number +SESSION_PROCESSING_CONCURRENCY=5 # Number of sessions to process in parallel + +# Example configurations: +# - For development (no schedulers): SCHEDULER_ENABLED=false +# - For testing (every 5 minutes): CSV_IMPORT_INTERVAL=*/5 * * * * +# - For limited processing: SESSION_PROCESSING_BATCH_SIZE=10 +# - For high concurrency: SESSION_PROCESSING_CONCURRENCY=10 diff --git a/lib/processingScheduler.ts b/lib/processingScheduler.ts index fd4c34f..ef1e24e 100644 --- a/lib/processingScheduler.ts +++ b/lib/processingScheduler.ts @@ -1,32 +1,8 @@ -// Session processing scheduler - TypeScript version +// Session processing scheduler with configurable intervals and batch sizes import cron from "node-cron"; import { PrismaClient } from "@prisma/client"; import fetch from "node-fetch"; -import { readFileSync } from "fs"; -import { fileURLToPath } from "url"; -import { dirname, join } from "path"; - -// Load environment variables from .env.local -const __filename = fileURLToPath(import.meta.url); -const __dirname = dirname(__filename); -const envPath = join(__dirname, '..', '.env.local'); - -try { - const envFile = readFileSync(envPath, 'utf8'); - const envVars = envFile.split('\n').filter(line => line.trim() && !line.startsWith('#')); - - envVars.forEach(line => { - const [key, ...valueParts] = line.split('='); - if (key && valueParts.length > 0) { - const value = valueParts.join('=').trim(); - if (!process.env[key.trim()]) { - process.env[key.trim()] = value; - } - } - }); -} catch (error) { - // Silently fail if .env.local doesn't exist -} +import { getSchedulerConfig } from "./schedulerConfig"; const prisma = new PrismaClient(); const OPENAI_API_KEY = process.env.OPENAI_API_KEY; @@ -399,21 +375,30 @@ export async function processUnprocessedSessions(batchSize: number | null = null } /** - * Start the processing scheduler + * Start the processing scheduler with configurable settings */ export function startProcessingScheduler(): void { - // Process unprocessed sessions every hour - cron.schedule("0 * * * *", async () => { + const config = getSchedulerConfig(); + + if (!config.enabled) { + console.log('[Processing Scheduler] Disabled via configuration'); + return; + } + + console.log(`[Processing Scheduler] Starting with interval: ${config.sessionProcessing.interval}`); + console.log(`[Processing Scheduler] Batch size: ${config.sessionProcessing.batchSize === 0 ? 'unlimited' : config.sessionProcessing.batchSize}`); + console.log(`[Processing Scheduler] Concurrency: ${config.sessionProcessing.concurrency}`); + + cron.schedule(config.sessionProcessing.interval, async () => { try { - await processUnprocessedSessions(); + await processUnprocessedSessions( + config.sessionProcessing.batchSize === 0 ? null : config.sessionProcessing.batchSize, + config.sessionProcessing.concurrency + ); } catch (error) { process.stderr.write( `[ProcessingScheduler] Error in scheduler: ${error}\n` ); } }); - - process.stdout.write( - "[ProcessingScheduler] Started processing scheduler (runs hourly).\n" - ); } diff --git a/lib/scheduler.ts b/lib/scheduler.ts index fcfaa27..da05fe5 100644 --- a/lib/scheduler.ts +++ b/lib/scheduler.ts @@ -1,10 +1,20 @@ -// node-cron job to auto-refresh session data every 15 mins +// CSV import scheduler with configurable intervals import cron from "node-cron"; import { prisma } from "./prisma"; import { fetchAndParseCsv } from "./csvFetcher"; +import { getSchedulerConfig } from "./schedulerConfig"; -export function startScheduler() { - cron.schedule("*/15 * * * *", async () => { +export function startCsvImportScheduler() { + const config = getSchedulerConfig(); + + if (!config.enabled) { + console.log('[CSV Import Scheduler] Disabled via configuration'); + return; + } + + console.log(`[CSV Import Scheduler] Starting with interval: ${config.csvImport.interval}`); + + cron.schedule(config.csvImport.interval, async () => { const companies = await prisma.company.findMany(); for (const company of companies) { try { diff --git a/lib/schedulerConfig.ts b/lib/schedulerConfig.ts new file mode 100644 index 0000000..fbb89e8 --- /dev/null +++ b/lib/schedulerConfig.ts @@ -0,0 +1,82 @@ +// Unified scheduler configuration +import { readFileSync } from "fs"; +import { fileURLToPath } from "url"; +import { dirname, join } from "path"; + +// Load environment variables from .env.local +const __filename = fileURLToPath(import.meta.url); +const __dirname = dirname(__filename); +const envPath = join(__dirname, '..', '.env.local'); + +// Load .env.local if it exists +try { + const envFile = readFileSync(envPath, 'utf8'); + const envVars = envFile.split('\n').filter(line => line.trim() && !line.startsWith('#')); + + envVars.forEach(line => { + const [key, ...valueParts] = line.split('='); + if (key && valueParts.length > 0) { + const value = valueParts.join('=').trim(); + if (!process.env[key.trim()]) { + process.env[key.trim()] = value; + } + } + }); +} catch (error) { + // Silently fail if .env.local doesn't exist +} + +export interface SchedulerConfig { + enabled: boolean; + csvImport: { + interval: string; + }; + sessionProcessing: { + interval: string; + batchSize: number; // 0 = unlimited + concurrency: number; + }; +} + +/** + * Get scheduler configuration from environment variables + */ +export function getSchedulerConfig(): SchedulerConfig { + const enabled = process.env.SCHEDULER_ENABLED === 'true'; + + // Default values + const defaults = { + csvImportInterval: '*/15 * * * *', // Every 15 minutes + sessionProcessingInterval: '0 * * * *', // Every hour + sessionProcessingBatchSize: 0, // Unlimited + sessionProcessingConcurrency: 5, + }; + + return { + enabled, + csvImport: { + interval: process.env.CSV_IMPORT_INTERVAL || defaults.csvImportInterval, + }, + sessionProcessing: { + interval: process.env.SESSION_PROCESSING_INTERVAL || defaults.sessionProcessingInterval, + batchSize: parseInt(process.env.SESSION_PROCESSING_BATCH_SIZE || '0', 10) || defaults.sessionProcessingBatchSize, + concurrency: parseInt(process.env.SESSION_PROCESSING_CONCURRENCY || '5', 10) || defaults.sessionProcessingConcurrency, + }, + }; +} + +/** + * Log scheduler configuration + */ +export function logSchedulerConfig(config: SchedulerConfig): void { + if (!config.enabled) { + console.log('[Scheduler] Schedulers are DISABLED (SCHEDULER_ENABLED=false)'); + return; + } + + console.log('[Scheduler] Configuration:'); + console.log(` CSV Import: ${config.csvImport.interval}`); + console.log(` Session Processing: ${config.sessionProcessing.interval}`); + console.log(` Batch Size: ${config.sessionProcessing.batchSize === 0 ? 'unlimited' : config.sessionProcessing.batchSize}`); + console.log(` Concurrency: ${config.sessionProcessing.concurrency}`); +} diff --git a/package.json b/package.json index 66efff1..2d2ccb5 100644 --- a/package.json +++ b/package.json @@ -5,15 +5,15 @@ "private": true, "scripts": { "build": "next build", - "dev": "next dev --turbopack", - "dev:with-schedulers": "node server.mjs", + "dev": "tsx server.ts", + "dev:next-only": "next dev --turbopack", "format": "npx prettier --write .", "format:check": "npx prettier --check .", "lint": "next lint", "lint:fix": "npx eslint --fix", "prisma:generate": "prisma generate", "prisma:migrate": "prisma migrate dev", - "prisma:seed": "node prisma/seed.mjs", + "prisma:seed": "tsx prisma/seed.ts", "prisma:push": "prisma db push", "prisma:push:force": "prisma db push --force-reset", "prisma:studio": "prisma studio", @@ -26,6 +26,7 @@ "@rapideditor/country-coder": "^5.4.0", "@types/d3": "^7.4.3", "@types/d3-cloud": "^1.2.9", + "@types/d3-selection": "^3.0.11", "@types/geojson": "^7946.0.16", "@types/leaflet": "^1.9.18", "@types/node-fetch": "^2.6.12", @@ -35,6 +36,7 @@ "csv-parse": "^5.5.0", "d3": "^7.9.0", "d3-cloud": "^1.2.7", + "d3-selection": "^3.0.0", "i18n-iso-countries": "^7.14.0", "iso-639-1": "^3.1.5", "leaflet": "^1.9.4", @@ -71,6 +73,7 @@ "prisma": "^6.10.1", "tailwindcss": "^4.1.7", "ts-node": "^10.9.2", + "tsx": "^4.20.3", "typescript": "^5.0.0" }, "prettier": { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2eee81b..c0b507f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,9 @@ importers: '@types/d3-cloud': specifier: ^1.2.9 version: 1.2.9 + '@types/d3-selection': + specifier: ^3.0.11 + version: 3.0.11 '@types/geojson': specifier: ^7946.0.16 version: 7946.0.16 @@ -47,6 +50,9 @@ importers: d3-cloud: specifier: ^1.2.7 version: 1.2.7 + d3-selection: + specifier: ^3.0.0 + version: 3.0.0 i18n-iso-countries: specifier: ^7.14.0 version: 7.14.0 @@ -150,6 +156,9 @@ importers: ts-node: specifier: ^10.9.2 version: 10.9.2(@types/node@22.15.21)(typescript@5.8.3) + tsx: + specifier: ^4.20.3 + version: 4.20.3 typescript: specifier: ^5.0.0 version: 5.8.3 @@ -181,6 +190,156 @@ packages: '@emnapi/wasi-threads@1.0.2': resolution: {integrity: sha512-5n3nTJblwRi8LlXkJ9eBzu+kZR8Yxcc7ubakyQTFzPMtIhFpUBRbsnc2Dv88IZDIbCDlBiWrknhB4Lsz7mg6BA==} + '@esbuild/aix-ppc64@0.25.5': + resolution: {integrity: sha512-9o3TMmpmftaCMepOdA5k/yDw8SfInyzWWTjYTFCX3kPSDJMROQTb8jg+h9Cnwnmm1vOzvxN7gIfB5V2ewpjtGA==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [aix] + + '@esbuild/android-arm64@0.25.5': + resolution: {integrity: sha512-VGzGhj4lJO+TVGV1v8ntCZWJktV7SGCs3Pn1GRWI1SBFtRALoomm8k5E9Pmwg3HOAal2VDc2F9+PM/rEY6oIDg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [android] + + '@esbuild/android-arm@0.25.5': + resolution: {integrity: sha512-AdJKSPeEHgi7/ZhuIPtcQKr5RQdo6OO2IL87JkianiMYMPbCtot9fxPbrMiBADOWWm3T2si9stAiVsGbTQFkbA==} + engines: {node: '>=18'} + cpu: [arm] + os: [android] + + '@esbuild/android-x64@0.25.5': + resolution: {integrity: sha512-D2GyJT1kjvO//drbRT3Hib9XPwQeWd9vZoBJn+bu/lVsOZ13cqNdDeqIF/xQ5/VmWvMduP6AmXvylO/PIc2isw==} + engines: {node: '>=18'} + cpu: [x64] + os: [android] + + '@esbuild/darwin-arm64@0.25.5': + resolution: {integrity: sha512-GtaBgammVvdF7aPIgH2jxMDdivezgFu6iKpmT+48+F8Hhg5J/sfnDieg0aeG/jfSvkYQU2/pceFPDKlqZzwnfQ==} + engines: {node: '>=18'} + cpu: [arm64] + os: [darwin] + + '@esbuild/darwin-x64@0.25.5': + resolution: {integrity: sha512-1iT4FVL0dJ76/q1wd7XDsXrSW+oLoquptvh4CLR4kITDtqi2e/xwXwdCVH8hVHU43wgJdsq7Gxuzcs6Iq/7bxQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [darwin] + + '@esbuild/freebsd-arm64@0.25.5': + resolution: {integrity: sha512-nk4tGP3JThz4La38Uy/gzyXtpkPW8zSAmoUhK9xKKXdBCzKODMc2adkB2+8om9BDYugz+uGV7sLmpTYzvmz6Sw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [freebsd] + + '@esbuild/freebsd-x64@0.25.5': + resolution: {integrity: sha512-PrikaNjiXdR2laW6OIjlbeuCPrPaAl0IwPIaRv+SMV8CiM8i2LqVUHFC1+8eORgWyY7yhQY+2U2fA55mBzReaw==} + engines: {node: '>=18'} + cpu: [x64] + os: [freebsd] + + '@esbuild/linux-arm64@0.25.5': + resolution: {integrity: sha512-Z9kfb1v6ZlGbWj8EJk9T6czVEjjq2ntSYLY2cw6pAZl4oKtfgQuS4HOq41M/BcoLPzrUbNd+R4BXFyH//nHxVg==} + engines: {node: '>=18'} + cpu: [arm64] + os: [linux] + + '@esbuild/linux-arm@0.25.5': + resolution: {integrity: sha512-cPzojwW2okgh7ZlRpcBEtsX7WBuqbLrNXqLU89GxWbNt6uIg78ET82qifUy3W6OVww6ZWobWub5oqZOVtwolfw==} + engines: {node: '>=18'} + cpu: [arm] + os: [linux] + + '@esbuild/linux-ia32@0.25.5': + resolution: {integrity: sha512-sQ7l00M8bSv36GLV95BVAdhJ2QsIbCuCjh/uYrWiMQSUuV+LpXwIqhgJDcvMTj+VsQmqAHL2yYaasENvJ7CDKA==} + engines: {node: '>=18'} + cpu: [ia32] + os: [linux] + + '@esbuild/linux-loong64@0.25.5': + resolution: {integrity: sha512-0ur7ae16hDUC4OL5iEnDb0tZHDxYmuQyhKhsPBV8f99f6Z9KQM02g33f93rNH5A30agMS46u2HP6qTdEt6Q1kg==} + engines: {node: '>=18'} + cpu: [loong64] + os: [linux] + + '@esbuild/linux-mips64el@0.25.5': + resolution: {integrity: sha512-kB/66P1OsHO5zLz0i6X0RxlQ+3cu0mkxS3TKFvkb5lin6uwZ/ttOkP3Z8lfR9mJOBk14ZwZ9182SIIWFGNmqmg==} + engines: {node: '>=18'} + cpu: [mips64el] + os: [linux] + + '@esbuild/linux-ppc64@0.25.5': + resolution: {integrity: sha512-UZCmJ7r9X2fe2D6jBmkLBMQetXPXIsZjQJCjgwpVDz+YMcS6oFR27alkgGv3Oqkv07bxdvw7fyB71/olceJhkQ==} + engines: {node: '>=18'} + cpu: [ppc64] + os: [linux] + + '@esbuild/linux-riscv64@0.25.5': + resolution: {integrity: sha512-kTxwu4mLyeOlsVIFPfQo+fQJAV9mh24xL+y+Bm6ej067sYANjyEw1dNHmvoqxJUCMnkBdKpvOn0Ahql6+4VyeA==} + engines: {node: '>=18'} + cpu: [riscv64] + os: [linux] + + '@esbuild/linux-s390x@0.25.5': + resolution: {integrity: sha512-K2dSKTKfmdh78uJ3NcWFiqyRrimfdinS5ErLSn3vluHNeHVnBAFWC8a4X5N+7FgVE1EjXS1QDZbpqZBjfrqMTQ==} + engines: {node: '>=18'} + cpu: [s390x] + os: [linux] + + '@esbuild/linux-x64@0.25.5': + resolution: {integrity: sha512-uhj8N2obKTE6pSZ+aMUbqq+1nXxNjZIIjCjGLfsWvVpy7gKCOL6rsY1MhRh9zLtUtAI7vpgLMK6DxjO8Qm9lJw==} + engines: {node: '>=18'} + cpu: [x64] + os: [linux] + + '@esbuild/netbsd-arm64@0.25.5': + resolution: {integrity: sha512-pwHtMP9viAy1oHPvgxtOv+OkduK5ugofNTVDilIzBLpoWAM16r7b/mxBvfpuQDpRQFMfuVr5aLcn4yveGvBZvw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [netbsd] + + '@esbuild/netbsd-x64@0.25.5': + resolution: {integrity: sha512-WOb5fKrvVTRMfWFNCroYWWklbnXH0Q5rZppjq0vQIdlsQKuw6mdSihwSo4RV/YdQ5UCKKvBy7/0ZZYLBZKIbwQ==} + engines: {node: '>=18'} + cpu: [x64] + os: [netbsd] + + '@esbuild/openbsd-arm64@0.25.5': + resolution: {integrity: sha512-7A208+uQKgTxHd0G0uqZO8UjK2R0DDb4fDmERtARjSHWxqMTye4Erz4zZafx7Di9Cv+lNHYuncAkiGFySoD+Mw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openbsd] + + '@esbuild/openbsd-x64@0.25.5': + resolution: {integrity: sha512-G4hE405ErTWraiZ8UiSoesH8DaCsMm0Cay4fsFWOOUcz8b8rC6uCvnagr+gnioEjWn0wC+o1/TAHt+It+MpIMg==} + engines: {node: '>=18'} + cpu: [x64] + os: [openbsd] + + '@esbuild/sunos-x64@0.25.5': + resolution: {integrity: sha512-l+azKShMy7FxzY0Rj4RCt5VD/q8mG/e+mDivgspo+yL8zW7qEwctQ6YqKX34DTEleFAvCIUviCFX1SDZRSyMQA==} + engines: {node: '>=18'} + cpu: [x64] + os: [sunos] + + '@esbuild/win32-arm64@0.25.5': + resolution: {integrity: sha512-O2S7SNZzdcFG7eFKgvwUEZ2VG9D/sn/eIiz8XRZ1Q/DO5a3s76Xv0mdBzVM5j5R639lXQmPmSo0iRpHqUUrsxw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [win32] + + '@esbuild/win32-ia32@0.25.5': + resolution: {integrity: sha512-onOJ02pqs9h1iMJ1PQphR+VZv8qBMQ77Klcsqv9CNW2w6yLqoURLcgERAIurY6QE63bbLuqgP9ATqajFLK5AMQ==} + engines: {node: '>=18'} + cpu: [ia32] + os: [win32] + + '@esbuild/win32-x64@0.25.5': + resolution: {integrity: sha512-TXv6YnJ8ZMVdX+SXWVBo/0p8LTcrUYngpWjvm91TMjjBQii7Oz11Lw5lbDV5Y0TzuhSJHwiH4hEtC1I42mMS0g==} + engines: {node: '>=18'} + cpu: [x64] + os: [win32] + '@eslint-community/eslint-utils@4.7.0': resolution: {integrity: sha512-dyybb3AcajC7uha6CvhdVRJqaKyn7w2YKqKyAN37NKYgZT36w+iRb0Dymmc5qEJ549c/S31cMMSFd75bteCpCw==} engines: {node: ^12.22.0 || ^14.17.0 || >=16.0.0} @@ -1396,6 +1555,11 @@ packages: resolution: {integrity: sha512-w+5mJ3GuFL+NjVtJlvydShqE1eN3h3PbI7/5LAsYJP/2qtuMXjfL2LpHSRqo4b4eSF5K/DH1JXKUAHSB2UW50g==} engines: {node: '>= 0.4'} + esbuild@0.25.5: + resolution: {integrity: sha512-P8OtKZRv/5J5hhz0cUAdu/cLuPIKXpQl1R9pZtvmHWQvrAUVd0UNIPT4IB4W3rNOqVO0rlqHmCIbSwxh/c9yUQ==} + engines: {node: '>=18'} + hasBin: true + escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} @@ -1607,6 +1771,11 @@ packages: engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + fsevents@2.3.3: + resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} + engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} + os: [darwin] + function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} @@ -2728,6 +2897,11 @@ packages: tslib@2.8.1: resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsx@4.20.3: + resolution: {integrity: sha512-qjbnuR9Tr+FJOMBqJCW5ehvIo/buZq7vH7qD7JziU98h6l3qGy0a/yPFjwO+y0/T7GFpNgNAvEcPPVfyT8rrPQ==} + engines: {node: '>=18.0.0'} + hasBin: true + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -2891,6 +3065,81 @@ snapshots: tslib: 2.8.1 optional: true + '@esbuild/aix-ppc64@0.25.5': + optional: true + + '@esbuild/android-arm64@0.25.5': + optional: true + + '@esbuild/android-arm@0.25.5': + optional: true + + '@esbuild/android-x64@0.25.5': + optional: true + + '@esbuild/darwin-arm64@0.25.5': + optional: true + + '@esbuild/darwin-x64@0.25.5': + optional: true + + '@esbuild/freebsd-arm64@0.25.5': + optional: true + + '@esbuild/freebsd-x64@0.25.5': + optional: true + + '@esbuild/linux-arm64@0.25.5': + optional: true + + '@esbuild/linux-arm@0.25.5': + optional: true + + '@esbuild/linux-ia32@0.25.5': + optional: true + + '@esbuild/linux-loong64@0.25.5': + optional: true + + '@esbuild/linux-mips64el@0.25.5': + optional: true + + '@esbuild/linux-ppc64@0.25.5': + optional: true + + '@esbuild/linux-riscv64@0.25.5': + optional: true + + '@esbuild/linux-s390x@0.25.5': + optional: true + + '@esbuild/linux-x64@0.25.5': + optional: true + + '@esbuild/netbsd-arm64@0.25.5': + optional: true + + '@esbuild/netbsd-x64@0.25.5': + optional: true + + '@esbuild/openbsd-arm64@0.25.5': + optional: true + + '@esbuild/openbsd-x64@0.25.5': + optional: true + + '@esbuild/sunos-x64@0.25.5': + optional: true + + '@esbuild/win32-arm64@0.25.5': + optional: true + + '@esbuild/win32-ia32@0.25.5': + optional: true + + '@esbuild/win32-x64@0.25.5': + optional: true + '@eslint-community/eslint-utils@4.7.0(eslint@9.27.0(jiti@2.4.2))': dependencies: eslint: 9.27.0(jiti@2.4.2) @@ -4129,6 +4378,34 @@ snapshots: is-date-object: 1.1.0 is-symbol: 1.1.1 + esbuild@0.25.5: + optionalDependencies: + '@esbuild/aix-ppc64': 0.25.5 + '@esbuild/android-arm': 0.25.5 + '@esbuild/android-arm64': 0.25.5 + '@esbuild/android-x64': 0.25.5 + '@esbuild/darwin-arm64': 0.25.5 + '@esbuild/darwin-x64': 0.25.5 + '@esbuild/freebsd-arm64': 0.25.5 + '@esbuild/freebsd-x64': 0.25.5 + '@esbuild/linux-arm': 0.25.5 + '@esbuild/linux-arm64': 0.25.5 + '@esbuild/linux-ia32': 0.25.5 + '@esbuild/linux-loong64': 0.25.5 + '@esbuild/linux-mips64el': 0.25.5 + '@esbuild/linux-ppc64': 0.25.5 + '@esbuild/linux-riscv64': 0.25.5 + '@esbuild/linux-s390x': 0.25.5 + '@esbuild/linux-x64': 0.25.5 + '@esbuild/netbsd-arm64': 0.25.5 + '@esbuild/netbsd-x64': 0.25.5 + '@esbuild/openbsd-arm64': 0.25.5 + '@esbuild/openbsd-x64': 0.25.5 + '@esbuild/sunos-x64': 0.25.5 + '@esbuild/win32-arm64': 0.25.5 + '@esbuild/win32-ia32': 0.25.5 + '@esbuild/win32-x64': 0.25.5 + escape-string-regexp@4.0.0: {} eslint-config-next@15.3.2(eslint@9.27.0(jiti@2.4.2))(typescript@5.8.3): @@ -4414,6 +4691,9 @@ snapshots: fsevents@2.3.2: optional: true + fsevents@2.3.3: + optional: true + function-bind@1.1.2: {} function.prototype.name@1.1.8: @@ -5832,6 +6112,13 @@ snapshots: tslib@2.8.1: {} + tsx@4.20.3: + dependencies: + esbuild: 0.25.5 + get-tsconfig: 4.10.1 + optionalDependencies: + fsevents: 2.3.3 + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 diff --git a/prisma/seed.ts b/prisma/seed.ts index 971f8d5..c7557f5 100644 --- a/prisma/seed.ts +++ b/prisma/seed.ts @@ -10,7 +10,7 @@ async function main() { const company = await prisma.company.create({ data: { name: "Demo Company", - csvUrl: "https://example.com/data.csv", // Replace with a real URL if available + csvUrl: "https://proto.notso.ai/jumbo/chats", // Replace with a real URL if available }, }); diff --git a/sample-csv-import-file.csv b/sample-csv-import-file.csv new file mode 100644 index 0000000..1f9d787 --- /dev/null +++ b/sample-csv-import-file.csv @@ -0,0 +1,6 @@ +a068d62a-439b-4d70-924f-3d45ffba673b,30.04.2025 11:38:14,30.04.2025 11:38:14,31.176.221.57,BA,,1,,,,https://proto.notso.ai/jumbo/chats/a068d62a-439b-4d70-924f-3d45ffba673b.txt,2.051,6470,0.0009,,test +284c6849-51ba-41b8-8afd-c1a70e7bd997,30.04.2025 11:41:48,30.04.2025 11:41:48,31.176.221.57,BA,english,1,happy,no,no,https://proto.notso.ai/jumbo/chats/284c6849-51ba-41b8-8afd-c1a70e7bd997.txt,3.977,6537,0.0010,Greeting,Good day +ef6b43f6-e46f-4d6c-9bf7-3d8f3b658d40,01.05.2025 12:11:18,01.05.2025 12:14:53,31.176.221.57,BA,Dutch,8,excited,no,no,https://proto.notso.ai/jumbo/chats/ef6b43f6-e46f-4d6c-9bf7-3d8f3b658d40.txt,3.458,56027,0.0083,Onboarding,whats up +e5c6d4d1-7a02-4c0e-9d93-214ea06d6764,01.05.2025 12:37:43,01.05.2025 12:37:43,31.176.221.57,BA,turkish,1,happy,no,no,https://proto.notso.ai/jumbo/chats/e5c6d4d1-7a02-4c0e-9d93-214ea06d6764.txt,3.004,6549,0.0010,Language inquiry,Spreek je ook turks? +461086bd-bac0-496b-a541-d76468b96f44,01.05.2025 12:48:13,01.05.2025 12:48:21,31.176.221.57,BA,dutch,2,happy,no,no,https://proto.notso.ai/jumbo/chats/461086bd-bac0-496b-a541-d76468b96f44.txt,2.442,13220,0.0020,General,Lalalaposie +689ae197-2005-4f09-b993-d9f6fa16fc1f,01.05.2025 12:52:07,01.05.2025 12:57:14,31.176.221.57,BA,,3,,,,https://proto.notso.ai/jumbo/chats/689ae197-2005-4f09-b993-d9f6fa16fc1f.txt,1.487,19751,0.0029,,hi liza diff --git a/server.ts b/server.ts index ce8d58b..3dd6bd9 100644 --- a/server.ts +++ b/server.ts @@ -1,9 +1,10 @@ -// Custom Next.js server with scheduler initialization +// Custom Next.js server with configurable scheduler initialization import { createServer } from "http"; import { parse } from "url"; import next from "next"; -import { startScheduler } from "./lib/scheduler.js"; +import { startCsvImportScheduler } from "./lib/scheduler.js"; import { startProcessingScheduler } from "./lib/processingScheduler.js"; +import { getSchedulerConfig, logSchedulerConfig } from "./lib/schedulerConfig.js"; const dev = process.env.NODE_ENV !== "production"; const hostname = "localhost"; @@ -14,11 +15,17 @@ const app = next({ dev, hostname, port }); const handle = app.getRequestHandler(); app.prepare().then(() => { - // Initialize schedulers when the server starts - console.log("Starting schedulers..."); - startScheduler(); - startProcessingScheduler(); - console.log("All schedulers initialized successfully"); + // Get and log scheduler configuration + const config = getSchedulerConfig(); + logSchedulerConfig(config); + + // Initialize schedulers based on configuration + if (config.enabled) { + console.log("Initializing schedulers..."); + startCsvImportScheduler(); + startProcessingScheduler(); + console.log("All schedulers initialized successfully"); + } createServer(async (req, res) => { try {