352 lines
9.2 KiB
JavaScript
352 lines
9.2 KiB
JavaScript
require('dotenv').config();
|
|
const express = require('express');
|
|
const cors = require('cors');
|
|
const path = require('path');
|
|
const { PrismaClient } = require('@prisma/client');
|
|
const scraper = require('./scraper');
|
|
|
|
const prisma = new PrismaClient();
|
|
const app = express();
|
|
const PORT = process.env.PORT || 3000;
|
|
|
|
app.use(cors());
|
|
app.use(express.json());
|
|
|
|
// Request logging middleware
|
|
app.use((req, res, next) => {
|
|
console.log(`[HTTP] ${req.method} ${req.url}`);
|
|
next();
|
|
});
|
|
|
|
// Serve static frontend files
|
|
app.use(express.static(path.join(__dirname, '../public')));
|
|
|
|
|
|
// SSE clients for live scraping updates
|
|
let sseClients = [];
|
|
|
|
// Helper to broadcast state to SSE clients
|
|
function broadcastState() {
|
|
const stats = scraper.getStats();
|
|
const data = JSON.stringify(stats);
|
|
sseClients.forEach(client => {
|
|
client.res.write(`data: ${data}\n\n`);
|
|
});
|
|
}
|
|
|
|
// Set up periodic stats broadcast to active frontends
|
|
setInterval(() => {
|
|
if (sseClients.length > 0) {
|
|
broadcastState();
|
|
}
|
|
}, 1000);
|
|
|
|
// API: SSE Stream for Scraper Updates
|
|
app.get('/api/scrape/stream', (req, res) => {
|
|
res.writeHead(200, {
|
|
'Content-Type': 'text/event-stream',
|
|
'Cache-Control': 'no-cache',
|
|
'Connection': 'keep-alive'
|
|
});
|
|
|
|
res.write('\n');
|
|
const clientId = Date.now();
|
|
sseClients.push({ id: clientId, res });
|
|
|
|
// Send initial state
|
|
const stats = scraper.getStats();
|
|
res.write(`data: ${JSON.stringify(stats)}\n\n`);
|
|
|
|
req.on('close', () => {
|
|
sseClients = sseClients.filter(client => client.id !== clientId);
|
|
});
|
|
});
|
|
|
|
// API: Initialize URL database queue from HTML file
|
|
app.post('/api/scrape/initialize', async (req, res) => {
|
|
try {
|
|
// Fire-and-forget initialization in background so API doesn't timeout
|
|
scraper.initializeQueue().then(() => {
|
|
broadcastState();
|
|
});
|
|
|
|
res.json({ success: true, message: 'Queue initialization started in the background.' });
|
|
} catch (error) {
|
|
res.status(500).json({ success: false, error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Get current scraper status and logs
|
|
app.get('/api/stats', async (req, res) => {
|
|
try {
|
|
await scraper.updateCounts();
|
|
const stats = scraper.getStats();
|
|
res.json(stats);
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Start the scraper
|
|
app.post('/api/scrape/start', async (req, res) => {
|
|
const { concurrency, delayMs } = req.body;
|
|
try {
|
|
const threadCount = parseInt(concurrency) || 5;
|
|
const delay = parseInt(delayMs) !== undefined ? parseInt(delayMs) : 500;
|
|
|
|
scraper.startScraping(threadCount, delay);
|
|
broadcastState();
|
|
|
|
res.json({ success: true, message: 'Scraper started successfully.' });
|
|
} catch (error) {
|
|
res.status(500).json({ success: false, error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Pause the scraper
|
|
app.post('/api/scrape/pause', (req, res) => {
|
|
try {
|
|
scraper.pauseScraping();
|
|
broadcastState();
|
|
res.json({ success: true, message: 'Scraper paused.' });
|
|
} catch (error) {
|
|
res.status(500).json({ success: false, error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Reset failed items to retry them
|
|
app.post('/api/scrape/retry', async (req, res) => {
|
|
try {
|
|
const resetCount = await scraper.retryFailed();
|
|
broadcastState();
|
|
res.json({ success: true, message: `Reset ${resetCount} failed items to pending.` });
|
|
} catch (error) {
|
|
res.status(500).json({ success: false, error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Get products (Search & Pagination)
|
|
app.get('/api/products', async (req, res) => {
|
|
try {
|
|
const { q, page = 1, limit = 20, category, company } = req.query;
|
|
|
|
const pageNum = parseInt(page);
|
|
const limitNum = parseInt(limit);
|
|
const offset = (pageNum - 1) * limitNum;
|
|
|
|
// Build Prisma query filter
|
|
const where = {};
|
|
|
|
if (q) {
|
|
where.OR = [
|
|
{ name: { contains: q } },
|
|
{ barcode: { contains: q } },
|
|
{ company: { contains: q } },
|
|
{ category: { contains: q } }
|
|
];
|
|
}
|
|
|
|
if (category) {
|
|
where.category = category;
|
|
}
|
|
|
|
if (company) {
|
|
where.company = company;
|
|
}
|
|
|
|
const [products, total] = await Promise.all([
|
|
prisma.product.findMany({
|
|
where,
|
|
orderBy: { updatedAt: 'desc' },
|
|
skip: offset,
|
|
take: limitNum
|
|
}),
|
|
prisma.product.count({ where })
|
|
]);
|
|
|
|
// Get categories and companies for frontend filters
|
|
// Optimize: run these queries only when page is 1
|
|
let categories = [];
|
|
let companies = [];
|
|
if (pageNum === 1) {
|
|
const catResults = await prisma.product.groupBy({
|
|
by: ['category'],
|
|
_count: { category: true },
|
|
where: { category: { not: null } },
|
|
orderBy: {
|
|
_count: {
|
|
category: 'desc'
|
|
}
|
|
}
|
|
});
|
|
categories = catResults.map(c => ({ name: c.category, count: c._count.category }));
|
|
|
|
const compResults = await prisma.product.groupBy({
|
|
by: ['company'],
|
|
_count: { company: true },
|
|
where: { company: { not: null } },
|
|
orderBy: {
|
|
_count: {
|
|
company: 'desc'
|
|
}
|
|
},
|
|
take: 30 // Top 30 companies
|
|
});
|
|
companies = compResults.map(c => ({ name: c.company, count: c._count.company }));
|
|
}
|
|
|
|
|
|
res.json({
|
|
products,
|
|
pagination: {
|
|
total,
|
|
page: pageNum,
|
|
limit: limitNum,
|
|
totalPages: Math.ceil(total / limitNum)
|
|
},
|
|
filters: {
|
|
categories,
|
|
companies
|
|
}
|
|
});
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Get a single product by ID or Barcode
|
|
app.get('/api/products/:idOrBarcode', async (req, res) => {
|
|
const { idOrBarcode } = req.params;
|
|
try {
|
|
let product = null;
|
|
|
|
if (isNaN(idOrBarcode)) {
|
|
product = await prisma.product.findFirst({
|
|
where: { OR: [{ barcode: idOrBarcode }, { url: idOrBarcode }] }
|
|
});
|
|
} else {
|
|
product = await prisma.product.findUnique({
|
|
where: { id: parseInt(idOrBarcode) }
|
|
});
|
|
|
|
if (!product) {
|
|
product = await prisma.product.findUnique({
|
|
where: { barcode: idOrBarcode }
|
|
});
|
|
}
|
|
}
|
|
|
|
if (!product) {
|
|
return res.status(404).json({ error: 'Product not found.' });
|
|
}
|
|
|
|
res.json(product);
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Add a product manually
|
|
app.post('/api/products', async (req, res) => {
|
|
const { barcode, name, cleanName, amountRaw, amount, unit, image, category, subcategory, company, url } = req.body;
|
|
|
|
if (!barcode || !name) {
|
|
return res.status(400).json({ error: 'Barcode and Name are required fields.' });
|
|
}
|
|
|
|
try {
|
|
const existing = await prisma.product.findUnique({ where: { barcode } });
|
|
if (existing) {
|
|
return res.status(400).json({ error: 'A product with this barcode already exists.' });
|
|
}
|
|
|
|
// Set clean name from utils if not provided
|
|
const parsedName = cleanName ? { cleanName, amountRaw, amount, unit } : require('./utils').parseName(name);
|
|
|
|
const product = await prisma.product.create({
|
|
data: {
|
|
barcode,
|
|
name,
|
|
cleanName: parsedName.cleanName,
|
|
amountRaw: parsedName.amountRaw,
|
|
amount: parsedName.amount,
|
|
unit: parsedName.unit,
|
|
image: image || null,
|
|
category: category || null,
|
|
subcategory: subcategory || null,
|
|
company: company || null,
|
|
url: url || `manual-${Date.now()}`
|
|
}
|
|
});
|
|
|
|
res.status(201).json(product);
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Update a product
|
|
app.put('/api/products/:id', async (req, res) => {
|
|
const { id } = req.params;
|
|
const { name, cleanName, amountRaw, amount, unit, image, category, subcategory, company, barcode } = req.body;
|
|
|
|
try {
|
|
const productId = parseInt(id);
|
|
const existing = await prisma.product.findUnique({ where: { id: productId } });
|
|
if (!existing) {
|
|
return res.status(404).json({ error: 'Product not found.' });
|
|
}
|
|
|
|
const updateData = {
|
|
name,
|
|
cleanName,
|
|
amountRaw,
|
|
amount: amount !== undefined ? (amount ? parseFloat(amount) : null) : undefined,
|
|
unit,
|
|
image,
|
|
category,
|
|
subcategory,
|
|
company,
|
|
barcode
|
|
};
|
|
|
|
// Clean undefined values
|
|
Object.keys(updateData).forEach(key => updateData[key] === undefined && delete updateData[key]);
|
|
|
|
const updated = await prisma.product.update({
|
|
where: { id: productId },
|
|
data: updateData
|
|
});
|
|
|
|
res.json(updated);
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
|
|
// API: Delete a product
|
|
app.delete('/api/products/:id', async (req, res) => {
|
|
const { id } = req.params;
|
|
try {
|
|
const productId = parseInt(id);
|
|
await prisma.product.delete({
|
|
where: { id: productId }
|
|
});
|
|
res.json({ success: true, message: 'Product deleted successfully.' });
|
|
} catch (error) {
|
|
res.status(500).json({ error: error.message });
|
|
}
|
|
});
|
|
|
|
// Start express server
|
|
app.listen(PORT, async () => {
|
|
console.log(`Server running at http://localhost:${PORT}`);
|
|
|
|
// Auto initialize queue in background on startup if empty
|
|
try {
|
|
await scraper.initializeQueue();
|
|
} catch (e) {
|
|
console.error('Auto initialization of queue failed on startup:', e);
|
|
}
|
|
});
|