first commit
This commit is contained in:
hOLOlu
446
src/scraper.js
Normal file
446
src/scraper.js
Normal file
@@ -0,0 +1,446 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const readline = require('readline');
|
||||
const axios = require('axios');
|
||||
const cheerio = require('cheerio');
|
||||
const { PrismaClient } = require('@prisma/client');
|
||||
const { parseName, sleep } = require('./utils');
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
|
||||
// Scraper state
|
||||
const state = {
|
||||
isRunning: false,
|
||||
concurrency: 5,
|
||||
delayMs: 500,
|
||||
totalUrls: 0,
|
||||
completedCount: 0,
|
||||
failedCount: 0,
|
||||
pendingCount: 0,
|
||||
logs: [],
|
||||
activeWorkers: 0
|
||||
};
|
||||
|
||||
// Log helper
|
||||
function logMsg(message, type = 'info') {
|
||||
const logEntry = {
|
||||
timestamp: new Date().toISOString(),
|
||||
message,
|
||||
type // info, success, warning, error
|
||||
};
|
||||
state.logs.push(logEntry);
|
||||
if (state.logs.length > 300) {
|
||||
state.logs.shift(); // Keep last 300 logs
|
||||
}
|
||||
console.log(`[${type.toUpperCase()}] ${message}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the database by importing URLs from adresler.html if the queue is empty.
|
||||
*/
|
||||
async function initializeQueue() {
|
||||
try {
|
||||
const count = await prisma.scrapeQueue.count();
|
||||
logMsg(`Current database queue has ${count} items. Syncing with entries in adresler.html...`, 'info');
|
||||
|
||||
|
||||
logMsg('Initializing queue from adresler.html... This may take a moment.', 'info');
|
||||
const filePath = path.join(__dirname, '../adresler.html');
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
logMsg('adresler.html file not found in the root directory!', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
const fileStream = fs.createReadStream(filePath);
|
||||
const rl = readline.createInterface({
|
||||
input: fileStream,
|
||||
crlfDelay: Infinity
|
||||
});
|
||||
|
||||
const urlRegex = /href="(https:\/\/barkodist\.com\/[^"]+)"/;
|
||||
let urls = [];
|
||||
let lineCount = 0;
|
||||
let importedCount = 0;
|
||||
|
||||
for await (const line of rl) {
|
||||
lineCount++;
|
||||
const match = line.match(urlRegex);
|
||||
if (match) {
|
||||
const url = match[1];
|
||||
urls.push({ url, status: 'pending' });
|
||||
|
||||
// Insert in batches of 5000 for efficiency
|
||||
if (urls.length >= 5000) {
|
||||
await prisma.scrapeQueue.createMany({
|
||||
data: urls,
|
||||
skipDuplicates: true
|
||||
});
|
||||
importedCount += urls.length;
|
||||
logMsg(`Imported ${importedCount} URLs...`, 'info');
|
||||
urls = [];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Insert remaining URLs
|
||||
if (urls.length > 0) {
|
||||
await prisma.scrapeQueue.createMany({
|
||||
data: urls,
|
||||
skipDuplicates: true
|
||||
});
|
||||
importedCount += urls.length;
|
||||
}
|
||||
|
||||
logMsg(`Queue initialization complete. Total imported: ${importedCount} URLs.`, 'success');
|
||||
state.totalUrls = importedCount;
|
||||
await updateCounts();
|
||||
} catch (error) {
|
||||
logMsg(`Queue initialization failed: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates completed, failed, and pending counts in state.
|
||||
*/
|
||||
async function updateCounts() {
|
||||
try {
|
||||
state.totalUrls = await prisma.scrapeQueue.count();
|
||||
state.completedCount = await prisma.scrapeQueue.count({ where: { status: 'completed' } });
|
||||
state.failedCount = await prisma.scrapeQueue.count({ where: { status: 'failed' } });
|
||||
state.pendingCount = await prisma.scrapeQueue.count({ where: { status: 'pending' } });
|
||||
} catch (error) {
|
||||
console.error('Failed to update counts:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrapes a single product URL.
|
||||
*/
|
||||
async function scrapeUrl(queueItem) {
|
||||
const { id, url } = queueItem;
|
||||
|
||||
try {
|
||||
// Fetch page content
|
||||
const response = await axios.get(url, {
|
||||
timeout: 10000,
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
|
||||
'Accept-Language': 'tr-TR,tr;q=0.9,en-US;q=0.8,en;q=0.7'
|
||||
}
|
||||
});
|
||||
|
||||
const html = response.data;
|
||||
const $ = cheerio.load(html);
|
||||
|
||||
// 1. Barcode (very critical)
|
||||
let barcode = null;
|
||||
|
||||
// Attempt 1: From JS script JsBarcode call
|
||||
const jsBarcodeMatch = html.match(/JsBarcode\("#barcodeProduct",\s*"(\d+)"\)/);
|
||||
if (jsBarcodeMatch && jsBarcodeMatch[1]) {
|
||||
barcode = jsBarcodeMatch[1].trim();
|
||||
}
|
||||
|
||||
// Attempt 2: From hidden inputs (HataBildirModal form)
|
||||
if (!barcode) {
|
||||
const hiddenBarcodeMatch = html.match(/name="HataBarkod"\s+value="(\d+)"/) || html.match(/value="(\d+)"\s+name="HataBarkod"/);
|
||||
if (hiddenBarcodeMatch && hiddenBarcodeMatch[1]) {
|
||||
barcode = hiddenBarcodeMatch[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt 3: From paragraph strong tags containing barcode numbers
|
||||
if (!barcode) {
|
||||
const pText = $('.BarcodeGeneratorInfo p').text() || $('p').text();
|
||||
const textBarcodeMatch = pText.match(/barkod numarası\s*<strong>(\d+)<\/strong>/i) || pText.match(/barkod numarası\s+(\d+)/i);
|
||||
if (textBarcodeMatch && textBarcodeMatch[1]) {
|
||||
barcode = textBarcodeMatch[1].trim();
|
||||
}
|
||||
}
|
||||
|
||||
if (!barcode) {
|
||||
throw new Error('Barcode number could not be found on the page.');
|
||||
}
|
||||
|
||||
// 2. Product Name
|
||||
const rawName = $('h1.barkodTitle').text().trim() || $('title').text().replace(' | Barkodist', '').replace(' Barkodu', '').trim();
|
||||
if (!rawName) {
|
||||
throw new Error('Product name could not be found.');
|
||||
}
|
||||
|
||||
// Parse clean name, amount, unit
|
||||
const parsed = parseName(rawName);
|
||||
|
||||
// 3. Product Image
|
||||
let image = null;
|
||||
const imgEl = $('.BarkodPageContainer img').first();
|
||||
if (imgEl.length) {
|
||||
const src = imgEl.attr('src');
|
||||
if (src) {
|
||||
// Clean double slashes e.g. barkodist.com//assets -> barkodist.com/assets
|
||||
image = src.replace(/([^:]\/)\/+/g, "$1");
|
||||
if (!image.startsWith('http')) {
|
||||
image = 'https://barkodist.com' + (image.startsWith('/') ? '' : '/') + image;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 4. Category and Subcategory
|
||||
let category = null;
|
||||
let subcategory = null;
|
||||
|
||||
// From page description paragraph
|
||||
// Example: "Gıda Ürünleri kategorisi ve Makarna alt kategorisinde yer alan..."
|
||||
const infoText = $('.BarcodeGeneratorInfo').text() || $('body').text();
|
||||
const catMatch = infoText.match(/([a-zA-ZçıöşğüÇİÖŞĞÜ\s]+)\s+kategorisi\s+ve\s+([a-zA-ZçıöşğüÇİÖŞĞÜ\s]+)\s+alt\s+kategorisinde/i);
|
||||
if (catMatch) {
|
||||
category = catMatch[1].trim();
|
||||
subcategory = catMatch[2].trim();
|
||||
}
|
||||
|
||||
// 5. Company (Owner Company)
|
||||
let company = null;
|
||||
const companyMatch = html.match(/<b>(.*?)<\/b>\s*şirketine aittir/i) || html.match(/<b>(.*?)<\/b>\s*firmasına aittir/i);
|
||||
if (companyMatch && companyMatch[1]) {
|
||||
company = companyMatch[1].trim();
|
||||
}
|
||||
|
||||
// Save product to database
|
||||
await prisma.product.upsert({
|
||||
where: { barcode },
|
||||
update: {
|
||||
name: rawName,
|
||||
cleanName: parsed.cleanName,
|
||||
amountRaw: parsed.amountRaw,
|
||||
amount: parsed.amount,
|
||||
unit: parsed.unit,
|
||||
image,
|
||||
category,
|
||||
subcategory,
|
||||
company,
|
||||
url,
|
||||
updatedAt: new Date()
|
||||
},
|
||||
create: {
|
||||
barcode,
|
||||
name: rawName,
|
||||
cleanName: parsed.cleanName,
|
||||
amountRaw: parsed.amountRaw,
|
||||
amount: parsed.amount,
|
||||
unit: parsed.unit,
|
||||
image,
|
||||
category,
|
||||
subcategory,
|
||||
company,
|
||||
url
|
||||
}
|
||||
});
|
||||
|
||||
// Update queue status
|
||||
await prisma.scrapeQueue.update({
|
||||
where: { id },
|
||||
data: {
|
||||
status: 'completed',
|
||||
error: null
|
||||
}
|
||||
});
|
||||
|
||||
logMsg(`Successfully scraped: "${parsed.cleanName}" (${barcode})`, 'success');
|
||||
} catch (error) {
|
||||
const errorMsg = error.response ? `HTTP ${error.response.status}` : error.message;
|
||||
logMsg(`Failed to scrape ${url}: ${errorMsg}`, 'error');
|
||||
|
||||
await prisma.scrapeQueue.update({
|
||||
where: { id },
|
||||
data: {
|
||||
status: 'failed',
|
||||
error: errorMsg,
|
||||
retryCount: { increment: 1 }
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Central queue manager in memory to avoid MySQL deadlock/transaction errors
|
||||
let queueBuffer = [];
|
||||
let isFetchingQueue = false;
|
||||
|
||||
async function getNextQueueItem() {
|
||||
if (queueBuffer.length > 0) {
|
||||
return queueBuffer.shift();
|
||||
}
|
||||
|
||||
if (isFetchingQueue) {
|
||||
await sleep(100);
|
||||
return getNextQueueItem();
|
||||
}
|
||||
|
||||
isFetchingQueue = true;
|
||||
try {
|
||||
const items = await prisma.scrapeQueue.findMany({
|
||||
where: { status: 'pending' },
|
||||
orderBy: { id: 'asc' },
|
||||
take: 20
|
||||
});
|
||||
|
||||
if (items.length === 0) {
|
||||
isFetchingQueue = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
const ids = items.map(item => item.id);
|
||||
await prisma.scrapeQueue.updateMany({
|
||||
where: { id: { in: ids } },
|
||||
data: { status: 'processing' }
|
||||
});
|
||||
|
||||
queueBuffer = items;
|
||||
} catch (error) {
|
||||
console.error('Error fetching queue batch:', error);
|
||||
} finally {
|
||||
isFetchingQueue = false;
|
||||
}
|
||||
|
||||
return queueBuffer.shift() || null;
|
||||
}
|
||||
|
||||
async function resetQueueBuffer() {
|
||||
if (queueBuffer.length > 0) {
|
||||
try {
|
||||
const ids = queueBuffer.map(item => item.id);
|
||||
queueBuffer = [];
|
||||
await prisma.scrapeQueue.updateMany({
|
||||
where: { id: { in: ids } },
|
||||
data: { status: 'pending' }
|
||||
});
|
||||
console.log(`Reset ${ids.length} buffered items back to pending.`);
|
||||
} catch (error) {
|
||||
console.error('Error resetting queue buffer:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Worker loop that grabs a pending item and scrapes it.
|
||||
*/
|
||||
async function scraperWorker() {
|
||||
state.activeWorkers++;
|
||||
while (state.isRunning) {
|
||||
let queueItem = null;
|
||||
|
||||
try {
|
||||
queueItem = await getNextQueueItem();
|
||||
|
||||
if (!queueItem) {
|
||||
// No more pending items
|
||||
break;
|
||||
}
|
||||
|
||||
await scrapeUrl(queueItem);
|
||||
await updateCounts();
|
||||
|
||||
// Be nice: wait after scraping
|
||||
if (state.delayMs > 0) {
|
||||
await sleep(state.delayMs);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Worker error:', error);
|
||||
if (queueItem) {
|
||||
// Reset status to pending so it can be retried
|
||||
await prisma.scrapeQueue.update({
|
||||
where: { id: queueItem.id },
|
||||
data: { status: 'pending' }
|
||||
}).catch(() => {});
|
||||
}
|
||||
await sleep(2000);
|
||||
}
|
||||
}
|
||||
state.activeWorkers--;
|
||||
|
||||
if (state.activeWorkers === 0 && state.isRunning) {
|
||||
state.isRunning = false;
|
||||
logMsg('Scraping job completed or queue empty.', 'success');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Starts scraping with the specified settings.
|
||||
*/
|
||||
async function startScraping(concurrency = 5, delayMs = 500) {
|
||||
if (state.isRunning) {
|
||||
logMsg('Scraper is already running.', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
state.isRunning = true;
|
||||
state.concurrency = concurrency;
|
||||
state.delayMs = delayMs;
|
||||
|
||||
logMsg(`Starting scraper with concurrency: ${concurrency}, delay: ${delayMs}ms`, 'info');
|
||||
await updateCounts();
|
||||
|
||||
// Spawn workers
|
||||
for (let i = 0; i < concurrency; i++) {
|
||||
scraperWorker();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Pauses the scraper.
|
||||
*/
|
||||
async function pauseScraping() {
|
||||
if (!state.isRunning) {
|
||||
logMsg('Scraper is not running.', 'warning');
|
||||
return;
|
||||
}
|
||||
|
||||
state.isRunning = false;
|
||||
logMsg('Pausing scraper... Active workers will finish their current URL.', 'warning');
|
||||
await resetQueueBuffer();
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Resets failed items in the queue back to pending to retry them.
|
||||
*/
|
||||
async function retryFailed() {
|
||||
try {
|
||||
const result = await prisma.scrapeQueue.updateMany({
|
||||
where: { status: 'failed' },
|
||||
data: { status: 'pending' }
|
||||
});
|
||||
logMsg(`Reset ${result.count} failed URLs to pending status.`, 'success');
|
||||
await updateCounts();
|
||||
return result.count;
|
||||
} catch (error) {
|
||||
logMsg(`Failed to reset retry queue: ${error.message}`, 'error');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns current scraping statistics and latest logs.
|
||||
*/
|
||||
function getStats() {
|
||||
return {
|
||||
isRunning: state.isRunning,
|
||||
activeWorkers: state.activeWorkers,
|
||||
concurrency: state.concurrency,
|
||||
delayMs: state.delayMs,
|
||||
totalUrls: state.totalUrls,
|
||||
completedCount: state.completedCount,
|
||||
failedCount: state.failedCount,
|
||||
pendingCount: state.pendingCount,
|
||||
logs: state.logs
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
initializeQueue,
|
||||
startScraping,
|
||||
pauseScraping,
|
||||
retryFailed,
|
||||
getStats,
|
||||
updateCounts
|
||||
};
|
||||
351
src/server.js
Normal file
351
src/server.js
Normal file
@@ -0,0 +1,351 @@
|
||||
require('dotenv').config();
|
||||
const express = require('express');
|
||||
const cors = require('cors');
|
||||
const path = require('path');
|
||||
const { PrismaClient } = require('@prisma/client');
|
||||
const scraper = require('./scraper');
|
||||
|
||||
const prisma = new PrismaClient();
|
||||
const app = express();
|
||||
const PORT = process.env.PORT || 3000;
|
||||
|
||||
app.use(cors());
|
||||
app.use(express.json());
|
||||
|
||||
// Request logging middleware
|
||||
app.use((req, res, next) => {
|
||||
console.log(`[HTTP] ${req.method} ${req.url}`);
|
||||
next();
|
||||
});
|
||||
|
||||
// Serve static frontend files
|
||||
app.use(express.static(path.join(__dirname, '../public')));
|
||||
|
||||
|
||||
// SSE clients for live scraping updates
|
||||
let sseClients = [];
|
||||
|
||||
// Helper to broadcast state to SSE clients
|
||||
function broadcastState() {
|
||||
const stats = scraper.getStats();
|
||||
const data = JSON.stringify(stats);
|
||||
sseClients.forEach(client => {
|
||||
client.res.write(`data: ${data}\n\n`);
|
||||
});
|
||||
}
|
||||
|
||||
// Set up periodic stats broadcast to active frontends
|
||||
setInterval(() => {
|
||||
if (sseClients.length > 0) {
|
||||
broadcastState();
|
||||
}
|
||||
}, 1000);
|
||||
|
||||
// API: SSE Stream for Scraper Updates
|
||||
app.get('/api/scrape/stream', (req, res) => {
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive'
|
||||
});
|
||||
|
||||
res.write('\n');
|
||||
const clientId = Date.now();
|
||||
sseClients.push({ id: clientId, res });
|
||||
|
||||
// Send initial state
|
||||
const stats = scraper.getStats();
|
||||
res.write(`data: ${JSON.stringify(stats)}\n\n`);
|
||||
|
||||
req.on('close', () => {
|
||||
sseClients = sseClients.filter(client => client.id !== clientId);
|
||||
});
|
||||
});
|
||||
|
||||
// API: Initialize URL database queue from HTML file
|
||||
app.post('/api/scrape/initialize', async (req, res) => {
|
||||
try {
|
||||
// Fire-and-forget initialization in background so API doesn't timeout
|
||||
scraper.initializeQueue().then(() => {
|
||||
broadcastState();
|
||||
});
|
||||
|
||||
res.json({ success: true, message: 'Queue initialization started in the background.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Get current scraper status and logs
|
||||
app.get('/api/stats', async (req, res) => {
|
||||
try {
|
||||
await scraper.updateCounts();
|
||||
const stats = scraper.getStats();
|
||||
res.json(stats);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Start the scraper
|
||||
app.post('/api/scrape/start', async (req, res) => {
|
||||
const { concurrency, delayMs } = req.body;
|
||||
try {
|
||||
const threadCount = parseInt(concurrency) || 5;
|
||||
const delay = parseInt(delayMs) !== undefined ? parseInt(delayMs) : 500;
|
||||
|
||||
scraper.startScraping(threadCount, delay);
|
||||
broadcastState();
|
||||
|
||||
res.json({ success: true, message: 'Scraper started successfully.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Pause the scraper
|
||||
app.post('/api/scrape/pause', (req, res) => {
|
||||
try {
|
||||
scraper.pauseScraping();
|
||||
broadcastState();
|
||||
res.json({ success: true, message: 'Scraper paused.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Reset failed items to retry them
|
||||
app.post('/api/scrape/retry', async (req, res) => {
|
||||
try {
|
||||
const resetCount = await scraper.retryFailed();
|
||||
broadcastState();
|
||||
res.json({ success: true, message: `Reset ${resetCount} failed items to pending.` });
|
||||
} catch (error) {
|
||||
res.status(500).json({ success: false, error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Get products (Search & Pagination)
|
||||
app.get('/api/products', async (req, res) => {
|
||||
try {
|
||||
const { q, page = 1, limit = 20, category, company } = req.query;
|
||||
|
||||
const pageNum = parseInt(page);
|
||||
const limitNum = parseInt(limit);
|
||||
const offset = (pageNum - 1) * limitNum;
|
||||
|
||||
// Build Prisma query filter
|
||||
const where = {};
|
||||
|
||||
if (q) {
|
||||
where.OR = [
|
||||
{ name: { contains: q } },
|
||||
{ barcode: { contains: q } },
|
||||
{ company: { contains: q } },
|
||||
{ category: { contains: q } }
|
||||
];
|
||||
}
|
||||
|
||||
if (category) {
|
||||
where.category = category;
|
||||
}
|
||||
|
||||
if (company) {
|
||||
where.company = company;
|
||||
}
|
||||
|
||||
const [products, total] = await Promise.all([
|
||||
prisma.product.findMany({
|
||||
where,
|
||||
orderBy: { updatedAt: 'desc' },
|
||||
skip: offset,
|
||||
take: limitNum
|
||||
}),
|
||||
prisma.product.count({ where })
|
||||
]);
|
||||
|
||||
// Get categories and companies for frontend filters
|
||||
// Optimize: run these queries only when page is 1
|
||||
let categories = [];
|
||||
let companies = [];
|
||||
if (pageNum === 1) {
|
||||
const catResults = await prisma.product.groupBy({
|
||||
by: ['category'],
|
||||
_count: { category: true },
|
||||
where: { category: { not: null } },
|
||||
orderBy: {
|
||||
_count: {
|
||||
category: 'desc'
|
||||
}
|
||||
}
|
||||
});
|
||||
categories = catResults.map(c => ({ name: c.category, count: c._count.category }));
|
||||
|
||||
const compResults = await prisma.product.groupBy({
|
||||
by: ['company'],
|
||||
_count: { company: true },
|
||||
where: { company: { not: null } },
|
||||
orderBy: {
|
||||
_count: {
|
||||
company: 'desc'
|
||||
}
|
||||
},
|
||||
take: 30 // Top 30 companies
|
||||
});
|
||||
companies = compResults.map(c => ({ name: c.company, count: c._count.company }));
|
||||
}
|
||||
|
||||
|
||||
res.json({
|
||||
products,
|
||||
pagination: {
|
||||
total,
|
||||
page: pageNum,
|
||||
limit: limitNum,
|
||||
totalPages: Math.ceil(total / limitNum)
|
||||
},
|
||||
filters: {
|
||||
categories,
|
||||
companies
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Get a single product by ID or Barcode
|
||||
app.get('/api/products/:idOrBarcode', async (req, res) => {
|
||||
const { idOrBarcode } = req.params;
|
||||
try {
|
||||
let product = null;
|
||||
|
||||
if (isNaN(idOrBarcode)) {
|
||||
product = await prisma.product.findFirst({
|
||||
where: { OR: [{ barcode: idOrBarcode }, { url: idOrBarcode }] }
|
||||
});
|
||||
} else {
|
||||
product = await prisma.product.findUnique({
|
||||
where: { id: parseInt(idOrBarcode) }
|
||||
});
|
||||
|
||||
if (!product) {
|
||||
product = await prisma.product.findUnique({
|
||||
where: { barcode: idOrBarcode }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!product) {
|
||||
return res.status(404).json({ error: 'Product not found.' });
|
||||
}
|
||||
|
||||
res.json(product);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Add a product manually
|
||||
app.post('/api/products', async (req, res) => {
|
||||
const { barcode, name, cleanName, amountRaw, amount, unit, image, category, subcategory, company, url } = req.body;
|
||||
|
||||
if (!barcode || !name) {
|
||||
return res.status(400).json({ error: 'Barcode and Name are required fields.' });
|
||||
}
|
||||
|
||||
try {
|
||||
const existing = await prisma.product.findUnique({ where: { barcode } });
|
||||
if (existing) {
|
||||
return res.status(400).json({ error: 'A product with this barcode already exists.' });
|
||||
}
|
||||
|
||||
// Set clean name from utils if not provided
|
||||
const parsedName = cleanName ? { cleanName, amountRaw, amount, unit } : require('./utils').parseName(name);
|
||||
|
||||
const product = await prisma.product.create({
|
||||
data: {
|
||||
barcode,
|
||||
name,
|
||||
cleanName: parsedName.cleanName,
|
||||
amountRaw: parsedName.amountRaw,
|
||||
amount: parsedName.amount,
|
||||
unit: parsedName.unit,
|
||||
image: image || null,
|
||||
category: category || null,
|
||||
subcategory: subcategory || null,
|
||||
company: company || null,
|
||||
url: url || `manual-${Date.now()}`
|
||||
}
|
||||
});
|
||||
|
||||
res.status(201).json(product);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Update a product
|
||||
app.put('/api/products/:id', async (req, res) => {
|
||||
const { id } = req.params;
|
||||
const { name, cleanName, amountRaw, amount, unit, image, category, subcategory, company, barcode } = req.body;
|
||||
|
||||
try {
|
||||
const productId = parseInt(id);
|
||||
const existing = await prisma.product.findUnique({ where: { id: productId } });
|
||||
if (!existing) {
|
||||
return res.status(404).json({ error: 'Product not found.' });
|
||||
}
|
||||
|
||||
const updateData = {
|
||||
name,
|
||||
cleanName,
|
||||
amountRaw,
|
||||
amount: amount !== undefined ? (amount ? parseFloat(amount) : null) : undefined,
|
||||
unit,
|
||||
image,
|
||||
category,
|
||||
subcategory,
|
||||
company,
|
||||
barcode
|
||||
};
|
||||
|
||||
// Clean undefined values
|
||||
Object.keys(updateData).forEach(key => updateData[key] === undefined && delete updateData[key]);
|
||||
|
||||
const updated = await prisma.product.update({
|
||||
where: { id: productId },
|
||||
data: updateData
|
||||
});
|
||||
|
||||
res.json(updated);
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// API: Delete a product
|
||||
app.delete('/api/products/:id', async (req, res) => {
|
||||
const { id } = req.params;
|
||||
try {
|
||||
const productId = parseInt(id);
|
||||
await prisma.product.delete({
|
||||
where: { id: productId }
|
||||
});
|
||||
res.json({ success: true, message: 'Product deleted successfully.' });
|
||||
} catch (error) {
|
||||
res.status(500).json({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Start express server
|
||||
app.listen(PORT, async () => {
|
||||
console.log(`Server running at http://localhost:${PORT}`);
|
||||
|
||||
// Auto initialize queue in background on startup if empty
|
||||
try {
|
||||
await scraper.initializeQueue();
|
||||
} catch (e) {
|
||||
console.error('Auto initialization of queue failed on startup:', e);
|
||||
}
|
||||
});
|
||||
74
src/utils.js
Normal file
74
src/utils.js
Normal file
@@ -0,0 +1,74 @@
|
||||
/**
|
||||
* Sleep helper function
|
||||
* @param {number} ms
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
|
||||
/**
|
||||
* Parses product name to extract amount and unit from parentheses at the end of the name.
|
||||
* Handles patterns like (75 g), (1200 ml), (6x 200 ml), (5 x 75 g), (90 Yaprak), etc.
|
||||
*
|
||||
* @param {string} fullName
|
||||
* @returns {{cleanName: string, amountRaw: string|null, amount: number|null, unit: string|null}}
|
||||
*/
|
||||
function parseName(fullName) {
|
||||
if (!fullName) {
|
||||
return { cleanName: '', amountRaw: null, amount: null, unit: null };
|
||||
}
|
||||
|
||||
// Look for text inside parentheses at the end of the name
|
||||
const match = fullName.match(/\(([^)]+)\)\s*$/);
|
||||
if (!match) {
|
||||
return { cleanName: fullName.trim(), amountRaw: null, amount: null, unit: null };
|
||||
}
|
||||
|
||||
const parenContent = match[1].trim(); // e.g., "75 g" or "6x 200 ml" or "90 Yaprak"
|
||||
const cleanName = fullName.replace(/\(([^)]+)\)\s*$/, '').trim();
|
||||
|
||||
// Regex to capture numeric part + multiplier expressions (digits, spaces, x, *, ,, ., -)
|
||||
// followed by the alphabetic unit part (e.g. g, ml, L, adet, Yaprak)
|
||||
const amountUnitMatch = parenContent.match(/^([\d\s.x*X,-]+)\s*([a-zA-ZçıöşğüÇİÖŞĞÜ]+.*)$/);
|
||||
|
||||
if (amountUnitMatch) {
|
||||
const amountRaw = amountUnitMatch[1].trim();
|
||||
const unit = amountUnitMatch[2].trim();
|
||||
|
||||
let amount = null;
|
||||
try {
|
||||
// Clean up multiplier characters (convert x, X to *, replace Turkish decimal comma with dot)
|
||||
const evalString = amountRaw.toLowerCase().replace(/x/g, '*').replace(/,/g, '.');
|
||||
|
||||
// Ensure only numbers, math operations, and spaces exist in the evaluation string to prevent arbitrary code execution
|
||||
if (/^[0-9\s.*+-]+$/.test(evalString)) {
|
||||
// Safe evaluation
|
||||
amount = Function(`"use strict"; return (${evalString})`)();
|
||||
} else {
|
||||
amount = parseFloat(amountRaw.replace(/,/g, '.'));
|
||||
}
|
||||
} catch (e) {
|
||||
// Fallback
|
||||
amount = parseFloat(amountRaw.replace(/,/g, '.'));
|
||||
}
|
||||
|
||||
return {
|
||||
cleanName,
|
||||
amountRaw,
|
||||
amount: (amount && !isNaN(amount)) ? amount : null,
|
||||
unit
|
||||
};
|
||||
}
|
||||
|
||||
// Fallback if structure is different
|
||||
return {
|
||||
cleanName,
|
||||
amountRaw: parenContent,
|
||||
amount: null,
|
||||
unit: null
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
sleep,
|
||||
parseName
|
||||
};
|
||||
Reference in New Issue
Block a user