const fetch = require('node-fetch');
async function generateToken() {
const data = {
'username': 'YOUR USER',
'password': 'YOUR PASSWORD'
};
try {
const response = await fetch("https://api.scrapingpros.com/login", {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(data)
});
if (response.ok) {
const jsonResponse = await response.json();
return jsonResponse.token;
} else {
console.log(`Error: ${response.statusText} (${response.status})`);
}
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function createProject(token) {
const data = {
priority: "integer",
name: "string",
description: "string"
};
try {
const response = await fetch("https://api.scrapingpros.com/projects", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
},
body: JSON.stringify(data)
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function pauseProject(token, projectId) {
try {
const response = await fetch(`https://api.scrapingpros.com/projects/${projectId}/pause`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function resumeProject(token, projectId) {
try {
const response = await fetch(`https://api.scrapingpros.com/projects/${projectId}/resume`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function closeProject(token, projectId) {
try {
const response = await fetch(`https://api.scrapingpros.com/projects/${projectId}/close`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function createBatch(token, projectId) {
const data = {
project: projectId,
name: "string",
priority: "integer"
};
try {
const response = await fetch("https://api.scrapingpros.com/batches", {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
},
body: JSON.stringify(data)
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function appendJobsToBatch(token, batchId) {
const data = {
jobs: [
{
url: "string",
scrap_mode: "YOUR SCRAP MODE",
arguments: {}
}
]
};
try {
const response = await fetch(`https://api.scrapingpros.com/batches/${batchId}/append-jobs`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
},
body: JSON.stringify(data)
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function runBatch(token, batchId) {
try {
const response = await fetch(`https://api.scrapingpros.com/batches/${batchId}/run`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function pauseBatch(token, batchId) {
try {
const response = await fetch(`https://api.scrapingpros.com/batches/${batchId}/pause`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function resumeBatch(token, batchId) {
try {
const response = await fetch(`https://api.scrapingpros.com/batches/${batchId}/resume`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function getProjects(token, filter) {
try {
const response = await fetch(`https://api.scrapingpros.com/projects?filter_by=${filter}`, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function getProjectsSummary(token) {
try {
const response = await fetch("https://api.scrapingpros.com/projects/summary", {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
return await response.json();
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function getCreditsSummary(token, timePeriod) {
const apiUrl = "https://api.scrapingpros.com/credits/summary";
const data = {
since: timePeriod
};
try {
const response = await fetch(apiUrl, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
},
body: JSON.stringify(data)
});
if (response.ok) {
return await response.json();
} else {
console.log(`Error: ${response.statusText} (${response.status})`);
}
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function getJobs(token, batchId) {
const apiUrl = `https://api.scrapingpros.com/jobs?batch=${batchId}`;
try {
const response = await fetch(apiUrl, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
if (response.ok) {
return await response.json();
} else {
console.log(`Error: ${response.statusText} (${response.status})`);
}
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}
async function getData(token, jobId) {
const apiUrl = `https://api.scrapingpros.com/get_data/${jobId}`;
try {
const response = await fetch(apiUrl, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
}
});
if (response.ok) {
return await response.json();
} else {
console.log(`Error: ${response.statusText} (${response.status})`);
}
} catch (error) {
console.error(`An error occurred: ${error.message}`);
}
}