JavaScript

Here is a step by step guide to use the Scraping Pros API on JavaScript:

Generate Token

In order to use the API-endpoints, we need to generate a token.

const fetch = require('node-fetch');

async function generateToken() {
    const data = {
        'username': 'YOUR USER',
        'password': 'YOUR PASSWORD'
    };
    try {
        const response = await fetch("https://api.scrapingpros.com/login", {
            method: 'POST',
            headers: { 'Content-Type': 'application/json' },
            body: JSON.stringify(data)
        });
        if (response.ok) {
            const jsonResponse = await response.json();
            return jsonResponse.token;
        } else {
            console.log(`Error: ${response.statusText} (${response.status})`);
        }
    } catch (error) {
        console.error(`An error occurred: ${error.message}`);
    }
}

Create Project

Next, we'll show usage of all project-related endpoints. Please note that you'll need the token generated above in order to use them. The next endpoint, create_project, returns the project_id of the created project, please keep that in mind, since we'll need that project_id for the upcoming endpoints.

async function createProject(token) {
    const data = {
        name: "string",
        priority: "integer",
        description: "string"
    };
    try {
        const response = await fetch("https://api.scrapingpros.com/projects", {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${token}`
            },
            body: JSON.stringify(data)
        });
        return await response.json();
    } catch (error) {
        console.error(`An error occurred: ${error.message}`);
    }
}

Create Batch

Now, using the project_id, we'll call the endpoint create_batch, which returns the batch_id created, please keep that in mind, since we'll be using it in the upcoming endpoints. Let's check some useful batch-related endpoints.

async function createBatch(token, projectId) {
    const data = {
        project: projectId,
        name: "string",
        priority: "integer"
    };
    try {
        const response = await fetch("https://api.scrapingpros.com/batches", {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${token}`
            },
            body: JSON.stringify(data)
        });
        return await response.json();
    } catch (error) {
        console.error(`An error occurred: ${error.message}`);
    }
}

Append jobs to batch

In the appendJobsToBatch endpoint, we add all jobs that we want to run. Especifically, we should add them to the jobs array.

async function appendJobsToBatch(token, batchId) {
    const data = {
        jobs: [
            {
                url: "string",
                scrap_mode: "YOUR SCRAP MODE",
                arguments: {}
            }
        ]
    };
    try {
        const response = await fetch(`https://api.scrapingpros.com/batches/${batchId}/append-jobs`, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${token}`
            },
            body: JSON.stringify(data)
        });
        return await response.json();
    } catch (error) {
        console.error(`An error occurred: ${error.message}`);
    }
}

Run batch

Finally, we set this batch to run. Now all that's left is to wait for it to complete. Once it has finished, we can see the data retrieved.

async function runBatch(token, batchId) {
    try {
        const response = await fetch(`https://api.scrapingpros.com/batches/${batchId}/run`, {
            method: 'POST',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${token}`
            }
        });
        return await response.json();
    } catch (error) {
        console.error(`An error occurred: ${error.message}`);
    }
}

Get data from a batch

Next, we can check all data for that batch.

async function getData(token, batchId) {
    const apiUrl = `https://api.scrapingpros.com/get_data/batch/${batchId}`;
    try {
        const response = await fetch(apiUrl, {
            method: 'GET',
            headers: {
                'Content-Type': 'application/json',
                'Authorization': `Bearer ${token}`
            }
        });
        if (response.ok) {
            return await response.json();
        } else {
            console.log(`Error: ${response.statusText} (${response.status})`);
        }
    } catch (error) {
        console.error(`An error occurred: ${error.message}`);
    }
}