Skip to main content

Ruby

require 'net/http'
require 'json'
require 'uri'

def generate_token()
begin
data = { 'username' => 'YOUR USER', 'password' => 'YOUR PASSWORD' }
api_url = URI.parse("https://api.scrapingpros.com/login")
json_data = data.to_json
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Post.new(api_url.path, {'Content-Type' => 'application/json'})
request.body = json_data
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
json_proxy = JSON.parse(response.body)
token = json_proxy['token']
return token
else
puts "Error: #{response.message} (#{response.code})"
end
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end
end

def create_project()
begin
data = {
"name": "string",
"priority": "integer",
"description": "string"
}
api_url = URI.parse("https://api.scrapingpros.com/projects")
json_data = data.to_json
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Post.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
request.body = json_data
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end

def pause_project()
begin
api_url = URI.parse("https://api.scrapingpros.com/projects/PROJECT ID/pause")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Put.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end

def resume_project()
begin
api_url = URI.parse("https://api.scrapingpros.com/projects/PROJECT ID/resume")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Put.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end

def close_project()
begin
api_url = URI.parse("https://api.scrapingpros.com/projects/PROJECT ID/close")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Put.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end

def create_batch()
begin
data = { 'project' => 'YOUR PROJECT ID', 'name' => "string", 'priority' => "integer"}
api_url = URI.parse("https://api.scrapingpros.com/batches")
json_data = data.to_json
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Post.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
request.body = json_data
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end

def append_jobs_to_batch()
api_url = URI.parse("https://api.scrapingpros.com/batches/YOUR BATCH ID/append-jobs")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Post.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
data = { "jobs": [] }
data[:jobs] << { "url": "string", "scrap_mode": "YOUR SCRAP MODE", "arguments": {}}
json_data = data.to_json
request.body = json_data
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def run_batch()
api_url = URI.parse("https://api.scrapingpros.com/batches/YOUR BATCH ID/run")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Post.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def pause_batch()
api_url = URI.parse("https://api.scrapingpros.com/batches/YOUR BATCH ID/pause")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Put.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def resume_batch()
api_url = URI.parse("https://api.scrapingpros.com/batches/YOUR BATCH ID/resume")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Put.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def get_projects()
api_url = URI.parse("https://api.scrapingpros.com/projects")
filter = {
"filter_by": "String" [Either A, P, or C]
}
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Get.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
json_data = filter.to_json
request.body = json_data
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def get_projects_summary()
api_url = URI.parse("https://api.scrapingpros.com/projects/summary")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Get.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def get_credits_summary()
api_url = URI.parse("https://api.scrapingpros.com/credits/summary")
time_period = {
"since": "String" [Either "last_month" or "year_months"]
}
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Get.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
json_data = time_period.to_json
request.body = json_data
response = http.request(request)
if response.is_a?(Net::HTTPSuccess)
response_json = JSON.parse(response.body)
return response_json
else
puts "Error: #{response.message} (#{response.code})"
end
end

def get_jobs()
begin
api_url = URI.parse("https://api.scrapingpros.com/jobs?batch=YOUR BATCH ID")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Get.new(api_url, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
json_response = JSON.parse(response.body)
return json_response
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end

def get_data()
begin
api_url = URI.parse("https://api.scrapingpros.com/get_data/YOUR ID JOB")
http = Net::HTTP.new(api_url.host, api_url.port)
http.use_ssl = (api_url.scheme == "https")
request = Net::HTTP::Get.new(api_url.path, {
'Content-Type' => 'application/json',
'Authorization' => "YOUR TOKEN"
})
response = http.request(request)
json_response = JSON.parse(response.body)
return json_response
rescue StandardError => e
puts "An error occurred: #{e.message}"
end
end