Skip to content

updated to work for python3 #6

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
Salil999 wants to merge 1 commit into from
Closed

updated to work for python3 #6

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 15 additions & 12 deletions client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import requests


class DiffbotClient(object):

base_url = 'http://api.diffbot.com/'
Expand Down Expand Up @@ -30,28 +31,29 @@ def format_version_string(version_number):
"""
return 'v{}'.format(version_number)


class DiffbotJob(DiffbotClient):
"""
Various calls for managing a Crawlbot or Bulk API job.
"""

def request(self,params):
response = requests.get(self.compose_url(self.jobType,3),params=params)
def request(self, params):
response = requests.get(self.compose_url(self.jobType, 3), params=params)
response.raise_for_status
try:
return response.json()
except:
print response.text
except Exception:
print(response.text)

def start(self,params):
def start(self, params):
response = self.request(params)
return response

def status(self):
response = self.request(self.params)
return response

def update(self,**kwargs):
def update(self, **kwargs):
temp_params = self.params
temp_params.update(kwargs)
response = self.request(self.params)
Expand All @@ -69,27 +71,28 @@ def restart(self):
response = self.request(temp_params)
return response

def download(self,data_format="json"):
def download(self, data_format="json"):
"""
downloads the JSON output of a crawl or bulk job
"""

download_url = '{}/v3/{}/download/{}-{}_data.{}'.format(
self.base_url,self.jobType,self.params['token'],self.params['name'],data_format
)
self.base_url, self.jobType, self.params['token'], self.params['name'], data_format
)
download = requests.get(download_url)
download.raise_for_status()
if data_format == "csv":
return download.content
else:
return download.json()


class DiffbotCrawl(DiffbotJob):
"""
Initializes a Diffbot crawl. Pass additional arguments as necessary.
"""

def __init__(self,token,name,seeds=None,api=None,apiVersion=3,**kwargs):
def __init__(self, token, name, seeds=None, api=None, apiVersion=3, **kwargs):
self.params = {
"token": token,
"name": name,
Expand All @@ -98,7 +101,7 @@ def __init__(self,token,name,seeds=None,api=None,apiVersion=3,**kwargs):
if seeds:
startParams['seeds'] = seeds
if api:
startParams['apiUrl'] = self.compose_url(api,apiVersion)
startParams['apiUrl'] = self.compose_url(api, apiVersion)
startParams.update(kwargs)
self.jobType = "crawl"
self.start(startParams)
self.start(startParams)
58 changes: 29 additions & 29 deletions example.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,83 +1,83 @@
from client import DiffbotClient,DiffbotCrawl
from client import DiffbotClient, DiffbotCrawl
from config import API_TOKEN
import pprint
import time

print "Calling article API endpoint on the url: http://shichuan.github.io/javascript-patterns/...\n"
print("Calling article API endpoint on the url: http://shichuan.github.io/javascript-patterns/...\n")
diffbot = DiffbotClient()
token = API_TOKEN
url = "http://shichuan.github.io/javascript-patterns/"
api = "article"
response = diffbot.request(url, token, api)
print "\nPrinting response:\n"
print("\nPrinting response:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(response)
print(pp.pprint(response))

print
print "Calling article API endpoint with fields specified on the url: http://shichuan.github.io/javascript-patterns/...\n"
print()
print("Calling article API endpoint with fields specified on the url: http://shichuan.github.io/javascript-patterns/...\n")
diffbot = DiffbotClient()
token = API_TOKEN
url = "http://shichuan.github.io/javascript-patterns/"
api = "article"
response = diffbot.request(url, token, api, fields=['title', 'type'])
print "\nPrinting response:\n"
print("\nPrinting response:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(response)
print(pp.pprint(response))

print
print "Calling frontpage API endpoint on the url: http://www.huffingtonpost.com/...\n"
print()
print("Calling frontpage API endpoint on the url: http://www.huffingtonpost.com/...\n")
diffbot = DiffbotClient()
token = API_TOKEN
url = "http://www.huffingtonpost.com/"
api = "frontpage"
response = diffbot.request(url, token, api)
print "\nPrinting response:\n"
print("\nPrinting response:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(response)
print(pp.pprint(response))

print
print "Calling product API endpoint on the url: http://www.overstock.com/Home-Garden/iRobot-650-Roomba-Vacuuming-Robot/7886009/product.html...\n"
print()
print("Calling product API endpoint on the url: http://www.overstock.com/Home-Garden/iRobot-650-Roomba-Vacuuming-Robot/7886009/product.html...\n")
diffbot = DiffbotClient()
token = API_TOKEN
url = "http://www.overstock.com/Home-Garden/iRobot-650-Roomba-Vacuuming-Robot/7886009/product.html"
api = "product"
response = diffbot.request(url, token, api)
print "\nPrinting response:\n"
print("\nPrinting response:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(response)
print(pp.pprint(response))

print
print "Calling image API endpoint on the url: http://www.google.com/...\n"
print()
print("Calling image API endpoint on the url: http://www.google.com/...\n")
diffbot = DiffbotClient()
token = API_TOKEN
url = "http://www.google.com/"
api = "image"
response = diffbot.request(url, token, api)
print "\nPrinting response:\n"
print("\nPrinting response:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(response)
print(pp.pprint(response))

print
print "Calling classifier API endpoint on the url: http://www.twitter.com/...\n"
print()
print("Calling classifier API endpoint on the url: http://www.twitter.com/...\n")
diffbot = DiffbotClient()
token = API_TOKEN
url = "http://www.twitter.com/"
api = "analyze"
response = diffbot.request(url, token, api)
print "\nPrinting response:\n"
print("\nPrinting response:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(response)
print(pp.pprint(response))

print "Create a new crawl of http://support.diffbot.com/ using the Article API...\n"
print("Create a new crawl of http://support.diffbot.com/ using the Article API...\n")
token = API_TOKEN
seeds = "http://support.diffbot.com"
api = "article"
name = "testCrawl"
diffbot = DiffbotCrawl(token, name, seeds=seeds, api=api)
time.sleep(5)
status = diffbot.status()
print "\nPrinting status:\n"
print("\nPrinting status:\n")
pp = pprint.PrettyPrinter(indent=4)
print pp.pprint(status)
print "\nDeleting test crawl.\n"
diffbot.delete()
print(pp.pprint(status))
print("\nDeleting test crawl.\n")
diffbot.delete()
8 changes: 4 additions & 4 deletions tests/unit_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,17 +1,17 @@
import os
import urlparse
from urllib.parse import urlparse
import unittest
import requests
from mock import patch
from unittest.mock import patch

from client import DiffbotClient


def fake_get(url, params):
path = urlparse.urlparse(url).path
path = urlparse(url).path
path_tuple = os.path.split(path)
api = path_tuple[len(path_tuple) - 1]
resource_file = os.path.normpath('tests/test_fixtures/%s.json' % api)
resource_file = os.path.normpath('test_fixtures/%s.json' % api)
with open(resource_file, mode='rb') as file:
json = file.read()
r = requests.Response()
Expand Down