diff --git a/client.py b/client.py index 3343c35..ddda6c6 100644 --- a/client.py +++ b/client.py @@ -1,5 +1,6 @@ import requests + class DiffbotClient(object): base_url = 'http://api.diffbot.com/' @@ -30,20 +31,21 @@ def format_version_string(version_number): """ return 'v{}'.format(version_number) + class DiffbotJob(DiffbotClient): """ Various calls for managing a Crawlbot or Bulk API job. """ - def request(self,params): - response = requests.get(self.compose_url(self.jobType,3),params=params) + def request(self, params): + response = requests.get(self.compose_url(self.jobType, 3), params=params) response.raise_for_status try: return response.json() - except: - print response.text + except Exception: + print(response.text) - def start(self,params): + def start(self, params): response = self.request(params) return response @@ -51,7 +53,7 @@ def status(self): response = self.request(self.params) return response - def update(self,**kwargs): + def update(self, **kwargs): temp_params = self.params temp_params.update(kwargs) response = self.request(self.params) @@ -69,14 +71,14 @@ def restart(self): response = self.request(temp_params) return response - def download(self,data_format="json"): + def download(self, data_format="json"): """ downloads the JSON output of a crawl or bulk job """ download_url = '{}/v3/{}/download/{}-{}_data.{}'.format( - self.base_url,self.jobType,self.params['token'],self.params['name'],data_format - ) + self.base_url, self.jobType, self.params['token'], self.params['name'], data_format + ) download = requests.get(download_url) download.raise_for_status() if data_format == "csv": @@ -84,12 +86,13 @@ def download(self,data_format="json"): else: return download.json() + class DiffbotCrawl(DiffbotJob): """ Initializes a Diffbot crawl. Pass additional arguments as necessary. """ - def __init__(self,token,name,seeds=None,api=None,apiVersion=3,**kwargs): + def __init__(self, token, name, seeds=None, api=None, apiVersion=3, **kwargs): self.params = { "token": token, "name": name, @@ -98,7 +101,7 @@ def __init__(self,token,name,seeds=None,api=None,apiVersion=3,**kwargs): if seeds: startParams['seeds'] = seeds if api: - startParams['apiUrl'] = self.compose_url(api,apiVersion) + startParams['apiUrl'] = self.compose_url(api, apiVersion) startParams.update(kwargs) self.jobType = "crawl" - self.start(startParams) \ No newline at end of file + self.start(startParams) diff --git a/example.py b/example.py index b7de9e6..d871f05 100644 --- a/example.py +++ b/example.py @@ -1,74 +1,74 @@ -from client import DiffbotClient,DiffbotCrawl +from client import DiffbotClient, DiffbotCrawl from config import API_TOKEN import pprint import time -print "Calling article API endpoint on the url: http://shichuan.github.io/javascript-patterns/...\n" +print("Calling article API endpoint on the url: http://shichuan.github.io/javascript-patterns/...\n") diffbot = DiffbotClient() token = API_TOKEN url = "http://shichuan.github.io/javascript-patterns/" api = "article" response = diffbot.request(url, token, api) -print "\nPrinting response:\n" +print("\nPrinting response:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(response) +print(pp.pprint(response)) -print -print "Calling article API endpoint with fields specified on the url: http://shichuan.github.io/javascript-patterns/...\n" +print() +print("Calling article API endpoint with fields specified on the url: http://shichuan.github.io/javascript-patterns/...\n") diffbot = DiffbotClient() token = API_TOKEN url = "http://shichuan.github.io/javascript-patterns/" api = "article" response = diffbot.request(url, token, api, fields=['title', 'type']) -print "\nPrinting response:\n" +print("\nPrinting response:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(response) +print(pp.pprint(response)) -print -print "Calling frontpage API endpoint on the url: http://www.huffingtonpost.com/...\n" +print() +print("Calling frontpage API endpoint on the url: http://www.huffingtonpost.com/...\n") diffbot = DiffbotClient() token = API_TOKEN url = "http://www.huffingtonpost.com/" api = "frontpage" response = diffbot.request(url, token, api) -print "\nPrinting response:\n" +print("\nPrinting response:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(response) +print(pp.pprint(response)) -print -print "Calling product API endpoint on the url: http://www.overstock.com/Home-Garden/iRobot-650-Roomba-Vacuuming-Robot/7886009/product.html...\n" +print() +print("Calling product API endpoint on the url: http://www.overstock.com/Home-Garden/iRobot-650-Roomba-Vacuuming-Robot/7886009/product.html...\n") diffbot = DiffbotClient() token = API_TOKEN url = "http://www.overstock.com/Home-Garden/iRobot-650-Roomba-Vacuuming-Robot/7886009/product.html" api = "product" response = diffbot.request(url, token, api) -print "\nPrinting response:\n" +print("\nPrinting response:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(response) +print(pp.pprint(response)) -print -print "Calling image API endpoint on the url: http://www.google.com/...\n" +print() +print("Calling image API endpoint on the url: http://www.google.com/...\n") diffbot = DiffbotClient() token = API_TOKEN url = "http://www.google.com/" api = "image" response = diffbot.request(url, token, api) -print "\nPrinting response:\n" +print("\nPrinting response:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(response) +print(pp.pprint(response)) -print -print "Calling classifier API endpoint on the url: http://www.twitter.com/...\n" +print() +print("Calling classifier API endpoint on the url: http://www.twitter.com/...\n") diffbot = DiffbotClient() token = API_TOKEN url = "http://www.twitter.com/" api = "analyze" response = diffbot.request(url, token, api) -print "\nPrinting response:\n" +print("\nPrinting response:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(response) +print(pp.pprint(response)) -print "Create a new crawl of http://support.diffbot.com/ using the Article API...\n" +print("Create a new crawl of http://support.diffbot.com/ using the Article API...\n") token = API_TOKEN seeds = "http://support.diffbot.com" api = "article" @@ -76,8 +76,8 @@ diffbot = DiffbotCrawl(token, name, seeds=seeds, api=api) time.sleep(5) status = diffbot.status() -print "\nPrinting status:\n" +print("\nPrinting status:\n") pp = pprint.PrettyPrinter(indent=4) -print pp.pprint(status) -print "\nDeleting test crawl.\n" -diffbot.delete() \ No newline at end of file +print(pp.pprint(status)) +print("\nDeleting test crawl.\n") +diffbot.delete() diff --git a/tests/unit_test.py b/tests/unit_test.py index fe510a0..68140b6 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -1,17 +1,17 @@ import os -import urlparse +from urllib.parse import urlparse import unittest import requests -from mock import patch +from unittest.mock import patch from client import DiffbotClient def fake_get(url, params): - path = urlparse.urlparse(url).path + path = urlparse(url).path path_tuple = os.path.split(path) api = path_tuple[len(path_tuple) - 1] - resource_file = os.path.normpath('tests/test_fixtures/%s.json' % api) + resource_file = os.path.normpath('test_fixtures/%s.json' % api) with open(resource_file, mode='rb') as file: json = file.read() r = requests.Response()