pyechonest.util

1 #!/usr/bin/env python 2 # encoding: utf-8 3 4 """ 5 Copyright (c) 2010 The Echo Nest. All rights reserved. 6 Created by Tyler Williams on 2010-04-25. 7 8 Utility functions to support the Echo Nest web API interface. 9 """ 10 import urllib 11 import urllib2 12 import httplib 13 import config 14 import logging 15 import socket 16 import re 17 import time 18 import os 19 import subprocess 20 import traceback 21 from types import StringType, UnicodeType 22 23 try: 24 import json 25 except ImportError: 26 import simplejson as json 27 28 logging.basicConfig(level=logging.INFO) 29 logger = logging.getLogger(__name__) 30 TYPENAMES = ( 31 ('AR', 'artist'), 32 ('SO', 'song'), 33 ('RE', 'release'), 34 ('TR', 'track'), 35 ('PE', 'person'), 36 ('DE', 'device'), 37 ('LI', 'listener'), 38 ('ED', 'editor'), 39 ('TW', 'tweditor'), 40 ('CA', 'catalog'), 41 ) 42 foreign_regex = re.compile(r'^.+?:(%s):([^^]+)\^?([0-9\.]+)?' % r'|'.join(n[1] for n in TYPENAMES)) 43 short_regex = re.compile(r'^((%s)[0-9A-Z]{16})\^?([0-9\.]+)?' % r'|'.join(n[0] for n in TYPENAMES)) 44 long_regex = re.compile(r'music://id.echonest.com/.+?/(%s)/(%s)[0-9A-Z]{16}\^?([0-9\.]+)?' % (r'|'.join(n[0] for n in TYPENAMES), r'|'.join(n[0] for n in TYPENAMES))) 45 headers = [('User-Agent', 'Pyechonest %s' % (config.__version__,))] 46

47 -class MyBaseHandler(urllib2.BaseHandler):

48 - def default_open(self, request):

49 if config.TRACE_API_CALLS: 50 logger.info("%s" % (request.get_full_url(),)) 51 request.start_time = time.time() 52 return None

53

54 -class MyErrorProcessor(urllib2.HTTPErrorProcessor):

55 - def http_response(self, request, response):

56 code = response.code 57 if config.TRACE_API_CALLS: 58 logger.info("took %2.2fs: (%i)" % (time.time()-request.start_time,code)) 59 if code in [200, 400, 401, 403, 500]: 60 return response 61 else: 62 urllib2.HTTPErrorProcessor.http_response(self, request, response)

63 64 opener = urllib2.build_opener(MyBaseHandler(), MyErrorProcessor()) 65 opener.addheaders = headers 66

67 -class EchoNestException(Exception):

68 """ 69 Parent exception class. Catches API and URL/HTTP errors. 70 """

71 - def __init__(self, code, message, headers):

72 if code is None: 73 code = -1 74 message = 'Echo Nest Unknown Error' 75 76 if message is None: 77 super(EchoNestException, self).__init__('Echo Nest Error %d' % code,) 78 else: 79 super(EchoNestException, self).__init__(message,) 80 self.headers = headers 81 self.code = code

82

83 -class EchoNestAPIError(EchoNestException):

84 """ 85 API Specific Errors. 86 """

87 - def __init__(self, code, message, headers):

88 formatted_message = ('Echo Nest API Error %d: %s' % (code, message),) 89 super(EchoNestAPIError, self).__init__(code, formatted_message, headers)

90

91 -class EchoNestIOError(EchoNestException):

92 """ 93 URL and HTTP errors. 94 """

95 - def __init__(self, code=None, error=None, headers=headers):

96 formatted_message = ('Echo Nest IOError: %s' % headers,) 97 super(EchoNestIOError, self).__init__(code, formatted_message, headers)

98

99 -def get_successful_response(raw_json):

100 if hasattr(raw_json,'headers'): 101 headers = raw_json.headers 102 else: 103 headers = {'Headers':'No Headers'} 104 raw_json = raw_json.read() 105 try: 106 response_dict = json.loads(raw_json) 107 status_dict = response_dict['response']['status'] 108 code = int(status_dict['code']) 109 message = status_dict['message'] 110 if (code != 0): 111 # do some cute exception handling 112 raise EchoNestAPIError(code, message, headers) 113 del response_dict['response']['status'] 114 return response_dict 115 except ValueError: 116 logger.debug(traceback.format_exc()) 117 raise EchoNestAPIError(-1, "Unknown error.", headers)

118 119 120 # These two functions are to deal with the unknown encoded output of codegen (varies by platform and ID3 tag)

121 -def reallyunicode(s, encoding="utf-8"):

122 if type(s) is StringType: 123 for args in ((encoding,), ('utf-8',), ('latin-1',), ('ascii', 'replace')): 124 try: 125 s = s.decode(*args) 126 break 127 except UnicodeDecodeError: 128 continue 129 if type(s) is not UnicodeType: 130 raise ValueError, "%s is not a string at all." % s 131 return s

132

133 -def reallyUTF8(s):

134 return reallyunicode(s).encode("utf-8")

135

136 -def codegen(filename, start=0, duration=30):

137 # Run codegen on the file and return the json. If start or duration is -1 ignore them. 138 cmd = config.CODEGEN_BINARY_OVERRIDE 139 if not cmd: 140 # Is this is posix platform, or is it windows? 141 if hasattr(os, 'uname'): 142 if(os.uname()[0] == "Darwin"): 143 cmd = "codegen.Darwin" 144 else: 145 cmd = 'codegen.'+os.uname()[0]+'-'+os.uname()[4] 146 else: 147 cmd = "codegen.windows.exe" 148 149 if not os.path.exists(cmd): 150 raise Exception("Codegen binary not found.") 151 152 command = cmd + " \"" + filename + "\" " 153 if start >= 0: 154 command = command + str(start) + " " 155 if duration >= 0: 156 command = command + str(duration) 157 158 p = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 159 (json_block, errs) = p.communicate() 160 json_block = reallyUTF8(json_block) 161 162 try: 163 return json.loads(json_block) 164 except ValueError: 165 logger.debug("No JSON object came out of codegen: error was %s" % (errs)) 166 return None

167 168

169 -def callm(method, param_dict, POST=False, socket_timeout=None, data=None):

170 """ 171 Call the api! 172 Param_dict is a *regular* *python* *dictionary* so if you want to have multi-valued params 173 put them in a list. 174 175 ** note, if we require 2.6, we can get rid of this timeout munging. 176 """ 177 try: 178 param_dict['api_key'] = config.ECHO_NEST_API_KEY 179 param_list = [] 180 if not socket_timeout: 181 socket_timeout = config.CALL_TIMEOUT 182 183 for key,val in param_dict.iteritems(): 184 if isinstance(val, list): 185 param_list.extend( [(key,subval) for subval in val] ) 186 elif val is not None: 187 if isinstance(val, unicode): 188 val = val.encode('utf-8') 189 param_list.append( (key,val) ) 190 191 params = urllib.urlencode(param_list) 192 193 orig_timeout = socket.getdefaulttimeout() 194 socket.setdefaulttimeout(socket_timeout) 195 196 if(POST): 197 if (not method == 'track/upload') or ((method == 'track/upload') and 'url' in param_dict): 198 """ 199 this is a normal POST call 200 """ 201 url = 'http://%s/%s/%s/%s' % (config.API_HOST, config.API_SELECTOR, 202 config.API_VERSION, method) 203 204 if data is None: 205 data = '' 206 data = urllib.urlencode(data) 207 data = "&".join([data, params]) 208 209 f = opener.open(url, data=data) 210 else: 211 """ 212 upload with a local file is special, as the body of the request is the content of the file, 213 and the other parameters stay on the URL 214 """ 215 url = '/%s/%s/%s?%s' % (config.API_SELECTOR, config.API_VERSION, 216 method, params) 217 218 if ':' in config.API_HOST: 219 host, port = config.API_HOST.split(':') 220 else: 221 host = config.API_HOST 222 port = 80 223 224 if config.TRACE_API_CALLS: 225 logger.info("%s/%s" % (host+':'+str(port), url,)) 226 conn = httplib.HTTPConnection(host, port = port) 227 conn.request('POST', url, body = data, headers = dict([('Content-Type', 'application/octet-stream')]+headers)) 228 f = conn.getresponse() 229 230 else: 231 """ 232 just a normal GET call 233 """ 234 url = 'http://%s/%s/%s/%s?%s' % (config.API_HOST, config.API_SELECTOR, config.API_VERSION, 235 method, params) 236 237 f = opener.open(url) 238 239 socket.setdefaulttimeout(orig_timeout) 240 241 # try/except 242 response_dict = get_successful_response(f) 243 return response_dict 244 245 except IOError, e: 246 if hasattr(e, 'reason'): 247 print 'Failed to reach the Echo Nest server.' 248 print 'Reason: ', e.reason 249 raise EchoNestIOError(error=e.reason) 250 elif hasattr(e, 'code'): 251 print 'Echo Nest server couldn\'t fulfill the request.' 252 print 'Error code: ', e.code 253 raise EchoNestIOError(code=e.code) 254 else: 255 raise

256

257 -def oauthgetm(method, param_dict, socket_timeout=None):

258 try: 259 import oauth2 # lazy import this so oauth2 is not a hard dep 260 except ImportError: 261 raise Exception("You must install the python-oauth2 library to use this method.") 262 263 """ 264 Call the api! With Oauth! 265 Param_dict is a *regular* *python* *dictionary* so if you want to have multi-valued params 266 put them in a list. 267 268 ** note, if we require 2.6, we can get rid of this timeout munging. 269 """ 270 def build_request(url): 271 params = { 272 'oauth_version': "1.0", 273 'oauth_nonce': oauth2.generate_nonce(), 274 'oauth_timestamp': int(time.time()) 275 } 276 consumer = oauth2.Consumer(key=config.ECHO_NEST_CONSUMER_KEY, secret=config.ECHO_NEST_SHARED_SECRET) 277 params['oauth_consumer_key'] = config.ECHO_NEST_CONSUMER_KEY 278 279 req = oauth2.Request(method='GET', url=url, parameters=params) 280 signature_method = oauth2.SignatureMethod_HMAC_SHA1() 281 req.sign_request(signature_method, consumer, None) 282 return req

283 284 param_dict['api_key'] = config.ECHO_NEST_API_KEY 285 param_list = [] 286 if not socket_timeout: 287 socket_timeout = config.CALL_TIMEOUT 288 289 for key,val in param_dict.iteritems(): 290 if isinstance(val, list): 291 param_list.extend( [(key,subval) for subval in val] ) 292 elif val is not None: 293 if isinstance(val, unicode): 294 val = val.encode('utf-8') 295 param_list.append( (key,val) ) 296 297 params = urllib.urlencode(param_list) 298 299 orig_timeout = socket.getdefaulttimeout() 300 socket.setdefaulttimeout(socket_timeout) 301 """ 302 just a normal GET call 303 """ 304 url = 'http://%s/%s/%s/%s?%s' % (config.API_HOST, config.API_SELECTOR, config.API_VERSION, 305 method, params) 306 req = build_request(url) 307 f = opener.open(req.to_url()) 308 309 socket.setdefaulttimeout(orig_timeout) 310 311 # try/except 312 response_dict = get_successful_response(f) 313 return response_dict 314 315

316 -def postChunked(host, selector, fields, files):

317 """ 318 Attempt to replace postMultipart() with nearly-identical interface. 319 (The files tuple no longer requires the filename, and we only return 320 the response body.) 321 Uses the urllib2_file.py originally from 322 http://fabien.seisen.org which was also drawn heavily from 323 http://code.activestate.com/recipes/146306/ . 324 325 This urllib2_file.py is more desirable because of the chunked 326 uploading from a file pointer (no need to read entire file into 327 memory) and the ability to work from behind a proxy (due to its 328 basis on urllib2). 329 """ 330 params = urllib.urlencode(fields) 331 url = 'http://%s%s?%s' % (host, selector, params) 332 u = urllib2.urlopen(url, files) 333 result = u.read() 334 [fp.close() for (key, fp) in files] 335 return result

336 337

338 -def fix(x):

339 # we need this to fix up all the dict keys to be strings, not unicode objects 340 assert(isinstance(x,dict)) 341 return dict((str(k), v) for (k,v) in x.iteritems())

342

Source Code for Module pyechonest.util