lib.py (3236B)
1 from __future__ import print_function 2 import os 3 import sys 4 import locale 5 import random 6 import time 7 import signal 8 from contextlib import contextmanager 9 10 import googleapiclient.errors 11 12 @contextmanager 13 def default_sigint(): 14 original_sigint_handler = signal.getsignal(signal.SIGINT) 15 signal.signal(signal.SIGINT, signal.SIG_DFL) 16 try: 17 yield 18 finally: 19 signal.signal(signal.SIGINT, original_sigint_handler) 20 21 def get_encoding(): 22 return locale.getpreferredencoding() 23 24 def to_utf8(s): 25 """Re-encode string from the default system encoding to UTF-8.""" 26 current = locale.getpreferredencoding() 27 if hasattr(s, 'decode'): #Python 3 workaround 28 return (s.decode(current).encode("UTF-8") if s and current != "UTF-8" else s) 29 elif isinstance(s, bytes): 30 return bytes.decode(s) 31 else: 32 return s 33 34 def debug(obj, fd=sys.stderr): 35 """Write obj to standard error.""" 36 print(obj, file=fd) 37 38 def catch_exceptions(exit_codes, fun, *args, **kwargs): 39 """ 40 Catch exceptions on fun(*args, **kwargs) and return the exit code specified 41 in the exit_codes dictionary. Return 0 if no exception is raised. 42 """ 43 try: 44 fun(*args, **kwargs) 45 return 0 46 except tuple(exit_codes.keys()) as exc: 47 debug("[{0}] {1}".format(exc.__class__.__name__, exc)) 48 return exit_codes[exc.__class__] 49 50 def first(it): 51 """Return first element in iterable.""" 52 return it.next() 53 54 def string_to_dict(string): 55 """Return dictionary from string "key1=value1, key2=value2".""" 56 if string: 57 pairs = [s.strip() for s in string.split(",")] 58 return dict(pair.split("=") for pair in pairs) 59 60 def get_first_existing_filename(prefixes, relative_path): 61 """Get the first existing filename of relative_path seeking on prefixes directories.""" 62 for prefix in prefixes: 63 path = os.path.join(prefix, relative_path) 64 if os.path.exists(path): 65 return path 66 67 def retriable_exceptions(fun, retriable_exceptions, max_retries=None): 68 """Run function and retry on some exceptions (with exponential backoff).""" 69 retry = 0 70 while 1: 71 try: 72 return fun() 73 except tuple(retriable_exceptions) as exc: 74 retry += 1 75 if type(exc) not in retriable_exceptions: 76 raise exc 77 # we want to retry 5xx errors only 78 elif type(exc) == googleapiclient.errors.HttpError and exc.resp.status < 500: 79 raise exc 80 elif max_retries is not None and retry > max_retries: 81 debug("[Retryable errors] Retry limit reached") 82 raise exc 83 else: 84 seconds = random.uniform(0, 2**retry) 85 message = ("[Retryable error {current_retry}/{total_retries}] " + 86 "{error_type} ({error_msg}). Wait {wait_time} seconds").format( 87 current_retry=retry, 88 total_retries=max_retries or "-", 89 error_type=type(exc).__name__, 90 error_msg=str(exc) or "-", 91 wait_time="%.1f" % seconds, 92 ) 93 debug(message) 94 time.sleep(seconds)