1. 多线程处理IO密集型任务优势

1.1 GIL对python多线程的影响？

编写一个多线程抓取网页的程序

import requests
import threading
import timedef time_decorator(func):def wrapper(*args, **kwargs):start = time.time()res = func(*args, **kwargs)end = time.time()print("run func {0} used time for {1}".format(func.__name__, end-start))return resreturn wrapper
def request_url(url):try:response = requests.get(url)print("response status code", response.status_code)except Exception as e:print("error fetch",  e)
@time_decorator
def multi_thread(urls):ts = []for url in urls:t = threading.Thread(target=request_url, args=url)t.start()ts.append(t)for t in ts:t.join()
@time_decorator
def single_fetch(urls):for url in urls:request_url(url)@time_decorator
def multi_process(urls):with multiprocessing.Pool(processes=len(urls)) as pool:results = pool.map(request_url, urls)
@time_decorator
def multi_process(urls):processes = []for url in urls:p = multiprocessing.Process(target=request_url, args=(url,))processes.append(p)p.start()for p in processes:p.join()
if__name__ == '__main__':urls = ["http://example.com","https://httpbin.org/get","https://www.python.org","https://www.google.com"]single_fetch(urls)multi_process(urls)multi_thread(urls)

run func single_fetch used time for 6.238506555557251
run func mult