multiprocessing and async error: Task was destroyed but it is pending
multiprocessing and async error: Task was destroyed but it is pending
我是 asynco 的新手，但我是多线程和多处理的长期用户。
我真的想弄清楚我在这里做错了什么，因为这个错误出现了。
这是 python 3.7 中完全可运行的代码 returns 错误，因此任何人都可以测试它。谁能告诉我我做错了什么，我的逻辑哪里出错了。我非常感谢反馈。
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
试试这个：
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
#loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
结果：
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://example.org/birthday
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://www.example.com/
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
Process finished with exit code 0
您只能从 SecondClass() 中删除 loop.close()，因为这会阻止脚本 运行 所有任务...
我是 asynco 的新手，但我是多线程和多处理的长期用户。 我真的想弄清楚我在这里做错了什么，因为这个错误出现了。
这是 python 3.7 中完全可运行的代码 returns 错误，因此任何人都可以测试它。谁能告诉我我做错了什么，我的逻辑哪里出错了。我非常感谢反馈。
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
试试这个：
import asyncio
import aiohttp
import logging
from time import sleep
from random import randrange
import multiprocessing as mp
class MainClass:
def gather(self):
urls = ['http://www.example.com/amount/appliance',
'http://example.org/birthday',
'http://www.example.com/']
pool = mp.Pool(1)
data = pool.map(self.tick_tack, urls)
pool.close()
pool.join()
# print("final data")
# print(data)
def tick_tack(self, url):
pid = mp.current_process().pid
crawler = SecondClass(url)
data = crawler.start()
print((pid, data))
return (pid, data)
class SecondClass():
def __init__(self, url):
self.conn_limiter = asyncio.BoundedSemaphore(10)
self.session = aiohttp.ClientSession()
self.url = url
def start(self):
future = asyncio.Task(self.crawl())
loop = asyncio.get_event_loop()
loop.run_until_complete(future)
#loop.close()
result = future.result()
return result
async def crawl(self):
print("- url", self.url)
sleep(randrange(1,5))
html = await self.http_request(self.url)
await self.session.close()
return html
async def http_request(self, url):
"""Makes request on desired page and returns html result"""
async with self.conn_limiter:
try:
async with self.session.get(url, timeout=30) as response:
html = await response.read()
return html
except Exception as e:
logging.warning('Exception at SecondClass.http_request: {}'.format(e))
if __name__ == '__main__':
api = MainClass()
api.gather()
结果：
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://example.org/birthday
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
- url http://www.example.com/
(16171, b'<!doctype html>\n<html>\n<head>\n <title>Example Domain</title>\n\n <meta charset="utf-8" />\n <meta http-equiv="Content-type" content="text/html; charset=utf-8" />\n <meta name="viewport" content="width=device-width, initial-scale=1" />\n <style type="text/css">\n body {\n background-color: #f0f0f2;\n margin: 0;\n padding: 0;\n font-family: "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;\n \n }\n div {\n width: 600px;\n margin: 5em auto;\n padding: 50px;\n background-color: #fff;\n border-radius: 1em;\n }\n a:link, a:visited {\n color: #38488f;\n text-decoration: none;\n }\n @media (max-width: 700px) {\n body {\n background-color: #fff;\n }\n div {\n width: auto;\n margin: 0 auto;\n border-radius: 0;\n padding: 1em;\n }\n }\n </style> \n</head>\n\n<body>\n<div>\n <h1>Example Domain</h1>\n <p>This domain is established to be used for illustrative examples in documents. You may use this\n domain in examples without prior coordination or asking for permission.</p>\n <p><a href="http://www.iana.org/domains/example">More information...</a></p>\n</div>\n</body>\n</html>\n')
Process finished with exit code 0
您只能从 SecondClass() 中删除 loop.close()，因为这会阻止脚本 运行 所有任务...