how do POST API requests use parallel processing in python? requests.exceptions.ConnectionError:
how do POST API requests use parallel processing in python? requests.exceptions.ConnectionError:
我有这样的代码：
import requests
import multiprocessing as mp
import json
import time
BASE_URL = 'http://127.0.0.1:3000/employees'
with open('data.json', 'r') as f:
array = json.load(f)
def internet_resource_getter(post_data):
stuff_got = []
response = requests.post(BASE_URL, json=post_data)
stuff_got.append(response.json())
print(stuff_got)
time.sleep(1)
return stuff_got
if __name__ == '__main__':
# freeze_support() here if program needs to be frozen
start=time.time()
with mp.Pool(mp.cpu_count()) as pool:
pool.map(internet_resource_getter, array)
elapsed = (time.time() - start)
print("\n","time elapsed is :", elapsed)
在文件data.json中包含500条记录，例如：
[{"first_name":"John","last_name":"Swen"},{"first_name":"Ricard","last_name":"Candra"}]
在BASE_URL中有这样的数据：
[
{
"id": 1,
"first_name": "Sebastian",
"last_name": "Eschweiler"
},
{
"id": 2,
"first_name": "Steve",
"last_name": "Palmer"
},
{
"id": 3,
"first_name": "Ann",
"last_name": "Smith"
}
]
POST API 后的预期输出：
[
{
"id": 1,
"first_name": "Sebastian",
"last_name": "Eschweiler"
},
{
"id": 2,
"first_name": "Steve",
"last_name": "Palmer"
},
{
"id": 3,
"first_name": "Ann",
"last_name": "Smith"
},
{
"id": 4,
"first_name": "John",
"last_name": "Swen"
},
{
"id": 5,
"first_name": "Ricard",
"last_name": "Candra"
},
]
用我上面的代码，进入url的数据只有420条记录，尽管我的data.json是500条记录。我该如何解决这个问题，以便 post 500 条记录到 url。不知道为什么只处理了400条数据。
我有这样的错误：
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\retry.py", line 531, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "paralel_pro.py", line 28, in <module>
pool.map(internet_resource_getter, array)
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\multiprocessing\pool.py", line 364, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\multiprocessing\pool.py", line 771, in get
raise self._value
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
要获得答案，请检查您的服务器日志。我怀疑错误数据 and/or 服务器没有正常处理意外数据。
客户端检查
看看这80条记录是否总是相同的80条记录。
服务器端检查
您能在 http://127.0.0.1:3000/employees 上找到 运行 服务器的日志吗？看起来它在您的同一台机器上是 运行。服务器的应用程序或服务日志应该有确凿证据。
我有这样的代码：
import requests
import multiprocessing as mp
import json
import time
BASE_URL = 'http://127.0.0.1:3000/employees'
with open('data.json', 'r') as f:
array = json.load(f)
def internet_resource_getter(post_data):
stuff_got = []
response = requests.post(BASE_URL, json=post_data)
stuff_got.append(response.json())
print(stuff_got)
time.sleep(1)
return stuff_got
if __name__ == '__main__':
# freeze_support() here if program needs to be frozen
start=time.time()
with mp.Pool(mp.cpu_count()) as pool:
pool.map(internet_resource_getter, array)
elapsed = (time.time() - start)
print("\n","time elapsed is :", elapsed)
在文件data.json中包含500条记录，例如：
[{"first_name":"John","last_name":"Swen"},{"first_name":"Ricard","last_name":"Candra"}]
在BASE_URL中有这样的数据：
[
{
"id": 1,
"first_name": "Sebastian",
"last_name": "Eschweiler"
},
{
"id": 2,
"first_name": "Steve",
"last_name": "Palmer"
},
{
"id": 3,
"first_name": "Ann",
"last_name": "Smith"
}
]
POST API 后的预期输出：
[
{
"id": 1,
"first_name": "Sebastian",
"last_name": "Eschweiler"
},
{
"id": 2,
"first_name": "Steve",
"last_name": "Palmer"
},
{
"id": 3,
"first_name": "Ann",
"last_name": "Smith"
},
{
"id": 4,
"first_name": "John",
"last_name": "Swen"
},
{
"id": 5,
"first_name": "Ricard",
"last_name": "Candra"
},
]
用我上面的代码，进入url的数据只有420条记录，尽管我的data.json是500条记录。我该如何解决这个问题，以便 post 500 条记录到 url。不知道为什么只处理了400条数据。 我有这样的错误：
multiprocessing.pool.RemoteTraceback:
"""
Traceback (most recent call last):
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
http.client.RemoteDisconnected: Remote end closed connection without response
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\requests\adapters.py", line 439, in send
resp = conn.urlopen(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 755, in urlopen
retries = retries.increment(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\util\retry.py", line 531, in increment
raise six.reraise(type(error), error, _stacktrace)
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\packages\six.py", line 734, in reraise
raise value.with_traceback(tb)
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 699, in urlopen
httplib_response = self._make_request(
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 445, in _make_request
six.raise_from(e, None)
File "<string>", line 3, in raise_from
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\site-packages\urllib3\connectionpool.py", line 440, in _make_request
httplib_response = conn.getresponse()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 1347, in getresponse
response.begin()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 307, in begin
version, status, reason = self._read_status()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\http\client.py", line 276, in _read_status
raise RemoteDisconnected("Remote end closed connection without"
urllib3.exceptions.ProtocolError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "paralel_pro.py", line 28, in <module>
pool.map(internet_resource_getter, array)
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\multiprocessing\pool.py", line 364, in map
return self._map_async(func, iterable, mapstar, chunksize).get()
File "C:\Users\admin\AppData\Local\Programs\Python\Python38\lib\multiprocessing\pool.py", line 771, in get
raise self._value
requests.exceptions.ConnectionError: ('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))
要获得答案，请检查您的服务器日志。我怀疑错误数据 and/or 服务器没有正常处理意外数据。
客户端检查
看看这80条记录是否总是相同的80条记录。
服务器端检查
您能在 http://127.0.0.1:3000/employees 上找到 运行 服务器的日志吗？看起来它在您的同一台机器上是 运行。服务器的应用程序或服务日志应该有确凿证据。