('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) Python Post 请求不工作
('Connection aborted.', RemoteDisconnected('Remote end closed connection without response')) Python Post requests Not working
我想抓取：
这个 link: https://fccprod.servicenowservices.com/rmd?id=rmd_listings
所以，我检查了网络选项卡，发现 https://fccprod.servicenowservices.com/api/now/sp/widget/2ba6f55c1b72a89089df9796bc4bcb10?id=rmd_listings
这个 link 有我需要的所有数据。但是 post 请求不起作用。我已将所有 headers 添加到 post 请求中。这是我的代码：
from bs4 import BeautifulSoup as bs
import requests as req
headers = {
'Accept':'application/json',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Content-Length':'1906',
'Content-Type':'application/json;charset=UTF-8',
'Cookie':'JSESSIONID=47557C2D51CF5D41467BEB15ABBACAC8; glide_user_route=glide.79dba59629920b67318b94e9ba748ce8; BIGipServerpool_fccprod=2407616266.46142.0000; __CJ_g_startTime=%221642015559308%22',
'Host':'fccprod.servicenowservices.com',
'Origin':'https',
'Pragma':'no-cache',
'Referer':'https',
'sec-ch-ua':'" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"',
'Sec-Fetch-Dest':'empty',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Site':'same-origin',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
'x-portal':'ac2856301b92681048c6ed7bbc4bcb27',
'X-Transaction-Source':'Interface=Service-Portal,Interface-Type=rmd,Interface-SysID=ac2856301b92681048c6ed7bbc4bcb27',
'X-UserToken':'613c20491b894510e91484cae54bcb06b72867334929c3b83555105399d543ca7862b8fd',
}
link ='https://fccprod.servicenowservices.com/api/now/sp/widget/2ba6f55c1b72a89089df9796bc4bcb10?id=rmd_listings'
r = req.post(link,headers = headers)
soup = bs(r.content,'html.parser')
print(soup)
您发送的数据有问题如果您查看请求，您会发现正在发送以下数据，
{"css":"\n#x61108f231be6649089df9796bc4bcb22 .panel {\n\tpadding: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .panel-title {\n\tfont-size: 32px;\n\tfont-weight: 300;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .table > thead:first-child > tr:first-child > th {\n\tborder-left: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .dropdown-toggle {\n\tdisplay: none;\n}","color":"default","show_keywords":"true","roles":"public","sp_column_dv":"","sp_column":"a9f394611b566c1048c6ed7bbc4bcba2","sp_page":"rmd_form","title":"Robocall Mitigation Database","enable_filter":"false","sys_class_name":"sp_instance_table","size_dv":"Medium","window_size":"8","view":"service_portal","hide_header":false,"sys_name":"Robocall Mitigation Database","show_breadcrumbs":false,"table":"x_g_fmc_rmd_robocall_mitigation_database","sp_widget":"75af76231be6649089df9796bc4bcbdd","order":-1,"sys_class_name_dv":"Instance with Table","d":"desc","widget_parameters":"{\n\t\"show_keywords\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"allow_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"view\": {\n\t\t\"value\": \"service_portal\",\n\t\t\"displayValue\": \"service_portal\"\n\t},\n\t\"show_attachment_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"show_breadcrumbs\": {\n\t\t\"value\": false,\n\t\t\"displayValue\": false\n\t},\n\t\"hide_header\": {\n\t\t\"value\": false,\n\t\t\"displayValue\": false\n\t}\n}","active":false,"color_dv":"Default","order_direction":"desc","display_field":"business_address","sys_tags":"","o":"sys_updated_on","filter":"status=Published^EQ","maximum_entries":8,"allow_link":"true","sp_widget_dv":"","size":"md","show_attachment_link":"true","sp_page_dv":"rmd_form","order_by":"sys_updated_on","fields":"","order_direction_dv":"Descending","headerTitle":"Robocall Mitigation Database"}
将此包含在您的请求中将会得到您想要的回复
from bs4 import BeautifulSoup as bs
import requests as req
headers = {
'Accept':'application/json',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Content-Length':'1906',
'Content-Type':'application/json;charset=UTF-8',
'Cookie':'JSESSIONID=47557C2D51CF5D41467BEB15ABBACAC8; glide_user_route=glide.79dba59629920b67318b94e9ba748ce8; BIGipServerpool_fccprod=2407616266.46142.0000; __CJ_g_startTime=%221642015559308%22',
'Host':'fccprod.servicenowservices.com',
'Origin':'https',
'Pragma':'no-cache',
'Referer':'https',
'sec-ch-ua':'" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"',
'Sec-Fetch-Dest':'empty',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Site':'same-origin',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
'x-portal':'ac2856301b92681048c6ed7bbc4bcb27',
'X-Transaction-Source':'Interface=Service-Portal,Interface-Type=rmd,Interface-SysID=ac2856301b92681048c6ed7bbc4bcb27',
'X-UserToken':'a78bb85d1b418910222b0e9ee54bcb7b42f1e8f0a87eeb85195b6c731e5ad46b032c7e8a',
}
link ='https://fccprod.servicenowservices.com/api/now/sp/widget/2ba6f55c1b72a89089df9796bc4bcb10?id=rmd_listings'
r = req.post(link,headers = headers ,json = {"css":"\n#x61108f231be6649089df9796bc4bcb22 .panel {\n\tpadding: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .panel-title {\n\tfont-size: 32px;\n\tfont-weight: 300;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .table > thead:first-child > tr:first-child > th {\n\tborder-left: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .dropdown-toggle {\n\tdisplay: none;\n}","color":"default","show_keywords":"true","roles":"public","sp_column_dv":"","sp_column":"a9f394611b566c1048c6ed7bbc4bcba2","sp_page":"rmd_form","title":"Robocall Mitigation Database","enable_filter":"false","sys_class_name":"sp_instance_table","size_dv":"Medium","window_size":"8","view":"service_portal","hide_header":False,"sys_name":"Robocall Mitigation Database","show_breadcrumbs":False,"table":"x_g_fmc_rmd_robocall_mitigation_database","sp_widget":"75af76231be6649089df9796bc4bcbdd","order":-1,"sys_class_name_dv":"Instance with Table","d":"desc","widget_parameters":"{\n\t\"show_keywords\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"allow_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"view\": {\n\t\t\"value\": \"service_portal\",\n\t\t\"displayValue\": \"service_portal\"\n\t},\n\t\"show_attachment_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"show_breadcrumbs\": {\n\t\t\"value\":False,\n\t\t\"displayValue\": false\n\t},\n\t\"hide_header\": {\n\t\t\"value\": false,\n\t\t\"displayValue\": false\n\t}\n}","active":False,"color_dv":"Default","order_direction":"desc","display_field":"business_address","sys_tags":"","o":"sys_updated_on","filter":"status=Published^EQ","maximum_entries":8,"allow_link":"true","sp_widget_dv":"","size":"md","show_attachment_link":"true","sp_page_dv":"rmd_form","order_by":"sys_updated_on","fields":"","order_direction_dv":"Descending","headerTitle":"Robocall Mitigation Database"})
soup = bs(r.content,'html.parser')
print(soup)
我想抓取： 这个 link: https://fccprod.servicenowservices.com/rmd?id=rmd_listings
所以，我检查了网络选项卡，发现 https://fccprod.servicenowservices.com/api/now/sp/widget/2ba6f55c1b72a89089df9796bc4bcb10?id=rmd_listings 这个 link 有我需要的所有数据。但是 post 请求不起作用。我已将所有 headers 添加到 post 请求中。这是我的代码：
from bs4 import BeautifulSoup as bs
import requests as req
headers = {
'Accept':'application/json',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Content-Length':'1906',
'Content-Type':'application/json;charset=UTF-8',
'Cookie':'JSESSIONID=47557C2D51CF5D41467BEB15ABBACAC8; glide_user_route=glide.79dba59629920b67318b94e9ba748ce8; BIGipServerpool_fccprod=2407616266.46142.0000; __CJ_g_startTime=%221642015559308%22',
'Host':'fccprod.servicenowservices.com',
'Origin':'https',
'Pragma':'no-cache',
'Referer':'https',
'sec-ch-ua':'" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"',
'Sec-Fetch-Dest':'empty',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Site':'same-origin',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
'x-portal':'ac2856301b92681048c6ed7bbc4bcb27',
'X-Transaction-Source':'Interface=Service-Portal,Interface-Type=rmd,Interface-SysID=ac2856301b92681048c6ed7bbc4bcb27',
'X-UserToken':'613c20491b894510e91484cae54bcb06b72867334929c3b83555105399d543ca7862b8fd',
}
link ='https://fccprod.servicenowservices.com/api/now/sp/widget/2ba6f55c1b72a89089df9796bc4bcb10?id=rmd_listings'
r = req.post(link,headers = headers)
soup = bs(r.content,'html.parser')
print(soup)
您发送的数据有问题如果您查看请求，您会发现正在发送以下数据，
{"css":"\n#x61108f231be6649089df9796bc4bcb22 .panel {\n\tpadding: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .panel-title {\n\tfont-size: 32px;\n\tfont-weight: 300;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .table > thead:first-child > tr:first-child > th {\n\tborder-left: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .dropdown-toggle {\n\tdisplay: none;\n}","color":"default","show_keywords":"true","roles":"public","sp_column_dv":"","sp_column":"a9f394611b566c1048c6ed7bbc4bcba2","sp_page":"rmd_form","title":"Robocall Mitigation Database","enable_filter":"false","sys_class_name":"sp_instance_table","size_dv":"Medium","window_size":"8","view":"service_portal","hide_header":false,"sys_name":"Robocall Mitigation Database","show_breadcrumbs":false,"table":"x_g_fmc_rmd_robocall_mitigation_database","sp_widget":"75af76231be6649089df9796bc4bcbdd","order":-1,"sys_class_name_dv":"Instance with Table","d":"desc","widget_parameters":"{\n\t\"show_keywords\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"allow_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"view\": {\n\t\t\"value\": \"service_portal\",\n\t\t\"displayValue\": \"service_portal\"\n\t},\n\t\"show_attachment_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"show_breadcrumbs\": {\n\t\t\"value\": false,\n\t\t\"displayValue\": false\n\t},\n\t\"hide_header\": {\n\t\t\"value\": false,\n\t\t\"displayValue\": false\n\t}\n}","active":false,"color_dv":"Default","order_direction":"desc","display_field":"business_address","sys_tags":"","o":"sys_updated_on","filter":"status=Published^EQ","maximum_entries":8,"allow_link":"true","sp_widget_dv":"","size":"md","show_attachment_link":"true","sp_page_dv":"rmd_form","order_by":"sys_updated_on","fields":"","order_direction_dv":"Descending","headerTitle":"Robocall Mitigation Database"}
将此包含在您的请求中将会得到您想要的回复
from bs4 import BeautifulSoup as bs
import requests as req
headers = {
'Accept':'application/json',
'Accept-Encoding':'gzip, deflate, br',
'Accept-Language':'en-US,en;q=0.9',
'Cache-Control':'no-cache',
'Connection':'keep-alive',
'Content-Length':'1906',
'Content-Type':'application/json;charset=UTF-8',
'Cookie':'JSESSIONID=47557C2D51CF5D41467BEB15ABBACAC8; glide_user_route=glide.79dba59629920b67318b94e9ba748ce8; BIGipServerpool_fccprod=2407616266.46142.0000; __CJ_g_startTime=%221642015559308%22',
'Host':'fccprod.servicenowservices.com',
'Origin':'https',
'Pragma':'no-cache',
'Referer':'https',
'sec-ch-ua':'" Not;A Brand";v="99", "Google Chrome";v="97", "Chromium";v="97"',
'sec-ch-ua-mobile':'?0',
'sec-ch-ua-platform':'"Windows"',
'Sec-Fetch-Dest':'empty',
'Sec-Fetch-Mode':'cors',
'Sec-Fetch-Site':'same-origin',
'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36',
'x-portal':'ac2856301b92681048c6ed7bbc4bcb27',
'X-Transaction-Source':'Interface=Service-Portal,Interface-Type=rmd,Interface-SysID=ac2856301b92681048c6ed7bbc4bcb27',
'X-UserToken':'a78bb85d1b418910222b0e9ee54bcb7b42f1e8f0a87eeb85195b6c731e5ad46b032c7e8a',
}
link ='https://fccprod.servicenowservices.com/api/now/sp/widget/2ba6f55c1b72a89089df9796bc4bcb10?id=rmd_listings'
r = req.post(link,headers = headers ,json = {"css":"\n#x61108f231be6649089df9796bc4bcb22 .panel {\n\tpadding: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .panel-title {\n\tfont-size: 32px;\n\tfont-weight: 300;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .table > thead:first-child > tr:first-child > th {\n\tborder-left: 0;\n}\n\n#x61108f231be6649089df9796bc4bcb22 .panel-heading .dropdown-toggle {\n\tdisplay: none;\n}","color":"default","show_keywords":"true","roles":"public","sp_column_dv":"","sp_column":"a9f394611b566c1048c6ed7bbc4bcba2","sp_page":"rmd_form","title":"Robocall Mitigation Database","enable_filter":"false","sys_class_name":"sp_instance_table","size_dv":"Medium","window_size":"8","view":"service_portal","hide_header":False,"sys_name":"Robocall Mitigation Database","show_breadcrumbs":False,"table":"x_g_fmc_rmd_robocall_mitigation_database","sp_widget":"75af76231be6649089df9796bc4bcbdd","order":-1,"sys_class_name_dv":"Instance with Table","d":"desc","widget_parameters":"{\n\t\"show_keywords\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"allow_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"view\": {\n\t\t\"value\": \"service_portal\",\n\t\t\"displayValue\": \"service_portal\"\n\t},\n\t\"show_attachment_link\": {\n\t\t\"value\": \"true\",\n\t\t\"displayValue\": \"true\"\n\t},\n\t\"show_breadcrumbs\": {\n\t\t\"value\":False,\n\t\t\"displayValue\": false\n\t},\n\t\"hide_header\": {\n\t\t\"value\": false,\n\t\t\"displayValue\": false\n\t}\n}","active":False,"color_dv":"Default","order_direction":"desc","display_field":"business_address","sys_tags":"","o":"sys_updated_on","filter":"status=Published^EQ","maximum_entries":8,"allow_link":"true","sp_widget_dv":"","size":"md","show_attachment_link":"true","sp_page_dv":"rmd_form","order_by":"sys_updated_on","fields":"","order_direction_dv":"Descending","headerTitle":"Robocall Mitigation Database"})
soup = bs(r.content,'html.parser')
print(soup)