从 COSMIC 数据库中抓取 table 个突变

Question

我很难使用 rvest 从 COSMIC 数据库中抓取 MYC 基因中的 table 个突变。我只得到一个空列表。输出如下。我已经确认这些值在 HTML 文件中（即，不是 JAVA，因为这些值在 HTML 文件本身中）。我还确认我要抓取的元素是 table。我试过使用 xpath 和 CSS。我也已确认允许抓取。请指教

R 控制台输出

> library("rvest")
> library("dplyr")
> library("robotstxt")
> library("XML")
> library("RSelenium")
> library("splashr")
> library("reticulate")
> url = "https://cancer.sanger.ac.uk/cosmic/gene/analysis?ln=MYC#variants"
> paths_allowed(url)
 cancer.sanger.ac.uk                      No encoding supplied: defaulting to UTF-8.


[1] TRUE
> Xpath = "//*[@id= 'DataTables_Table_0']"
> a = read_html(url) %>% html_nodes(xpath = Xpath) %>% html_table()
> a
list()
> Selector = "#DataTables_Table_0"
> a = read_html(url) %>% html_nodes(css = Selector) %>% html_table()
> a
list()

Answer 1

您可以执行与网页相同的请求以动态检索结果（可在开发工具 F12 的网络选项卡中查看。）将 DisplayLength 参数更改为所有结果 (1394)或将其设置为初始高值并检查 return 以捕获实际的总结果计数并发出获取所有结果所需的任何进一步请求。

虽然你可以做一个简单的 rvest 请求

library(rvest)

url <- 'https://cancer.sanger.ac.uk/cosmic/gene/mutations?all_data=&coords=AA%3AAA&dr=&end=455&gd=&id=359910&ln=MYC&seqlen=455&src=gene&start=1&export=json&sEcho=2&iColumns=6&sColumns=&iDisplayStart=0&iDisplayLength=1394&mDataProp_0=0&sSearch_0=&bRegex_0=false&bSearchable_0=true&bSortable_0=true&mDataProp_1=1&sSearch_1=&bRegex_1=false&bSearchable_1=true&bSortable_1=true&mDataProp_2=2&sSearch_2=&bRegex_2=false&bSearchable_2=true&bSortable_2=true&mDataProp_3=3&sSearch_3=&bRegex_3=false&bSearchable_3=true&bSortable_3=true&mDataProp_4=4&sSearch_4=&bRegex_4=false&bSearchable_4=true&bSortable_4=true&mDataProp_5=5&sSearch_5=&bRegex_5=false&bSearchable_5=true&bSortable_5=true&sSearch=&bRegex=false&iSortCol_0=0&sSortDir_0=asc&iSortingCols=1'

r <- read_html(url) %>% html_node('p') %>% html_text()

write.table(r,file="data.txt", sep='\t', row.names = FALSE)

编辑@Snehal Patel 的上述内容以获得所需的格式：

x = read.table("data.txt", sep = "\t", skip = 2, fill = TRUE) 
colnames(x) = c("AA_Position", "CDS_Mutation", "AA_Mutation", "COSMIC_ID", "count", "Mutation_type")

使用 httr，传递各种 headers，并根据响应构建数据帧。

library(httr)
library(purrr)
library(rvest)

headers = c(
  'X-Requested-With' = 'XMLHttpRequest',
  'User-Agent' = 'Mozilla/5.0',
  'Referer' = 'https://cancer.sanger.ac.uk/cosmic/gene/analysis?ln=MYC'
)

params = list(
  'coords' = 'AA:AA',
  'end' = '455',
  'id' = '359910',
  'ln' = 'MYC',
  'seqlen' = '455',
  'src' = 'gene',
  'start' = '1',
  'export' = 'json',
  'sEcho' = '4',
  'iColumns' = '6',
  'iDisplayStart' = '0',
  'iDisplayLength' = '1394', #for all results. You can set to number higher than you expect then check first result for actual
  'mDataProp_0' = '0',
  'bRegex_0' = 'false',
  'bSearchable_0' = 'true',
  'bSortable_0' = 'true',
  'mDataProp_1' = '1',
  'bRegex_1' = 'false',
  'bSearchable_1' = 'true',
  'bSortable_1' = 'true',
  'mDataProp_2' = '2',
  'bRegex_2' = 'false',
  'bSearchable_2' = 'true',
  'bSortable_2' = 'true',
  'mDataProp_3' = '3',
  'bRegex_3' = 'false',
  'bSearchable_3' = 'true',
  'bSortable_3' = 'true',
  'mDataProp_4' = '4',
  'bRegex_4' = 'false',
  'bSearchable_4' = 'true',
  'bSortable_4' = 'true',
  'mDataProp_5' = '5',
  'bRegex_5' = 'false',
  'bSearchable_5' = 'true',
  'bSortable_5' = 'true',
  'bRegex' = 'false',
  'iSortCol_0' = '0',
  'sSortDir_0' = 'asc',
  'iSortingCols' = '1'
)

r <- content(httr::GET(url = 'https://cancer.sanger.ac.uk/cosmic/gene/mutations', httr::add_headers(.headers=headers), query = params)) %>% 
      .$aaData

df <- map_df(r, function(i) {

  data.frame(
   `Position` = read_html(i[[1]]) %>% html_node('a') %>% html_text() %>% as.numeric() ,
   `CDS Mutation` = read_html(i[[2]]) %>% html_node('a') %>% html_text(),
    `AA Mutation` = read_html(i[[3]]) %>% html_node('a') %>% html_text(),
   `Legacy Mutation ID` = i[[4]],
      `Count` = i[[5]] ,
      `Type` = i[[6]] , 
      stringsAsFactors=FALSE)
})

从 COSMIC 数据库中抓取 table 个突变

web scraping a table of mutations from the COSMIC database

web-scraping

rvest