使用 Github GraphQL API 从一个 Promise.All 中的不同数组中获取结果

Question

我正在为 Gatsby 制作一个自定义源代码插件，它将从 GitHub 存储库中获取降价文件。存储库有单独的文件（blob）和文件夹（树），它们也包含文件。我需要将所有文件（包括文件夹内的文件）集中在一个 Promise.all 中，但我不知道该怎么做。我已经设法从存储库中获取单个文件，并且我有一个函数 returns 来自树的文件数组。但是不知道怎么组合。

这是我的代码。 GraphQL 查询以获取存储库、树和文件信息：

const repositoryQuery = `
{
  viewer {
    repository(name: "repository-name") {
      object(expression: "master:") {
        ... on Tree {
          entries {
            name
            oid
            type
          }
        }
      }
    }
  }
}
`

const treeQuery = `
  query getTree($id: GitObjectID!) {
    viewer {
      repository(name: "repository-name") {
        object(oid: $id) {
          ... on Tree {
            entries {
              name
              oid
              type
            }
          }
        }
      }
    }
  }
`

const fileQuery = `
  query getFile($id: GitObjectID!) {
    viewer {
      repository(name: "repository-name") {
        object(oid: $id) {
          ... on Blob {
            text
          }
        }
      }
    }
  }
`

以及函数本身：

const data = await client.request(repositoryQuery)

const getTree = async entry => {
  const data = await client.request(treeQuery, { id: entry.oid })
  const array = await data.viewer.repository.object.entries
  return array
}

const getFile = async entry => {
  const data = await client.request(fileQuery, { id: entry.oid })
  const result = await data.viewer.repository.object
  return result
}

const files = await Promise.all(
  data.viewer.repository.object.entries
    .filter(entry => entry.type !== "tree")
    .map(entry => {
      return (
        getFile(entry)
        .then(file => {
          return {
            data: file.text
          }
        })
      )
    }
  )
)

files.forEach(file =>
  createNode({...})
)

如何更新 const files 以便它：

运行getFile(),如果entry.type !== "tree"
如果 entry.type 是 tree，使用 getTree() 获取树中的文件数组，然后为每个文件获取运行 getFile()。
将所有结果合并到一个数组中，以便我可以应用到它们 createNode。

非常感谢您的帮助。

Answer 1

首先你可以运行遍历每棵树，然后得到每棵树的文件数组，这会给你一个二维数组：

.map(async entry => {
  const files = await getTree(entry);
  return Promise.all(
    files.map(file => getFile(file).then(fileRes => ({ data: fileRes.text })))
  );
)

然后需要将结果展平，使其成为一维数组：

const files = allFiles.flat();

希望我已经正确理解了你的问题； getTree() 的结果是文件的一维数组（即 [file1, file2, file3]）而不是多维数组（即 [[file1, file2], [[file1, file2], [file1]], file1, file2]）。

Answer 2

您可以从用于递归遍历目录的 walk 函数中获得一些灵感。来自 there。它看起来像这样：

async function walk(entry, isRoot) {
  if (isRoot){
    return await processEntry(entry);
  }
  let files = await getTreeEntryFromTree(repository, entry.oid);
  files = await Promise.all(files.data.viewer.repository.object.entries.map(async file => {
    return await processEntry(file);
  }));
  return files.reduce((all, folderContents) => all.concat(folderContents), []);
}

async function processEntry(entry){
  if (entry.type === "tree") {
    return walk(entry, false); 
  } else {
    let res = await getBlob(repository, entry.oid);
    return [{
      name: entry.name,
      oid: entry.oid,
      data:res.data.viewer.repository.object.text
    }];
  }
}

因此，它只是用树替换目录，并在 return 文件时请求每个文件的数据内容。

源插件的以下 gatsby-node.js 代码（没有 createSchemaCustomization）：

const { ApolloClient } = require("apollo-client")
const { InMemoryCache } = require("apollo-cache-inmemory")
const { HttpLink } = require("apollo-link-http")
const fetch = require("node-fetch")
const gql = require("graphql-tag")
const { setContext } = require('apollo-link-context');

const token = "YOUR_TOKEN";
const repository = "YOUR_REPO";

const authLink = setContext((_, { headers }) => {
  return {
    headers: {
      ...headers,
      authorization: token ? `Bearer ${token}` : null,
    }
  }
});

const defaultOptions = {
  watchQuery: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'ignore',
  },
  query: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'all',
  },
}

const client = new ApolloClient({
  link: authLink.concat(new HttpLink({ uri: 'https://api.github.com/graphql', fetch: fetch  })),
  cache: new InMemoryCache(),
  defaultOptions: defaultOptions,
});

exports.sourceNodes = async function sourceNodes(
  {
    actions,
    cache,
    createContentDigest,
    createNodeId,
    getNodesByType,
    getNode,
  },
  pluginOptions
) {
  const { createNode, touchNode, deleteNode } = actions
  const { data } = await getTreeFromRepo(repository)

  let sourceData = data;

  fileArr = []
  sourceData.viewer.repository.object.entries.map(it => {
    fileArr.push(walk(it, true))
  });
  let res = await Promise.all(fileArr)
  let result = res.flat();
  console.log(result);
  console.log(`got ${result.length} results`);
  return
}

async function walk(entry, isRoot) {
  if (isRoot){
    return await processEntry(entry);
  }
  let files = await getTreeEntryFromTree(repository, entry.oid);
  files = await Promise.all(files.data.viewer.repository.object.entries.map(async file => {
    return await processEntry(file);
  }));
  return files.reduce((all, folderContents) => all.concat(folderContents), []);
}

async function processEntry(entry){
  if (entry.type === "tree") {
    return walk(entry, false); 
  } else {
    let res = await getBlob(repository, entry.oid);
    return [{
      name: entry.name,
      oid: entry.oid,
      data:res.data.viewer.repository.object.text
    }];
  }
}

async function getTreeFromRepo(repo) {
    return await client.query({
      query: gql`
        query {
          viewer {
            repository(name: "${repo}") {
              object(expression: "master:") {
                ... on Tree {
                  entries {
                    name
                    oid
                    type
                  }
                }
              }
            }
          }
        }
      `,
    })
}

async function getTreeEntryFromTree(repo, oid) {
  return await client.query({
    query: gql`
      query getTree($id: GitObjectID!) {
        viewer {
          repository(name: "${repo}") {
            object(oid: $id) {
              ... on Tree {
                entries {
                  name
                  oid
                  type
                }
              }
            }
          }
        }
      }
    `,
    variables: {
      id: oid
    }
  })
}

async function getBlob(repo, oid){
  return await client.query({
    query: gql`
      query getFile($id: GitObjectID!) {
        viewer {
          repository(name: "${repo}") {
            object(oid: $id) {
              ... on Blob {
                text
              }
            }
          }
        }
      }
    `,
    variables: {
      id: oid
    }
  })
}

您需要替换上面代码中的 Github 令牌和存储库名称。

它return是一个包含文件内容、名称和 oid 的对象数组

注意使用 ... on Blob { text } returns null for binary file :

text (String) UTF8 text data or null if the Blob is binary

此外，可以使用 Github API v3 在单个调用中递归地遍历树，从而大大减少请求的数量。你会得到这样的东西：

async function getAllEntries(repo, owner){
  return fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`,{
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

完整示例（针对 Gatsby 源插件）：

const { ApolloClient } = require("apollo-client")
const { InMemoryCache } = require("apollo-cache-inmemory")
const { HttpLink } = require("apollo-link-http")
const fetch = require("node-fetch")
const gql = require("graphql-tag")
const { setContext } = require('apollo-link-context');

const token = "YOUR_TOKEN";
const repository = "YOUR_REPO";
const owner = "YOUR_LOGIN";

const authLink = setContext((_, { headers }) => {
  return {
    headers: {
      ...headers,
      authorization: token ? `Bearer ${token}` : null,
    }
  }
});

const defaultOptions = {
  watchQuery: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'ignore',
  },
  query: {
    fetchPolicy: 'no-cache',
    errorPolicy: 'all',
  },
}

const client = new ApolloClient({
  link: authLink.concat(new HttpLink({ uri: 'https://api.github.com/graphql', fetch: fetch  })),
  cache: new InMemoryCache(),
  defaultOptions: defaultOptions,
});

exports.sourceNodes = async function sourceNodes(
  {
    actions,
    cache,
    createContentDigest,
    createNodeId,
    getNodesByType,
    getNode,
  },
  pluginOptions
) {
  const { createNode, touchNode, deleteNode } = actions
  const { tree } = await getAllEntries(repository, owner)
  fileArr = []
  tree.map(it => {
    fileArr.push(walk(it, true))
  });
  let res = await Promise.all(fileArr)
  let result = res.filter(value => Object.keys(value).length !== 0);
  console.log(result);
  console.log(`got ${result.length} results`);
  return
}

async function walk(entry){
  if (entry.type === "blob") {
    let res = await getBlob(repository, entry.sha);
    return {
      name: entry.path,
      oid: entry.sha,
      data: res.data.viewer.repository.object.text
    };
  }
  return {};
}

async function getAllEntries(repo, owner){
  return fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`,{
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

async function getBlob(repo, oid){
  return await client.query({
    query: gql`
      query getFile($id: GitObjectID!) {
        viewer {
          repository(name: "${repo}") {
            object(oid: $id) {
              ... on Blob {
                text
              }
            }
          }
        }
      }
    `,
    variables: {
      id: oid
    }
  })
}

如果您需要不惜一切代价获取二进制内容，则需要使用 Github API v3 直接在获取树结果中给出内容 url。内容URLreturn是base64编码的内容，见this file.

所以，如果你想要 base64 编码的内容（二进制 + 文本），你将有以下 gatsby-node.js（对于源插件）：

const fetch = require("node-fetch")

const token = "YOUR_TOKEN";
const repository = "YOUR_REPO";
const owner = "YOUR_LOGIN";

exports.sourceNodes = async function sourceNodes(
  {
    actions,
    cache,
    createContentDigest,
    createNodeId,
    getNodesByType,
    getNode,
  },
  pluginOptions
) {
  const { createNode, touchNode, deleteNode } = actions
  const { tree } = await getAllEntries(repository, owner)
  fileArr = []
  tree.map(it => {
    fileArr.push(walk(it, true))
  });
  let res = await Promise.all(fileArr)
  console.log(res);
  console.log(`got ${res.length} results`);
  return
}

async function walk(entry){
  if (entry.type === "blob") {
    let res = await getBlob(entry.url);
    return {
      name: entry.path,
      oid: entry.sha,
      data: res.content
    };
  }
  return {};
}

async function getAllEntries(repo, owner){
  return fetch(`https://api.github.com/repos/${owner}/${repo}/git/trees/master?recursive=1`, {
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

async function getBlob(url){
  return fetch(url, {
    headers: {
      'Authorization': `Bearer ${token}`,
    }
  })
  .then(response => response.json());
}

使用 Github GraphQL API 从一个 Promise.All 中的不同数组中获取结果

Get results from different arrays in one Promise.All with Github GraphQL API

javascript

github-api

graphql

gatsby