无法获取应用程序默认凭据。运行在本地

Question

我正在尝试 this 从 GCP Pub/Sub at DataFlow 检索数据的示例。

import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;

import avro.shaded.com.google.common.collect.Lists;
import com.google.auth.oauth2.GoogleCredentials;
import org.apache.beam.runners.dataflow.options.DataflowPipelineOptions;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO;
import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO;
import org.apache.beam.sdk.options.Default;
import org.apache.beam.sdk.options.Description;
import org.apache.beam.sdk.options.PipelineOptionsFactory;
import org.apache.beam.sdk.transforms.DoFn;
import org.apache.beam.sdk.transforms.ParDo;
import org.apache.beam.sdk.transforms.Sum;
import org.apache.beam.sdk.transforms.windowing.SlidingWindows;
import org.apache.beam.sdk.transforms.windowing.Window;
import org.joda.time.Duration;

import com.google.api.services.bigquery.model.TableFieldSchema;
import com.google.api.services.bigquery.model.TableRow;
import com.google.api.services.bigquery.model.TableSchema;

public class StreamDemoConsumer {

    public static interface MyOptions extends DataflowPipelineOptions {
        @Description("Output BigQuery table <project_id>:<dataset_id>.<table_id>")
        @Default.String("coexon-seoul-dev:ledger_data_set.ledger_data2")
        String getOutput();

        void setOutput(String s);

        @Description("Input topic")
        @Default.String("projects/coexon-seoul-dev/topics/trading")
        String getInput();
        
        void setInput(String s);
    }

    @SuppressWarnings("serial")
    public static void main(String[] args) throws IOException {

        MyOptions options = PipelineOptionsFactory.fromArgs(args).withValidation().as(MyOptions.class);

        options.setStreaming(true);
        Pipeline p = Pipeline.create(options);


        String topic = options.getInput();
        String output = options.getOutput();

        // Build the table schema for the output table.
        List<TableFieldSchema> fields = new ArrayList<>();
        fields.add(new TableFieldSchema().setName("timestamp").setType("TIMESTAMP"));
        fields.add(new TableFieldSchema().setName("num_words").setType("INTEGER"));
        TableSchema schema = new TableSchema().setFields(fields);

        p //
                .apply("GetMessages", PubsubIO.readStrings().fromTopic(topic)) //
                .apply("window",
                        Window.into(SlidingWindows//
                                .of(Duration.standardMinutes(2))//
                                .every(Duration.standardSeconds(30)))) //
                .apply("WordsPerLine", ParDo.of(new DoFn<String, Integer>() {
                    @ProcessElement
                    public void processElement(ProcessContext c) throws Exception {
                        String line = c.element();
                        c.output(line.split(" ").length);
                    }
                }))//
                .apply("WordsInTimeWindow", Sum.integersGlobally().withoutDefaults()) //
                .apply("ToBQRow", ParDo.of(new DoFn<Integer, TableRow>() {
                    @ProcessElement
                    public void processElement(ProcessContext c) throws Exception {
                        TableRow row = new TableRow();
                        row.set("timestamp", Instant.now().toString());
                        row.set("num_words", c.element());
                        c.output(row);
                    }
                })) //
                .apply(BigQueryIO.writeTableRows().to(output)//
                        .withSchema(schema)//
                        .withWriteDisposition(BigQueryIO.Write.WriteDisposition.WRITE_APPEND)
                        .withCreateDisposition(BigQueryIO.Write.CreateDisposition.CREATE_IF_NEEDED));

        p.run();
    }
}

我运行此代码使用以下命令。

sh run_oncloud4.sh coexon-seoul-dev ledgerbucket

然后代码运行宁好

run_oncloud4.sh 喜欢下面

#!/bin/bash

if [ "$#" -ne 2 ]; then
   echo "Usage:   ./run_oncloud.sh project-name  bucket-name"
   echo "Example: ./run_oncloud.sh cloud-training-demos  cloud-training-demos"
   exit
fi

PROJECT=
BUCKET=
MAIN=com.google.cloud.training.dataanalyst.javahelp.StreamDemoConsumer

echo "project=$PROJECT  bucket=$BUCKET  main=$MAIN"

export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin/:$PATH
mvn compile -e exec:java \
 -Dexec.mainClass=$MAIN \
      -Dexec.args="--project=$PROJECT \
      --stagingLocation=gs://$BUCKET/staging/ \
      --tempLocation=gs://$BUCKET/staging/ \
      --output=$PROJECT:demos.streamdemo \
      --input=projects/$PROJECT/topics/streamdemo \
      --runner=DataflowRunner"

但我运行上面的代码如下

sh run_locally.sh com.google.cloud.training.dataanalyst.javahelp.StreamDemoConsumer

然后出现无法获取应用程序默认凭据错误消息。

>SLF4J: Failed to load class "org.slf4j.impl.StaticLoggerBinder".
>SLF4J: Defaulting to no-operation (NOP) logger implementation
>SLF4J: See http://www.slf4j.org/codes.html#StaticLoggerBinder for further details.
>Exception in thread "main" java.lang.RuntimeException: Unable to get application default credentials. Please see https://developers.google.com/accounts/docs/application-default-credentials for details on how to specify credentials. This version of the SDK is dependent on the gcloud core component version 2015.02.05 or newer to be able to get credentials from the currently authorized user via gcloud auth.
>   at org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer.throwNullCredentialException(NullCredentialInitializer.java:60)
>   at org.apache.beam.sdk.extensions.gcp.auth.NullCredentialInitializer$NullCredentialHttpUnsuccessfulResponseHandler.handleResponse(NullCredentialInitializer.java:53)
>   at com.google.cloud.hadoop.util.ChainingHttpRequestInitializer.handleResponse(ChainingHttpRequestInitializer.java:111)
>   at com.google.api.client.http.HttpRequest.execute(HttpRequest.java:1015)
>   at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:419)
>   at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.executeUnparsed(AbstractGoogleClientRequest.java:352)
>   at com.google.api.client.googleapis.services.AbstractGoogleClientRequest.execute(AbstractGoogleClientRequest.java:469)
>   at org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl.executeWithRetries(BigQueryServicesImpl.java:854)
>   at org.apache.beam.sdk.io.gcp.bigquery.BigQueryServicesImpl$DatasetServiceImpl.getDataset(BigQueryServicesImpl.java:554)
>   at org.apache.beam.sdk.io.gcp.bigquery.BigQueryHelpers.verifyDatasetPresence(BigQueryHelpers.java:196)
>   at org.apache.beam.sdk.io.gcp.bigquery.BigQueryIO$Write.validate(BigQueryIO.java:1486)
>   at org.apache.beam.sdk.Pipeline$ValidateVisitor.enterCompositeTransform(Pipeline.java:640)
>   at org.apache.beam.sdk.runners.TransformHierarchy$Node.visit(TransformHierarchy.java:656)
>   at org.apache.beam.sdk.runners.TransformHierarchy$Node.visit(TransformHierarchy.java:660)
>   at org.apache.beam.sdk.runners.TransformHierarchy$Node.access0(TransformHierarchy.java:311)
>   at org.apache.beam.sdk.runners.TransformHierarchy.visit(TransformHierarchy.java:245)
>   at org.apache.beam.sdk.Pipeline.traverseTopologically(Pipeline.java:458)
>   at org.apache.beam.sdk.Pipeline.validate(Pipeline.java:575)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:310)
>   at org.apache.beam.sdk.Pipeline.run(Pipeline.java:297)
>   at com.google.cloud.training.dataanalyst.javahelp.StreamDemoConsumer.main(StreamDemoConsumer.java:115)
>
>Process finished with exit code 1

run_locally.sh

#!/bin/bash

if [ "$#" -ne 1 ]; then
   echo "Usage:   ./run_locally.sh mainclass-basename"
   echo "Example: ./run_oncloud.sh Grep"
   exit
fi

MAIN=com.google.cloud.training.dataanalyst.javahelp.

export PATH=/usr/lib/jvm/java-8-openjdk-amd64/bin/:$PATH
mvn compile -e exec:java -Dexec.mainClass=$MAIN

我已设置凭据

echo ${GOOGLE_APPLICATION_CREDENTIALS}

/Users/mattheu/coexon-seoul-dev-898d91a66539.json

但是出现授权错误

我该如何解决这个问题？

Answer 1

我遇到过类似的事情，以下步骤对我有用：

在您的笔记本电脑上安装 Google Cloud SDK。此处说明：https://cloud.google.com/sdk/install
关闭命令行并重新打开它。
运行 gcloud init 并按照说明进行操作，其中包括将 SDK 绑定到您的 GCP 帐户和项目。
按照说明手动设置服务帐户 (https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually)。您只需按照 "Obtaining and providing service account credentials manually." 下的说明进行操作，基本上您将使用您的服务帐户信息将一个文件保存到您的计算机，该文件将用于您正在尝试执行的工作。
在您的 shell 配置文件中（在从 Catalina 开始的 macOS 上，这是 ~/.zshenv），添加行 export GOOGLE_APPLICATION_CREDENTIALS="/path/to/the/file/you/saved/in/step/4"
关闭并重新打开您的 shell，您应该可以开始了。

我有点不确定是否有必要设置 SDK（第 1-3 步），但无论如何进行设置是件好事。

无法获取应用程序默认凭据。运行在本地

Unable to get application default credentials. run on locally

java

dataflow

google-cloud-platform

google-cloud-pubsub

无法获取应用程序默认凭据。 运行 在本地

Unable to get application default credentials. run on locally

java

dataflow

google-cloud-platform

google-cloud-pubsub

无法获取应用程序默认凭据。运行在本地