针对同一集群的多个 kafka 模式注册表

Question

我正在尝试针对同一个 kafka 和 zookeeper 集群安装两个 kafka 模式注册表实例。但 shemas 越来越混杂。当运行这两个注册表时，如果我使用 api "kafka-schema-registry" 注册一个模式，它似乎是在 "schema-registry-ui-other" 处创建的，而不是按预期在 "kafka-schema-registry-ui" 处显示。我的配置是：

version: '2.1'

services:
  zoo1:
    image: zookeeper:3.4.9
    restart: unless-stopped
    hostname: zoo1
    ports:
      - "2181:2181"
    environment:
        ZOO_MY_ID: 1
        ZOO_PORT: 2181
        ZOO_SERVERS: server.1=zoo1:2888:3888
    volumes:
      - ./full-stack/zoo1/data:/data
      - ./full-stack/zoo1/datalog:/datalog


  kafka1:
    image: confluentinc/cp-kafka:5.3.1
    hostname: kafka1
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
      KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
      KAFKA_BROKER_ID: 1
      KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO"
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    volumes:
      - ./full-stack/kafka1/data:/var/lib/kafka/data
    depends_on:
      - zoo1

  kafka-schema-registry:
    image: confluentinc/cp-schema-registry:5.3.1
    hostname: kafka-schema-registry
    ports:
      - "8081:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID: "schema-registry"
    depends_on:
      - zoo1
      - kafka1
  kafka-schema-registry-other:
    image: confluentinc/cp-schema-registry:5.3.1
    hostname: kafka-schema-registry-other
    ports:
      - "8092:8081"
    environment:
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
      SCHEMA_REGISTRY_HOST_NAME: kafka-schema-registry-other
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_SCHEMA_REGISTRY_ZK_NAMESPACE: schema_registry_other
      SCHEMA_REGISTRY_KAFKASTORE_TOPIC: "_schemas_other"
      SCHEMA_REGISTRY_KAFKASTORE_GROUP_ID: "schema-registry-other"
    depends_on:
      - zoo1
      - kafka1

  schema-registry-ui:
    image: landoop/schema-registry-ui:0.9.4
    hostname: kafka-schema-registry-ui
    ports:
      - "8001:8000"
    environment:
      SCHEMAREGISTRY_URL: http://kafka-schema-registry:8081/
      PROXY: "true"
    depends_on:
      - kafka-schema-registry

  schema-registry-ui-other:
    image: landoop/schema-registry-ui:0.9.4
    hostname: kafka-schema-registry-ui-other
    ports:
      - "8002:8000"
    environment:
      SCHEMAREGISTRY_URL: http://kafka-schema-registry-other:8081/
      PROXY: "true"
    depends_on:
      - kafka-schema-registry-other

  kafka-rest-proxy:
    image: confluentinc/cp-kafka-rest:5.3.1
    hostname: kafka-rest-proxy
    ports:
      - "8082:8082"
    environment:
      # KAFKA_REST_ZOOKEEPER_CONNECT: zoo1:2181
      KAFKA_REST_LISTENERS: http://0.0.0.0:8082/
      KAFKA_REST_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081/
      KAFKA_REST_HOST_NAME: kafka-rest-proxy
      KAFKA_REST_BOOTSTRAP_SERVERS: PLAINTEXT://kafka1:19092
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry

  kafka-topics-ui:
    image: landoop/kafka-topics-ui:0.9.4
    hostname: kafka-topics-ui
    ports:
      - "8000:8000"
    environment:
      KAFKA_REST_PROXY_URL: "http://kafka-rest-proxy:8082/"
      PROXY: "true"
    depends_on:
      - zoo1
      - kafka1
      - kafka-schema-registry
      - kafka-rest-proxy
  zoonavigator-web:
    image: elkozmon/zoonavigator-web:0.5.1
    ports:
     - "8004:8000"
    environment:
      API_HOST: "zoonavigator-api"
      API_PORT: 9000
    links:
     - zoonavigator-api
    depends_on:
     - zoonavigator-api

  zoonavigator-api:
    image: elkozmon/zoonavigator-api:0.5.1
    environment:
      SERVER_HTTP_PORT: 9000
    depends_on:
      - zoo1

不可能有两个单独的架构注册表？

Answer 1

您为 Schema Registry 服务器定义的组 ID 不正确，这意味着它们在同一个使用者组中，这意味着它们认为自己是同一个集群的一部分。第二个 Schema Registry 服务器成为主要服务器，因此执行所有写入。

您可以通过为两个 Schema Registry 服务器设置环境变量 SCHEMA_REGISTRY_SCHEMA_REGISTRY_GROUP_ID 来解决这个问题。然后它们将被视为两个不同的集群，这将按您预期的那样工作。

针对同一集群的多个 kafka 模式注册表

multiple kafka schema registry against same cluster

apache-kafka

confluent-schema-registry