docker · 2024-04-30 0

docker-compose 搭建 mongodb 分片集群

一、搭建

分片集群的构造：

  1. mongos ：数据路由，和客户端打交道的模块。mongos 本身没有任何数据，他也不知道该怎么处理这数据，去找 config server
  2. config server：所有存、取数据的方式，所有 shard 节点的信息，分片功能的一些配置信息。可以理解为真实数据的元数据
  3. shard：真正的数据存储位置，以 chunk 为单位存数据

mongos 本身并不持久化数据，sharded cluster 所有的元数据都会存储到 config server，而用户的数据会议分散存储到各个 shard。mongos启动后，会从配置服务器加载元数据，开始提供服务，将用户的请求正确路由到对应的碎片。

mongos的路由功能，当数据写入时，MongoDB Cluster根据分片键设计写入数据。

MongoDB 使用的默认TCP端口：

  • 27017：mongod 和 mongos 实例的默认端口
  • 27018：mongod 使用 --shardsvr 命令行选项运行时 的默认端口或 配置文件中设置的 shardsvr 值 clusterRole
  • 27019：mongod 使用 --configsvr 命令行选项运行时 的默认端口或 配置文件中设置的 configsvr 值 clusterRole

1.拉取镜像

docker pull mongo:4.2.21

2.使用 openssl 生成 keyFile 文件

生成 keyFile 文件：

openssl rand -base64 756 > key.file

修改 keyFile 文件权限：

chmod 400 key.file
chown 999 key.file

若不修改 keyFile 文件权限，会出现 permissions on /etc/key.file are too open，这是因为 mongo key 文件权限过大造成的。

3.docker compose 启动容器

docker-compose.yml 文件：

version: "2"
services:
  rs_config_server01:
    image: mongo:4.2.21
    container_name: rs_config_server01
    restart: always
    ports:
      - 27019:27019
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./rs_config_server01/data/db:/data/db
      - ./rs_config_server01/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    command: --configsvr --keyFile /etc/key.file --replSet rs_config
    networks:
      mongo:
        ipv4_address: 172.18.1.11

  rs_config_server02:
    image: mongo:4.2.21
    container_name: rs_config_server02
    restart: always
    ports:
      - 27029:27019
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./rs_config_server02/data/db:/data/db
      - ./rs_config_server02/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    command: --configsvr --keyFile /etc/key.file --replSet rs_config
    networks:
      mongo:
        ipv4_address: 172.18.1.12

  shard_server01:
    image: mongo:4.2.21
    container_name: shard_server01
    restart: always
    ports:
      - 27018:27018
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./shard_server01/data/db:/data/db
      - ./shard_server01/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    command: --shardsvr --keyFile /etc/key.file --replSet shard1
    depends_on:
      - rs_config_server01
      - rs_config_server02
    networks:
      mongo:
        ipv4_address: 172.18.1.21

  shard_server02:
    image: mongo:4.2.21
    container_name: shard_server02
    restart: always
    ports:
      - 27028:27018
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./shard_server02/data/db:/data/db
      - ./shard_server02/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    command: --shardsvr --keyFile /etc/key.file --replSet shard1
    depends_on:
      - rs_config_server01
      - rs_config_server02
    networks:
      mongo:
        ipv4_address: 172.18.1.22

  shard_server03:
    image: mongo:4.2.21
    container_name: shard_server03
    restart: always
    ports:
      - 27038:27018
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./shard_server03/data/db:/data/db
      - ./shard_server03/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    command: --shardsvr --keyFile /etc/key.file --replSet shard2
    depends_on:
      - rs_config_server01
      - rs_config_server02
    networks:
      mongo:
        ipv4_address: 172.18.1.23

  shard_server04:
    image: mongo:4.2.21
    container_name: shard_server04
    restart: always
    ports:
      - 27048:27018
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./shard_server04/data/db:/data/db
      - ./shard_server04/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    command: --shardsvr --keyFile /etc/key.file --replSet shard2
    depends_on:
      - rs_config_server01
      - rs_config_server02
    networks:
      mongo:
        ipv4_address: 172.18.1.24

  mongos_01:
    image: mongo:4.2.21
    container_name: mongos_01
    restart: always
    ports:
      - 27017:27017
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./mongos_01/data/db:/data/db
      - ./mongos_01/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    entrypoint: mongos
    command: --configdb rs_config/rs_config_server01:27019,rs_config_server02:27019 --keyFile /etc/key.file --bind_ip_all
    depends_on:
      - rs_config_server01
      - rs_config_server02
    networks:
      mongo:
        ipv4_address: 172.18.1.31

  mongos_02:
    image: mongo:4.2.21
    container_name: mongos_02
    restart: always
    ports:
      - 27027:27017
    environment:
      - MONGO_INITDB_ROOT_USERNAME=root
      - MONGO_INITDB_ROOT_PASSWORD=123456
    volumes:
      - ./mongos_02/data/db:/data/db
      - ./mongos_02/data/configdb:/data/configdb
      - ./key.file:/etc/key.file
    entrypoint: mongos
    command: --configdb rs_config/rs_config_server01:27019,rs_config_server02:27019 --keyFile /etc/key.file --bind_ip_all
    depends_on:
      - rs_config_server01
      - rs_config_server02
    networks:
      mongo:
        ipv4_address: 172.18.1.32

networks:
  mongo:
    driver: bridge
    ipam:
      config:
        - subnet: 172.18.1.0/24

进入 config 容器，可以看到实际启动命令，如下：

zxm@zxm-pc:~$ docker exec -it rs_config_server01 ps -ef
UID          PID    PPID  C STIME TTY          TIME CMD
mongodb        1       0  1 14:20 ?        00:00:30 mongod --configsvr --keyFile /etc/key.file --replSet rs_config --auth --bind_ip_all

进入 shard 容器，可以看到实际启动命令，如下：

zxm@zxm-pc:~$ docker exec -it shard_server01 ps -ef
UID          PID    PPID  C STIME TTY          TIME CMD
mongodb        1       0  1 14:20 ?        00:00:27 mongod --shardsvr --keyFile /etc/key.file --replSet shard1 --auth --bind_ip_all

进入 mongos 容器，可以看到实际启动命令，如下：

zxm@zxm-pc:~$ docker exec -it mongos_01 ps -ef
UID          PID    PPID  C STIME TTY          TIME CMD
root           1       0  0 15:04 ?        00:00:00 mongos --configdb rs_config/rs_config_server01:27019,rs_config_server02:27019 --keyFile /etc/key.file --bind_ip_all

4.初始化 config server 副本集

使用 rs.initiate，进行初始化 config server 副本集

rs.initiate({
    "_id" : "rs_config",
    "members" : [
        {
            "_id" : 0,
            "host" : "rs_config_server01:27019"
        },
        {
            "_id" : 1,
            "host" : "rs_config_server02:27019"
        }
    ]
}
)

示例：

zxm@zxm-pc:~$ docker exec -it rs_config_server01 mongo --port 27019
> use admin
switched to db admin
> db.auth('root', '123456')
1
> rs.initiate({
... "_id" : "rs_config",
... "members" : [
... {
... "_id" : 0,
... "host" : "rs_config_server01:27019"
... },
... {
... "_id" : 1,
... "host" : "rs_config_server02:27019"
... }
... ]
... }
... )
{
    "ok" : 1,
    "$gleStats" : {
        "lastOpTime" : Timestamp(1716825534, 1),
        "electionId" : ObjectId("000000000000000000000000")
    },
    "lastCommittedOpTime" : Timestamp(0, 0)
}
rs_config:SECONDARY> 
rs_config:SECONDARY> exit
bye

5.初始化 shard 副本集

使用 rs.initiate，进行初始化 config server 副本集

1) shard1

rs.initiate({
    "_id" : "shard1",
    "members" : [
        {
            "_id" : 0,
            "host" : "shard_server01:27018"
        },
        {
            "_id" : 1,
            "host" : "shard_server02:27018"
        }
    ]
}
)

示例：

zxm@zxm-pc:~$ docker exec -it shard_server01 mongo --port 27018
> use admin
switched to db admin
> db.auth('root', '123456')
1
> rs.initiate({
... "_id" : "rs_config",
... "members" : [
... {
... "_id" : 0,
... "host" : "rs_config_server01:27019"
... },
... {
... "_id" : 1,
... "host" : "rs_config_server02:27019"
... }
... ]
... }
... )
{
    "ok" : 1,
    "$gleStats" : {
        "lastOpTime" : Timestamp(1716825534, 1),
        "electionId" : ObjectId("000000000000000000000000")
    },
    "lastCommittedOpTime" : Timestamp(0, 0)
}
rs_config:SECONDARY> 
rs_config:SECONDARY> exit
bye

2) shard2

rs.initiate({
    "_id" : "shard2",
    "members" : [
        {
            "_id" : 0,
            "host" : "shard_server03:27018"
        },
        {
            "_id" : 1,
            "host" : "shard_server04:27018"
        }
    ]
}
)

示例：

zxm@zxm-pc:~$ docker exec -it shard_server03 mongo --port 27018
> use admin
switched to db admin
> db.auth('root', '123456')
1
> rs.initiate({
... "_id" : "shard2",
... "members" : [
... {
... "_id" : 0,
... "host" : "shard_server03:27018"
... },
... {
... "_id" : 1,
... "host" : "shard_server04:27018"
... }
... ]
... }
... )
{ "ok" : 1 }
shard2:SECONDARY> 
shard2:SECONDARY> exit
bye

6.配置 mongos

sh.status() 查看集群的信息

示例：

zxm@zxm-pc:~$ docker exec -it mongos_01 mongo --port 27017
mongos> use admin
switched to db admin
mongos> sh.addShard("shard1/shard_server01:27018,shard_server02:27018")
{
    "shardAdded" : "shard1",
    "ok" : 1,
    "operationTime" : Timestamp(1716825659, 9),
    "$clusterTime" : {
        "clusterTime" : Timestamp(1716825659, 9),
        "signature" : {
            "hash" : BinData(0,"AZdizt6UTs6NiRMXVC9ve5l6PGs="),
            "keyId" : NumberLong("7373709568712376337")
        }
    }
}
mongos> sh.addShard("shard2/shard_server03:27018,shard_server04:27018")
{
    "shardAdded" : "shard2",
    "ok" : 1,
    "operationTime" : Timestamp(1716825666, 7),
    "$clusterTime" : {
        "clusterTime" : Timestamp(1716825666, 7),
        "signature" : {
            "hash" : BinData(0,"vJI1cxVuQxFv/GWlyIESfVZdMog="),
            "keyId" : NumberLong("7373709568712376337")
        }
    }
}
mongos> sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("6654adc94481b33023c17a2a")
  }
  shards:
        {  "_id" : "shard1",  "host" : "shard1/shard_server01:27018,shard_server02:27018",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/shard_server03:27018,shard_server04:27018",  "state" : 1 }
  active mongoses:
        "4.2.21" : 2
  autosplit:
        Currently enabled: yes
  balancer:
        Currently enabled:  yes
        Currently running:  no
        Failed balancer rounds in last 5 attempts:  0
        Migration Results for the last 24 hours: 
                No recent migrations
  databases:
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }

7.测试

进入 mongos_01

zxm@zxm-pc:~$ docker exec -it mongos_01 mongo --port 27017
mongos> use admin
switched to db admin
mongos> db.auth('root', '123456')
1
mongos> sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("6654adc94481b33023c17a2a")
  }
  shards:
        {  "_id" : "shard1",  "host" : "shard1/shard_server01:27018,shard_server02:27018",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/shard_server03:27018,shard_server04:27018",  "state" : 1 }
  active mongoses:
        "4.2.21" : 2
  autosplit:
        Currently enabled: yes
  balancer:
        Currently enabled:  yes
        Currently running:  no
        Failed balancer rounds in last 5 attempts:  0
        Migration Results for the last 24 hours: 
                No recent migrations
  databases:
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }

1) 对数据库启用分片

默认添加数据是没有使用分片存储的，使用 sh.enableSharding("test1")，开启数据库分片

mongos> sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("6654adc94481b33023c17a2a")
  }
  shards:
        {  "_id" : "shard1",  "host" : "shard1/shard_server01:27018,shard_server02:27018",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/shard_server03:27018,shard_server04:27018",  "state" : 1 }
  active mongoses:
        "4.2.21" : 2
  autosplit:
        Currently enabled: yes
  balancer:
        Currently enabled:  yes
        Currently running:  no
        Failed balancer rounds in last 5 attempts:  0
        Migration Results for the last 24 hours: 
                No recent migrations
  databases:
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }
        {  "_id" : "test1",  "primary" : "shard1",  "partitioned" : false,  "version" : {  "uuid" : UUID("fd0835ff-eb69-465b-bb42-374cdb9ca05d"),  "lastMod" : 1 } }
mongos> sh.enableSharding("test1");
{
    "ok" : 1,
    "operationTime" : Timestamp(1716825762, 3),
    "$clusterTime" : {
        "clusterTime" : Timestamp(1716825762, 3),
        "signature" : {
            "hash" : BinData(0,"HIRcdX6abcIzyCEDNxzERknzrHw="),
            "keyId" : NumberLong("7373709568712376337")
        }
    }
}
mongos> sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("6654adc94481b33023c17a2a")
  }
  shards:
        {  "_id" : "shard1",  "host" : "shard1/shard_server01:27018,shard_server02:27018",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/shard_server03:27018,shard_server04:27018",  "state" : 1 }
  active mongoses:
        "4.2.21" : 2
  autosplit:
        Currently enabled: yes
  balancer:
        Currently enabled:  yes
        Currently running:  no
        Failed balancer rounds in last 5 attempts:  0
        Migration Results for the last 24 hours: 
                No recent migrations
  databases:
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }
        {  "_id" : "test1",  "primary" : "shard1",  "partitioned" : true,  "version" : {  "uuid" : UUID("fd0835ff-eb69-465b-bb42-374cdb9ca05d"),  "lastMod" : 1 } }

2) 对集合启用分片

如果集合已经存在数据，必须手动创建在分片键上创建索引，然后再对集合进行分片，如果集合为空，MongoDB 会在分片的时候自动在分片键上创建索引。

使用 sh.shardCollection("test1.person1",{"id":"hashed"})， 配置集合 hash 分片存储

mongos> sh.shardCollection("test1.person1",{"id":"hashed"})
{
    "collectionsharded" : "test1.person1",
    "collectionUUID" : UUID("0f4d33ac-aa4e-444b-9223-f52fbbdac76f"),
    "ok" : 1,
    "operationTime" : Timestamp(1716826413, 32),
    "$clusterTime" : {
        "clusterTime" : Timestamp(1716826413, 32),
        "signature" : {
            "hash" : BinData(0,"IxZauq+YHXe4H6XcJxraglUJEx8="),
            "keyId" : NumberLong("7373709568712376337")
        }
    }
}
mongos> sh.status()
--- Sharding Status --- 
  sharding version: {
    "_id" : 1,
    "minCompatibleVersion" : 5,
    "currentVersion" : 6,
    "clusterId" : ObjectId("6654adc94481b33023c17a2a")
  }
  shards:
        {  "_id" : "shard1",  "host" : "shard1/shard_server01:27018,shard_server02:27018",  "state" : 1 }
        {  "_id" : "shard2",  "host" : "shard2/shard_server03:27018,shard_server04:27018",  "state" : 1 }
  active mongoses:
        "4.2.21" : 2
  autosplit:
        Currently enabled: yes
  balancer:
        Currently enabled:  yes
        Currently running:  no
        Failed balancer rounds in last 5 attempts:  0
        Migration Results for the last 24 hours: 
                430 : Success
  databases:
        {  "_id" : "config",  "primary" : "config",  "partitioned" : true }
                config.system.sessions
                        shard key: { "_id" : 1 }
                        unique: false
                        balancing: true
                        chunks:
                                shard1  594
                                shard2  430
                        too many chunks to print, use verbose if you want to force print
        {  "_id" : "test1",  "primary" : "shard1",  "partitioned" : true,  "version" : {  "uuid" : UUID("fd0835ff-eb69-465b-bb42-374cdb9ca05d"),  "lastMod" : 1 } }
                test1.person1
                        shard key: { "id" : "hashed" }
                        unique: false
                        balancing: true
                        chunks:
                                shard1  2
                                shard2  2
                        { "id" : { "$minKey" : 1 } } -->> { "id" : NumberLong("-4611686018427387902") } on : shard1 Timestamp(1, 0) 
                        { "id" : NumberLong("-4611686018427387902") } -->> { "id" : NumberLong(0) } on : shard1 Timestamp(1, 1) 
                        { "id" : NumberLong(0) } -->> { "id" : NumberLong("4611686018427387902") } on : shard2 Timestamp(1, 2) 
                        { "id" : NumberLong("4611686018427387902") } -->> { "id" : { "$maxKey" : 1 } } on : shard2 Timestamp(1, 3)

插入数据

mongos> db.person1.insert({'id': 1, 'name': 'zhang1', 'age': '21'})
WriteResult({ "nInserted" : 1 })
mongos> db.person1.insert({'id': 1, 'name': 'zhang1', 'age': '21'})
WriteResult({ "nInserted" : 1 })
mongos> db.person1.insert({'id': 2, 'name': 'zhang2', 'age': '22'})
WriteResult({ "nInserted" : 1 })
mongos> db.person1.insert({'id': 3, 'name': 'zhang3', 'age': '23'})
WriteResult({ "nInserted" : 1 })
mongos> db.person1.insert({'id': 4, 'name': 'zhang4', 'age': '24'})
WriteResult({ "nInserted" : 1 })
mongos> db.person1.find()
{ "_id" : ObjectId("6654b3084ee1e0860b4cba20"), "id" : 3, "name" : "zhang3", "age" : "23" }
{ "_id" : ObjectId("6654b1f64ee1e0860b4cba1d"), "id" : 1, "name" : "zhang1", "age" : "21" }
{ "_id" : ObjectId("6654b2ee4ee1e0860b4cba1e"), "id" : 1, "name" : "zhang1", "age" : "21" }
{ "_id" : ObjectId("6654b3014ee1e0860b4cba1f"), "id" : 2, "name" : "zhang2", "age" : "22" }
{ "_id" : ObjectId("6654b3104ee1e0860b4cba21"), "id" : 4, "name" : "zhang4", "age" : "24" }

二、jdcb 操作

pom 依赖：

<dependency>
    <groupId>org.mongodb</groupId>
    <artifactId>mongo-java-driver</artifactId>
    <version>3.12.10</version>
</dependency>

<dependency>
    <groupId>junit</groupId>
    <artifactId>junit</artifactId>
    <version>4.13.2</version>
    <scope>test</scope>
</dependency>

测试连接：

@Test
public void testConnectionWithShard() {
    // 方式一，验证连接
    String database = "admin";
    String user = "root";
    char[] password = "123456".toCharArray();

    MongoCredential credential = MongoCredential.createCredential(user, database, password);
    MongoClientOptions options = MongoClientOptions.builder().sslEnabled(false).build();
    MongoClient mongoClient = new MongoClient(Arrays.asList(new ServerAddress("127.0.0.1", 27017),
            new ServerAddress("127.0.0.1", 27027)), credential, options);

    // 方式二，使用 url，验证连接
    // MongoClient mongoClient = new MongoClient(new MongoClientURI( "mongodb://root:123456@127.0.0.1:27017,127.0.0.1:27027/?authSource=admin&ssl=false"));

    MongoCursor<String> cursor = mongoClient.listDatabaseNames().cursor();

    while (cursor.hasNext()) {
        String next = cursor.next();
        System.out.println(next);
    }
}

标签：

您可能还喜欢...

发表回复

您的邮箱地址不会被公开。 必填项已用 * 标注