ELK 소개, Monstache와 MongoDB연동 그리고 Docker-compose

# 접속하고자 하는 mongodb의 url을 입력합니다. 사전에 구성한 레플리카셋의 이름도 작성해야 합니다. mongo-url = "mongodb://mongo1:27017/servicename?replicaSet=myReplicaSet" # Elasticsearch REST API를 연결한 URL elasticsearch-urls = ["<http://elasticsearch:9200>"] # 기존 구성되어있던 mongodb에서 복사할 컬렉션 direct-read-namespaces = ["test.mycollection","test.test"] # 데이터베이스의 컬렉션 전체를 실시간 동기화하려면 데이터베이스 이름만 기입하고 특정 컬렉션만 동기화하려면 "db.collection" # 배포는 빈문자열만 입력 change-stream-namespaces = ["test.mycollection"] # 주기적으로 누적된 통계를 출력 stats = true # stats와 index-stats모두 참일 경우 Elasticsearch에 인덱싱 진행 상황에 대한 통계를 기록 index-stats = true # use the following user name for Elasticsearch basic auth #elasticsearch-user = "elastic" # use the following password for Elasticsearch basic auth #elasticsearch-password = "elasticpass" # use 4 go routines concurrently pushing documents to Elasticsearch elasticsearch-max-conns = 4 elasticsearch-max-seconds = 5 elasticsearch-max-bytes = 800000 # use the following PEM file to connections to Elasticsearch # elasticsearch-pem-file = "/path/to/elasticCert.pem" # validate connections to Elasticsearch # elastic-validate-pem-file = true # 컬렉션이 삭제되도 elasticsearch에 매핑된 인덱스는 삭제하지 않음 dropped-collections = false # 데이터베이스가 삭제되도 elasticsearch에 매핑된 인덱스는 삭제하지 않음 dropped-databases = false # do not start processing at the beginning of the MongoDB oplog # if you set the replay to true you may see version conflict messages # in the log if you had synced previously. This just means that you are replaying old docs which are already # in Elasticsearch with a newer version. Elasticsearch is preventing the old docs from overwriting new ones. replay = false # resume processing from a timestamp saved in a previous run resume = false # do not validate that progress timestamps have been saved resume-write-unsafe = true # override the name under which resume state is saved resume-name = "default" # use a custom resume strategy (tokens) instead of the default strategy (timestamps) # tokens work with MongoDB API 3.6+ while timestamps work only with MongoDB API 4.0+ resume-strategy = 1 # exclude documents whose namespace matches the following pattern namespace-exclude-regex = '^mydb\\.ignorecollection$' # turn on indexing of GridFS file content index-files = false index-as-update = true index-oplog-time = true # turn on search result highlighting of GridFS content file-highlighting = false # index GridFS files inserted into the following collections # file-namespaces = ["users.fs.files"] # print detailed information including request traces verbose = true # enable clustering mode cluster-name = 'apollo' # do not exit after full-sync, rather continue tailing the oplog exit-after-direct-reads = false

$ docker exec -it mongo1 mongosh

rs.initiate({ "_id": "myReplicaSet", "members": [ { "_id": 0, "host": "mongo1:27017", "priority": 2, } ] });

for (var i = 1; i <= 25; i++) { db.mycollection.insert( { x : i } ) }

version: '3.9' services: mongo1: env_file: .env image: mongo:latest hostname: mongo1 container_name: mongo1 command: ["--replSet", "${MONGO_REPLSET_NAME}", "--port", "${MONGO_PORT}"] restart: always healthcheck: test: | test $$(sh /data/init-replicaSet.sh) -eq 1 interval: 10s start_period: 20s # expose: # - "27017" ports: - 27017:${MONGO_PORT} volumes: - ./mongo/init-replicaSet.sh:/data/init-replicaSet.sh - mongodbdata:/data/db # - ./mongo/key.file:/data/key.file:ro networks: - db elasticsearch: env_file: .env container_name: elasticsearch build: context: elasticsearch/ args: ELK_VERSION: $ELK_VERSION ports: - 9200:9200 - 9300:9300 volumes: - type: bind source: ./elasticsearch/config/elasticsearch.yml target: /usr/share/elasticsearch/config/elasticsearch.yml read_only: true - type: volume source: elasticsearchdata target: /usr/share/elasticsearch/data networks: - elk kibana: env_file: .env environment: - ELASTICSEARCH_HOSTS=http://elasticsearch:9200 image: kibana:8.11.1 container_name: kibana build: context: kibana/ args: ELK_VERSION: $ELK_VERSION ports: - 5601:5601 volumes: - type: bind source: ./kibana/config/kibana.yml target: /usr/share/kibana/config/kibana.yml read_only: true networks: - elk depends_on: - elasticsearch monstache: env_file: .env restart: always image: rwynn/monstache:latest container_name: monstache command: -f ./monstache.config.toml & #command: tail -F anything volumes: - ./monstache/monstache.config.toml:/monstache.config.toml - monstachedata:/monstache/ networks: - db - elk depends_on: - elasticsearch - mongo1 networks: db: driver: bridge elk: driver: bridge volumes: mongodbdata: elasticsearchdata: monstachedata: