diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-cluster.xml b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-cluster.xml
new file mode 100644
index 0000000000..da65602742
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-cluster.xml
@@ -0,0 +1,93 @@
+
+
+
+
+
+
+ zookeeper-1
+ 2181
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ clickhouse
+ 9000
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-config.xml b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-config.xml
new file mode 100644
index 0000000000..0c6626a93d
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-config.xml
@@ -0,0 +1,1161 @@
+
+
+
+
+
+
+ information
+
+ json
+
+ /var/log/clickhouse-server/clickhouse-server.log
+ /var/log/clickhouse-server/clickhouse-server.err.log
+
+ 1000M
+ 10
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 8123
+
+
+ 9000
+
+
+ 9004
+
+
+ 9005
+
+
+
+
+
+
+
+
+
+
+
+ 9009
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 4096
+
+
+ 3
+
+
+
+
+ false
+
+
+ /path/to/ssl_cert_file
+ /path/to/ssl_key_file
+
+
+ false
+
+
+ /path/to/ssl_ca_cert_file
+
+
+ none
+
+
+ 0
+
+
+ -1
+ -1
+
+
+ false
+
+
+
+
+
+
+
+
+
+
+ none
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ true
+ true
+ sslv2,sslv3
+ true
+
+
+
+ RejectCertificateHandler
+
+
+
+
+
+
+
+
+ 100
+
+
+ 0
+
+
+
+ 10000
+
+
+
+
+
+ 0.9
+
+
+ 4194304
+
+
+ 0
+
+
+
+
+
+ 8589934592
+
+
+ 5368709120
+
+
+
+ 1000
+
+
+ 134217728
+
+
+ 10000
+
+
+ /var/lib/clickhouse/
+
+
+ /var/lib/clickhouse/tmp/
+
+
+
+ `
+
+
+
+
+
+ /var/lib/clickhouse/user_files/
+
+
+
+
+
+
+
+
+
+
+
+
+ users.xml
+
+
+
+ /var/lib/clickhouse/access/
+
+
+
+
+
+
+ default
+
+
+
+
+
+
+
+
+
+
+
+ default
+
+
+
+
+
+
+
+
+ true
+
+
+ false
+
+ ' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb
+ clickhouse-jdbc-bridge &
+
+ * [CentOS/RHEL]
+ export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge
+ export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|')
+ wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm
+ clickhouse-jdbc-bridge &
+
+ Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information.
+ ]]>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 01
+ example01-01-1
+
+
+
+
+
+ 3600
+
+
+
+ 3600
+
+
+ 60
+
+
+
+
+
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+
+
+
+
+
+ 7500
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+ system
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+
+
+ system
+
+ 7500
+ 1000
+
+
+
+
+ system
+
+
+ 7000
+
+
+
+
+
+
+ engine MergeTree
+ partition by toYYYYMM(finish_date)
+ order by (finish_date, finish_time_us, trace_id)
+
+ system
+
+ 7500
+
+
+
+
+
+ system
+
+
+
+ 1000
+
+
+
+
+
+
+
+ system
+
+
+ toYYYYMM(event_date)
+ 7500
+
+
+
+
+
+
+
+
+
+ *_dictionary.xml
+
+
+ *function.xml
+ /var/lib/clickhouse/user_scripts/
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ /clickhouse/task_queue/ddl
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ click_cost
+ any
+
+ 0
+ 3600
+
+
+ 86400
+ 60
+
+
+
+ max
+
+ 0
+ 60
+
+
+ 3600
+ 300
+
+
+ 86400
+ 3600
+
+
+
+
+
+ /var/lib/clickhouse/format_schemas/
+
+
+
+
+ hide encrypt/decrypt arguments
+ ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\)
+
+ \1(???)
+
+
+
+
+
+
+
+
+
+ false
+
+ false
+
+
+ https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277
+
+
+
+
+
+
+
+
+
+
+ 268435456
+ true
+
+
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-storage.xml b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-storage.xml
new file mode 100644
index 0000000000..4a1eedf905
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-storage.xml
@@ -0,0 +1,59 @@
+
+
+
+
+
+
+ 10485760
+
+
+ s3
+
+ https://BUCKET-NAME.s3-REGION-NAME.amazonaws.com/data/
+ ACCESS-KEY-ID
+ SECRET-ACCESS-KEY
+
+
+
+
+
+
+
+
+
+
+ default
+
+
+ s3
+ 0
+
+
+
+
+
+
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-users.xml b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-users.xml
new file mode 100644
index 0000000000..6f89952af1
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/clickhouse-users.xml
@@ -0,0 +1,141 @@
+
+
+
+
+
+
+
+
+
+
+ 10000000000
+
+
+ random
+
+
+
+
+ 1
+
+
+
+
+
+
+
+
+
+
+
+
+ ::/0
+
+
+
+ default
+
+
+ default
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 3600
+
+
+ 0
+ 0
+ 0
+ 0
+ 0
+
+
+
+
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/custom-function.xml b/docker/docker-compose/log-system/signoz/clickhouse-setup/custom-function.xml
new file mode 100644
index 0000000000..8c05041afc
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/custom-function.xml
@@ -0,0 +1,39 @@
+
+
+
+ executable
+ histogramQuantile
+ Float64
+
+ Array(Float64)
+ buckets
+
+
+ Array(Float64)
+ counts
+
+
+ Float64
+ quantile
+
+ CSV
+ ./histogramQuantile
+
+
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/docker-compose-minimal.yaml b/docker/docker-compose/log-system/signoz/clickhouse-setup/docker-compose-minimal.yaml
new file mode 100644
index 0000000000..fa49358774
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/docker-compose-minimal.yaml
@@ -0,0 +1,295 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+x-clickhouse-defaults: &clickhouse-defaults
+ restart: on-failure
+ # addding non LTS version due to this fix https://github.com/ClickHouse/ClickHouse/commit/32caf8716352f45c1b617274c7508c86b7d1afab
+ image: clickhouse/clickhouse-server:24.1.2-alpine
+ tty: true
+ depends_on:
+ - zookeeper-1
+ # - zookeeper-2
+ # - zookeeper-3
+ logging:
+ options:
+ max-size: 50m
+ max-file: "3"
+ healthcheck:
+ # "clickhouse", "client", "-u ${CLICKHOUSE_USER}", "--password ${CLICKHOUSE_PASSWORD}", "-q 'SELECT 1'"
+ test:
+ [
+ "CMD",
+ "wget",
+ "--spider",
+ "-q",
+ "0.0.0.0:8123/ping"
+ ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ ulimits:
+ nproc: 65535
+ nofile:
+ soft: 262144
+ hard: 262144
+
+x-db-depend: &db-depend
+ depends_on:
+ clickhouse:
+ condition: service_healthy
+ otel-collector-migrator:
+ condition: service_completed_successfully
+ # clickhouse-2:
+ # condition: service_healthy
+ # clickhouse-3:
+ # condition: service_healthy
+
+services:
+
+ zookeeper-1:
+ image: bitnami/zookeeper:3.7.1
+ container_name: signoz-zookeeper-1
+ hostname: zookeeper-1
+ user: root
+ ports:
+ - "2181:2181"
+ - "2888:2888"
+ - "3888:3888"
+ volumes:
+ - ./data/zookeeper-1:/bitnami/zookeeper
+ environment:
+ - ZOO_SERVER_ID=1
+ # - ZOO_SERVERS=0.0.0.0:2888:3888,zookeeper-2:2888:3888,zookeeper-3:2888:3888
+ - ALLOW_ANONYMOUS_LOGIN=yes
+ - ZOO_AUTOPURGE_INTERVAL=1
+
+ # zookeeper-2:
+ # image: bitnami/zookeeper:3.7.0
+ # container_name: signoz-zookeeper-2
+ # hostname: zookeeper-2
+ # user: root
+ # ports:
+ # - "2182:2181"
+ # - "2889:2888"
+ # - "3889:3888"
+ # volumes:
+ # - ./data/zookeeper-2:/bitnami/zookeeper
+ # environment:
+ # - ZOO_SERVER_ID=2
+ # - ZOO_SERVERS=zookeeper-1:2888:3888,0.0.0.0:2888:3888,zookeeper-3:2888:3888
+ # - ALLOW_ANONYMOUS_LOGIN=yes
+ # - ZOO_AUTOPURGE_INTERVAL=1
+
+ # zookeeper-3:
+ # image: bitnami/zookeeper:3.7.0
+ # container_name: signoz-zookeeper-3
+ # hostname: zookeeper-3
+ # user: root
+ # ports:
+ # - "2183:2181"
+ # - "2890:2888"
+ # - "3890:3888"
+ # volumes:
+ # - ./data/zookeeper-3:/bitnami/zookeeper
+ # environment:
+ # - ZOO_SERVER_ID=3
+ # - ZOO_SERVERS=zookeeper-1:2888:3888,zookeeper-2:2888:3888,0.0.0.0:2888:3888
+ # - ALLOW_ANONYMOUS_LOGIN=yes
+ # - ZOO_AUTOPURGE_INTERVAL=1
+
+ clickhouse:
+ <<: *clickhouse-defaults
+ container_name: signoz-clickhouse
+ hostname: clickhouse
+ ports:
+ - "9000:9000"
+ - "8123:8123"
+ - "9181:9181"
+ volumes:
+ - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+ - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+ - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+ - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+ # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+ - ./data/clickhouse/:/var/lib/clickhouse/
+ - ./user_scripts:/var/lib/clickhouse/user_scripts/
+
+ # clickhouse-2:
+ # <<: *clickhouse-defaults
+ # container_name: signoz-clickhouse-2
+ # hostname: clickhouse-2
+ # ports:
+ # - "9001:9000"
+ # - "8124:8123"
+ # - "9182:9181"
+ # volumes:
+ # - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+ # - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+ # - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+ # - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+ # # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+ # - ./data/clickhouse-2/:/var/lib/clickhouse/
+ # - ./user_scripts:/var/lib/clickhouse/user_scripts/
+
+
+ # clickhouse-3:
+ # <<: *clickhouse-defaults
+ # container_name: signoz-clickhouse-3
+ # hostname: clickhouse-3
+ # ports:
+ # - "9002:9000"
+ # - "8125:8123"
+ # - "9183:9181"
+ # volumes:
+ # - ./clickhouse-config.xml:/etc/clickhouse-server/config.xml
+ # - ./clickhouse-users.xml:/etc/clickhouse-server/users.xml
+ # - ./custom-function.xml:/etc/clickhouse-server/custom-function.xml
+ # - ./clickhouse-cluster.xml:/etc/clickhouse-server/config.d/cluster.xml
+ # # - ./clickhouse-storage.xml:/etc/clickhouse-server/config.d/storage.xml
+ # - ./data/clickhouse-3/:/var/lib/clickhouse/
+ # - ./user_scripts:/var/lib/clickhouse/user_scripts/
+
+ alertmanager:
+ image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.7}
+ container_name: signoz-alertmanager
+ volumes:
+ - ./data/alertmanager:/data
+ depends_on:
+ query-service:
+ condition: service_healthy
+ restart: on-failure
+ command:
+ - --queryService.url=http://query-service:8085
+ - --storage.path=/data
+
+ # Notes for Maintainers/Contributors who will change Line Numbers of Frontend & Query-Section. Please Update Line Numbers in `./scripts/commentLinesForSetup.sh` & `./CONTRIBUTING.md`
+
+ query-service:
+ image: signoz/query-service:${DOCKER_TAG:-0.56.0}
+ container_name: signoz-query-service
+ command:
+ [
+ "-config=/root/config/prometheus.yml",
+ "--use-logs-new-schema=true"
+ ]
+ # ports:
+ # - "6060:6060" # pprof port
+ # - "8080:8080" # query-service port
+ volumes:
+ - ./prometheus.yml:/root/config/prometheus.yml
+ - ../dashboards:/root/config/dashboards
+ - ./data/signoz/:/var/lib/signoz/
+ environment:
+ - ClickHouseUrl=tcp://clickhouse:9000
+ - ALERTMANAGER_API_PREFIX=http://alertmanager:9093/api/
+ - SIGNOZ_LOCAL_DB_PATH=/var/lib/signoz/signoz.db
+ - DASHBOARDS_PATH=/root/config/dashboards
+ - STORAGE=clickhouse
+ - GODEBUG=netdns=go
+ - TELEMETRY_ENABLED=true
+ - DEPLOYMENT_TYPE=docker-standalone-amd
+ restart: on-failure
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "wget",
+ "--spider",
+ "-q",
+ "localhost:8080/api/v1/health"
+ ]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ <<: *db-depend
+
+ frontend:
+ image: signoz/frontend:${DOCKER_TAG:-0.56.0}
+ container_name: signoz-frontend
+ restart: on-failure
+ depends_on:
+ - alertmanager
+ - query-service
+ ports:
+ - "3301:3301"
+ volumes:
+ - ../common/nginx-config.conf:/etc/nginx/conf.d/default.conf
+
+ otel-collector-migrator:
+ image: signoz/signoz-schema-migrator:${OTELCOL_TAG:-0.102.10}
+ container_name: otel-migrator
+ command:
+ - "--dsn=tcp://clickhouse:9000"
+ depends_on:
+ clickhouse:
+ condition: service_healthy
+ # clickhouse-2:
+ # condition: service_healthy
+ # clickhouse-3:
+ # condition: service_healthy
+
+
+ otel-collector:
+ image: signoz/signoz-otel-collector:${OTELCOL_TAG:-0.102.12}
+ container_name: signoz-otel-collector
+ command:
+ [
+ "--config=/etc/otel-collector-config.yaml",
+ "--manager-config=/etc/manager-config.yaml",
+ "--copy-path=/var/tmp/collector-config.yaml",
+ "--feature-gates=-pkg.translator.prometheus.NormalizeName"
+ ]
+ user: root # required for reading docker container logs
+ volumes:
+ - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
+ - ./otel-collector-opamp-config.yaml:/etc/manager-config.yaml
+ - /var/lib/docker/containers:/var/lib/docker/containers:ro
+ - /:/hostfs:ro
+ environment:
+ - OTEL_RESOURCE_ATTRIBUTES=host.name=signoz-host,os.type=linux
+ - DOCKER_MULTI_NODE_CLUSTER=false
+ - LOW_CARDINAL_EXCEPTION_GROUPING=false
+ ports:
+ # - "1777:1777" # pprof extension
+ - "4317:4317" # OTLP gRPC receiver
+ - "4318:4318" # OTLP HTTP receiver
+ # - "8888:8888" # OtelCollector internal metrics
+ # - "8889:8889" # signoz spanmetrics exposed by the agent
+ # - "9411:9411" # Zipkin port
+ # - "13133:13133" # health check extension
+ # - "14250:14250" # Jaeger gRPC
+ # - "14268:14268" # Jaeger thrift HTTP
+ # - "55678:55678" # OpenCensus receiver
+ # - "55679:55679" # zPages extension
+ restart: on-failure
+ depends_on:
+ clickhouse:
+ condition: service_healthy
+ otel-collector-migrator:
+ condition: service_completed_successfully
+ query-service:
+ condition: service_healthy
+
+ logspout:
+ image: "gliderlabs/logspout:v3.2.14"
+ container_name: signoz-logspout
+ volumes:
+ - /etc/hostname:/etc/host_hostname:ro
+ - /var/run/docker.sock:/var/run/docker.sock
+ command: syslog+tcp://otel-collector:2255
+ depends_on:
+ - otel-collector
+ restart: on-failure
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/keeper_config.xml b/docker/docker-compose/log-system/signoz/clickhouse-setup/keeper_config.xml
new file mode 100644
index 0000000000..afe548b617
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/keeper_config.xml
@@ -0,0 +1,82 @@
+
+
+
+
+ information
+ /var/log/clickhouse-keeper/clickhouse-keeper.log
+ /var/log/clickhouse-keeper/clickhouse-keeper.err.log
+
+ 1000M
+ 10
+
+
+
+ 0.0.0.0
+ 4096
+
+
+ 9181
+
+
+ 1
+
+ /var/lib/clickhouse/coordination/logs
+ /var/lib/clickhouse/coordination/snapshots
+
+
+ 10000
+ 10000
+ 100000
+ information
+ false
+
+
+
+
+ true
+
+
+ 1
+
+
+ clickhouses-keeper-1
+ 9234
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/otel-collector-config.yaml b/docker/docker-compose/log-system/signoz/clickhouse-setup/otel-collector-config.yaml
new file mode 100644
index 0000000000..82e64cecb7
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/otel-collector-config.yaml
@@ -0,0 +1,206 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+receivers:
+ tcplog/docker:
+ listen_address: "0.0.0.0:2255"
+ operators:
+ - type: regex_parser
+ regex: '^<([0-9]+)>[0-9]+ (?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?([zZ]|([\+-])([01]\d|2[0-3]):?([0-5]\d)?)?) (?P\S+) (?P\S+) [0-9]+ - -( (?P.*))?'
+ timestamp:
+ parse_from: attributes.timestamp
+ layout: '%Y-%m-%dT%H:%M:%S.%LZ'
+ - type: move
+ from: attributes["body"]
+ to: body
+ - type: remove
+ field: attributes.timestamp
+ # please remove names from below if you want to collect logs from them
+ - type: filter
+ id: signoz_logs_filter
+ expr: 'attributes.container_name matches "^signoz-(logspout|frontend|alertmanager|query-service|otel-collector|clickhouse|zookeeper)"'
+ opencensus:
+ endpoint: 0.0.0.0:55678
+ otlp:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:4317
+ http:
+ endpoint: 0.0.0.0:4318
+ jaeger:
+ protocols:
+ grpc:
+ endpoint: 0.0.0.0:14250
+ thrift_http:
+ endpoint: 0.0.0.0:14268
+ # thrift_compact:
+ # endpoint: 0.0.0.0:6831
+ # thrift_binary:
+ # endpoint: 0.0.0.0:6832
+ hostmetrics:
+ collection_interval: 30s
+ root_path: /hostfs
+ scrapers:
+ cpu: {}
+ load: {}
+ memory: {}
+ disk: {}
+ filesystem: {}
+ network: {}
+ prometheus:
+ config:
+ global:
+ scrape_interval: 60s
+ scrape_configs:
+ # otel-collector internal metrics
+ - job_name: otel-collector
+ static_configs:
+ - targets:
+ - localhost:8888
+ labels:
+ job_name: otel-collector
+
+
+processors:
+ batch:
+ send_batch_size: 10000
+ send_batch_max_size: 11000
+ timeout: 10s
+ signozspanmetrics/cumulative:
+ metrics_exporter: clickhousemetricswrite
+ metrics_flush_interval: 60s
+ latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ]
+ dimensions_cache_size: 100000
+ dimensions:
+ - name: service.namespace
+ default: default
+ - name: deployment.environment
+ default: default
+ # This is added to ensure the uniqueness of the timeseries
+ # Otherwise, identical timeseries produced by multiple replicas of
+ # collectors result in incorrect APM metrics
+ - name: signoz.collector.id
+ - name: service.version
+ - name: browser.platform
+ - name: browser.mobile
+ - name: k8s.cluster.name
+ - name: k8s.node.name
+ - name: k8s.namespace.name
+ - name: host.name
+ - name: host.type
+ - name: container.name
+ # memory_limiter:
+ # # 80% of maximum memory up to 2G
+ # limit_mib: 1500
+ # # 25% of limit up to 2G
+ # spike_limit_mib: 512
+ # check_interval: 5s
+ #
+ # # 50% of the maximum memory
+ # limit_percentage: 50
+ # # 20% of max memory usage spike expected
+ # spike_limit_percentage: 20
+ # queued_retry:
+ # num_workers: 4
+ # queue_size: 100
+ # retry_on_failure: true
+ resourcedetection:
+ # Using OTEL_RESOURCE_ATTRIBUTES envvar, env detector adds custom labels.
+ detectors: [env, system] # include ec2 for AWS, gcp for GCP and azure for Azure.
+ timeout: 2s
+ signozspanmetrics/delta:
+ metrics_exporter: clickhousemetricswrite
+ metrics_flush_interval: 60s
+ latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ]
+ dimensions_cache_size: 100000
+ aggregation_temporality: AGGREGATION_TEMPORALITY_DELTA
+ enable_exp_histogram: true
+ dimensions:
+ - name: service.namespace
+ default: default
+ - name: deployment.environment
+ default: default
+ # This is added to ensure the uniqueness of the timeseries
+ # Otherwise, identical timeseries produced by multiple replicas of
+ # collectors result in incorrect APM metrics
+ - name: signoz.collector.id
+ - name: service.version
+ - name: browser.platform
+ - name: browser.mobile
+ - name: k8s.cluster.name
+ - name: k8s.node.name
+ - name: k8s.namespace.name
+ - name: host.name
+ - name: host.type
+ - name: container.name
+
+extensions:
+ health_check:
+ endpoint: 0.0.0.0:13133
+ zpages:
+ endpoint: 0.0.0.0:55679
+ pprof:
+ endpoint: 0.0.0.0:1777
+
+exporters:
+ clickhousetraces:
+ datasource: tcp://clickhouse:9000/signoz_traces
+ docker_multi_node_cluster: ${env:DOCKER_MULTI_NODE_CLUSTER}
+ low_cardinal_exception_grouping: ${env:LOW_CARDINAL_EXCEPTION_GROUPING}
+ clickhousemetricswrite:
+ endpoint: tcp://clickhouse:9000/signoz_metrics
+ resource_to_telemetry_conversion:
+ enabled: true
+ clickhousemetricswrite/prometheus:
+ endpoint: tcp://clickhouse:9000/signoz_metrics
+ clickhouselogsexporter:
+ dsn: tcp://clickhouse:9000/signoz_logs
+ docker_multi_node_cluster: ${env:DOCKER_MULTI_NODE_CLUSTER}
+ timeout: 10s
+ use_new_schema: true
+ # logging: {}
+
+service:
+ telemetry:
+ logs:
+ encoding: json
+ metrics:
+ address: 0.0.0.0:8888
+ extensions:
+ - health_check
+ - zpages
+ - pprof
+ pipelines:
+ traces:
+ receivers: [jaeger, otlp]
+ processors: [signozspanmetrics/cumulative, signozspanmetrics/delta, batch]
+ exporters: [clickhousetraces]
+ metrics:
+ receivers: [otlp]
+ processors: [batch]
+ exporters: [clickhousemetricswrite]
+ metrics/generic:
+ receivers: [hostmetrics]
+ processors: [resourcedetection, batch]
+ exporters: [clickhousemetricswrite]
+ metrics/prometheus:
+ receivers: [prometheus]
+ processors: [batch]
+ exporters: [clickhousemetricswrite/prometheus]
+ logs:
+ receivers: [otlp, tcplog/docker]
+ processors: [batch]
+ exporters: [clickhouselogsexporter]
\ No newline at end of file
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/otel-collector-opamp-config.yaml b/docker/docker-compose/log-system/signoz/clickhouse-setup/otel-collector-opamp-config.yaml
new file mode 100644
index 0000000000..d05b8b8dc0
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/otel-collector-opamp-config.yaml
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+server_endpoint: ws://query-service:4320/v1/opamp
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/prometheus.yml b/docker/docker-compose/log-system/signoz/clickhouse-setup/prometheus.yml
new file mode 100644
index 0000000000..8c3b0c3819
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/prometheus.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# my global config
+global:
+ scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+ evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+ # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+ alertmanagers:
+ - static_configs:
+ - targets:
+ - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+ # - "first_rules.yml"
+ # - "second_rules.yml"
+ - 'alerts.yml'
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs: []
+
+remote_read:
+ - url: tcp://clickhouse:9000/signoz_metrics
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/user_scripts/histogramQuantile b/docker/docker-compose/log-system/signoz/clickhouse-setup/user_scripts/histogramQuantile
new file mode 100644
index 0000000000..0978f8b85f
Binary files /dev/null and b/docker/docker-compose/log-system/signoz/clickhouse-setup/user_scripts/histogramQuantile differ
diff --git a/docker/docker-compose/log-system/signoz/clickhouse-setup/user_scripts/histogramQuantile.go b/docker/docker-compose/log-system/signoz/clickhouse-setup/user_scripts/histogramQuantile.go
new file mode 100644
index 0000000000..6576f974e1
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/clickhouse-setup/user_scripts/histogramQuantile.go
@@ -0,0 +1,253 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package main
+
+import (
+ "bufio"
+ "fmt"
+ "math"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+)
+
+// NOTE: executable must be built with target OS and architecture set to linux/amd64
+// env GOOS=linux GOARCH=amd64 go build -o histogramQuantile histogramQuantile.go
+
+// The following code is adapted from the following source:
+// https://github.com/prometheus/prometheus/blob/main/promql/quantile.go
+
+type bucket struct {
+ upperBound float64
+ count float64
+}
+
+// buckets implements sort.Interface.
+type buckets []bucket
+
+func (b buckets) Len() int { return len(b) }
+func (b buckets) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
+func (b buckets) Less(i, j int) bool { return b[i].upperBound < b[j].upperBound }
+
+// bucketQuantile calculates the quantile 'q' based on the given buckets. The
+// buckets will be sorted by upperBound by this function (i.e. no sorting
+// needed before calling this function). The quantile value is interpolated
+// assuming a linear distribution within a bucket. However, if the quantile
+// falls into the highest bucket, the upper bound of the 2nd highest bucket is
+// returned. A natural lower bound of 0 is assumed if the upper bound of the
+// lowest bucket is greater 0. In that case, interpolation in the lowest bucket
+// happens linearly between 0 and the upper bound of the lowest bucket.
+// However, if the lowest bucket has an upper bound less or equal 0, this upper
+// bound is returned if the quantile falls into the lowest bucket.
+//
+// There are a number of special cases (once we have a way to report errors
+// happening during evaluations of AST functions, we should report those
+// explicitly):
+//
+// If 'buckets' has 0 observations, NaN is returned.
+//
+// If 'buckets' has fewer than 2 elements, NaN is returned.
+//
+// If the highest bucket is not +Inf, NaN is returned.
+//
+// If q==NaN, NaN is returned.
+//
+// If q<0, -Inf is returned.
+//
+// If q>1, +Inf is returned.
+func bucketQuantile(q float64, buckets buckets) float64 {
+ if math.IsNaN(q) {
+ return math.NaN()
+ }
+ if q < 0 {
+ return math.Inf(-1)
+ }
+ if q > 1 {
+ return math.Inf(+1)
+ }
+ sort.Sort(buckets)
+ if !math.IsInf(buckets[len(buckets)-1].upperBound, +1) {
+ return math.NaN()
+ }
+
+ buckets = coalesceBuckets(buckets)
+ ensureMonotonic(buckets)
+
+ if len(buckets) < 2 {
+ return math.NaN()
+ }
+ observations := buckets[len(buckets)-1].count
+ if observations == 0 {
+ return math.NaN()
+ }
+ rank := q * observations
+ b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank })
+
+ if b == len(buckets)-1 {
+ return buckets[len(buckets)-2].upperBound
+ }
+ if b == 0 && buckets[0].upperBound <= 0 {
+ return buckets[0].upperBound
+ }
+ var (
+ bucketStart float64
+ bucketEnd = buckets[b].upperBound
+ count = buckets[b].count
+ )
+ if b > 0 {
+ bucketStart = buckets[b-1].upperBound
+ count -= buckets[b-1].count
+ rank -= buckets[b-1].count
+ }
+ return bucketStart + (bucketEnd-bucketStart)*(rank/count)
+}
+
+// coalesceBuckets merges buckets with the same upper bound.
+//
+// The input buckets must be sorted.
+func coalesceBuckets(buckets buckets) buckets {
+ last := buckets[0]
+ i := 0
+ for _, b := range buckets[1:] {
+ if b.upperBound == last.upperBound {
+ last.count += b.count
+ } else {
+ buckets[i] = last
+ last = b
+ i++
+ }
+ }
+ buckets[i] = last
+ return buckets[:i+1]
+}
+
+// The assumption that bucket counts increase monotonically with increasing
+// upperBound may be violated during:
+//
+// * Recording rule evaluation of histogram_quantile, especially when rate()
+// has been applied to the underlying bucket timeseries.
+// * Evaluation of histogram_quantile computed over federated bucket
+// timeseries, especially when rate() has been applied.
+//
+// This is because scraped data is not made available to rule evaluation or
+// federation atomically, so some buckets are computed with data from the
+// most recent scrapes, but the other buckets are missing data from the most
+// recent scrape.
+//
+// Monotonicity is usually guaranteed because if a bucket with upper bound
+// u1 has count c1, then any bucket with a higher upper bound u > u1 must
+// have counted all c1 observations and perhaps more, so that c >= c1.
+//
+// Randomly interspersed partial sampling breaks that guarantee, and rate()
+// exacerbates it. Specifically, suppose bucket le=1000 has a count of 10 from
+// 4 samples but the bucket with le=2000 has a count of 7 from 3 samples. The
+// monotonicity is broken. It is exacerbated by rate() because under normal
+// operation, cumulative counting of buckets will cause the bucket counts to
+// diverge such that small differences from missing samples are not a problem.
+// rate() removes this divergence.)
+//
+// bucketQuantile depends on that monotonicity to do a binary search for the
+// bucket with the φ-quantile count, so breaking the monotonicity
+// guarantee causes bucketQuantile() to return undefined (nonsense) results.
+//
+// As a somewhat hacky solution until ingestion is atomic per scrape, we
+// calculate the "envelope" of the histogram buckets, essentially removing
+// any decreases in the count between successive buckets.
+
+func ensureMonotonic(buckets buckets) {
+ max := buckets[0].count
+ for i := 1; i < len(buckets); i++ {
+ switch {
+ case buckets[i].count > max:
+ max = buckets[i].count
+ case buckets[i].count < max:
+ buckets[i].count = max
+ }
+ }
+}
+
+// End of copied code.
+
+func readLines() []string {
+ r := bufio.NewReader(os.Stdin)
+ bytes := []byte{}
+ lines := []string{}
+ for {
+ line, isPrefix, err := r.ReadLine()
+ if err != nil {
+ break
+ }
+ bytes = append(bytes, line...)
+ if !isPrefix {
+ str := strings.TrimSpace(string(bytes))
+ if len(str) > 0 {
+ lines = append(lines, str)
+ bytes = []byte{}
+ }
+ }
+ }
+ if len(bytes) > 0 {
+ lines = append(lines, string(bytes))
+ }
+ return lines
+}
+
+func main() {
+ lines := readLines()
+ for _, text := range lines {
+ // Example input
+ // "[1, 2, 4, 8, 16]", "[1, 5, 8, 10, 14]", 0.9"
+ // bounds - counts - quantile
+ parts := strings.Split(text, "\",")
+
+ var bucketNumbers []float64
+ // Strip the ends with square brackets
+ text = parts[0][2 : len(parts[0])-1]
+ // Parse the bucket bounds
+ for _, num := range strings.Split(text, ",") {
+ num = strings.TrimSpace(num)
+ number, err := strconv.ParseFloat(num, 64)
+ if err == nil {
+ bucketNumbers = append(bucketNumbers, number)
+ }
+ }
+
+ var bucketCounts []float64
+ // Strip the ends with square brackets
+ text = parts[1][2 : len(parts[1])-1]
+ // Parse the bucket counts
+ for _, num := range strings.Split(text, ",") {
+ num = strings.TrimSpace(num)
+ number, err := strconv.ParseFloat(num, 64)
+ if err == nil {
+ bucketCounts = append(bucketCounts, number)
+ }
+ }
+
+ // Parse the quantile
+ q, err := strconv.ParseFloat(parts[2], 64)
+ var b buckets
+
+ if err == nil {
+ for i := 0; i < len(bucketNumbers); i++ {
+ b = append(b, bucket{upperBound: bucketNumbers[i], count: bucketCounts[i]})
+ }
+ }
+ fmt.Println(bucketQuantile(q, b))
+ }
+}
diff --git a/docker/docker-compose/log-system/signoz/common/nginx-config.conf b/docker/docker-compose/log-system/signoz/common/nginx-config.conf
new file mode 100644
index 0000000000..5a4e18fff2
--- /dev/null
+++ b/docker/docker-compose/log-system/signoz/common/nginx-config.conf
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+map $http_upgrade $connection_upgrade {
+ default upgrade;
+ '' close;
+}
+
+server {
+ listen 3301;
+ server_name _;
+
+ gzip on;
+ gzip_static on;
+ gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
+ gzip_proxied any;
+ gzip_vary on;
+ gzip_comp_level 6;
+ gzip_buffers 16 8k;
+ gzip_http_version 1.1;
+
+ # to handle uri issue 414 from nginx
+ client_max_body_size 24M;
+ large_client_header_buffers 8 128k;
+
+ location / {
+ if ( $uri = '/index.html' ) {
+ add_header Cache-Control no-store always;
+ }
+ root /usr/share/nginx/html;
+ index index.html index.htm;
+ try_files $uri $uri/ /index.html;
+ }
+
+ location ~ ^/api/(v1|v3)/logs/(tail|livetail){
+ proxy_pass http://query-service:8080;
+ proxy_http_version 1.1;
+
+ # connection will be closed if no data is read for 600s between successive read operations
+ proxy_read_timeout 600s;
+
+ # dont buffer the data send it directly to client.
+ proxy_buffering off;
+ proxy_cache off;
+ }
+
+ location /api {
+ proxy_pass http://query-service:8080/api;
+ # connection will be closed if no data is read for 600s between successive read operations
+ proxy_read_timeout 600s;
+ }
+
+ location /ws {
+ proxy_pass http://query-service:8080/ws;
+ proxy_http_version 1.1;
+ proxy_set_header Upgrade "websocket";
+ proxy_set_header Connection "upgrade";
+ proxy_read_timeout 86400;
+ }
+
+ # redirect server error pages to the static page /50x.html
+ #
+ error_page 500 502 503 504 /50x.html;
+ location = /50x.html {
+ root /usr/share/nginx/html;
+ }
+}
\ No newline at end of file