Telegraf Generating Orphaned DBus Processes on RHEL Servers #2 #13635

elangovanseshan · 2023-07-17T15:40:49Z

Relevant telegraf.conf

#Telegraf Configuration
#
# Telegraf is entirely plugin driven. All metrics are gathered from the
# declared inputs, and sent to the declared outputs.
#
# Plugins must be declared in here to be active.
# To deactivate a plugin, comment out the name and any variables.
#
# Use 'telegraf -config telegraf.conf -test' to see what metrics a config
# file would generate.
#
# Environment variables can be used anywhere in this config file, simply prepend
# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)


# Global tags can be specified here in key="value" format.
[global_tags]
  env = "Production_Linux"
  # dc = "us-east-1" # will tag all metrics with dc=us-east-1
  # rack = "1a"
  ## Environment variables can be used as tags, and throughout the config file
  # user = "$USER"


# Configuration for telegraf agent
[agent]
  ## Default data collection interval for all inputs
  interval = "1m"
  ## Rounds collection interval to 'interval'
  ## ie, if interval="10s" then always collect on :00, :10, :20, etc.
  round_interval = true

  ## Telegraf will send metrics to outputs in batches of at most
  ## metric_batch_size metrics.
  ## This controls the size of writes that Telegraf sends to output plugins.
  metric_batch_size = 1000

  ## For failed writes, telegraf will cache metric_buffer_limit metrics for each
  ## output, and will flush this buffer on a successful write. Oldest metrics
  ## are dropped first when this buffer fills.
  ## This buffer only fills when writes fail to output plugin(s).
  metric_buffer_limit = 10000

  ## Collection jitter is used to jitter the collection by a random amount.
  ## Each plugin will sleep for a random time within jitter before collecting.
  ## This can be used to avoid many plugins querying things like sysfs at the
  ## same time, which can have a measurable effect on the system.
  collection_jitter = "0s"

  ## Default flushing interval for all outputs. Maximum flush_interval will be
  ## flush_interval + flush_jitter
  flush_interval = "10s"
  ## Jitter the flush interval by a random amount. This is primarily to avoid
  ## large write spikes for users running a large number of telegraf instances.
  ## ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s
  flush_jitter = "10s"

  ## By default or when set to "0s", precision will be set to the same
  ## timestamp order as the collection interval, with the maximum being 1s.
  ##   ie, when interval = "10s", precision will be "1s"
  ##       when interval = "250ms", precision will be "1ms"
  ## Precision will NOT be used for service inputs. It is up to each individual
  ## service input to set the timestamp at the appropriate precision.
  ## Valid time units are "ns", "us" (or "µs"), "ms", "s".
  precision = ""

  ## Logging configuration:
  ## Run telegraf with debug log messages.
  debug = false
  ## Run telegraf in quiet mode (error log messages only).
  quiet = false
  ## Specify the log file name. The empty string means to log to stderr.
  logfile = "/var/log/telegraf/telegraf.log"
  logfile_rotation_interval = "24h"
  logfile_rotation_max_archives = 2
  logfile_rotation_max_size = "50MB"
  ## Override default hostname, if empty use os.Hostname()
  hostname = ""
  ## If set to true, do no set the "host" tag in the telegraf agent.
  omit_hostname = false


###############################################################################
#                            OUTPUT PLUGINS                                   #
###############################################################################


# # # Send telegraf metrics to file(s)
# [[outputs.file]]
#   ## Files to write to, "stdout" is a specially handled file.
#   files = ["/var/log/telegraf/telegraf.out"]

#   ## Data format to output.
#   ## Each data format has its own unique set of configuration options, read
#   ## more about them here:
#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
#   data_format = "influx"


# # Configuration for Wavefront server to send metrics to
[[outputs.wavefront]]
#   ## DNS name of the wavefront proxy server
#   host = "wavefront.example.com"

url = "http://metrics****************:2878"
#
#   ## Port that the Wavefront proxy server listens on
#   port = 2878
#port = 2878
convert_paths = false
namepass = ["prod.*","qa.*","dev.*"]





#----------------------------------
#Linux Input Plugins 
#---------------------------------

###############################################################################################
#                              Linux Input Plugins                                            #
###############################################################################################

## NETWORK METRICS
[[inputs.net]]
  name_prefix = "prod.metrics."
  interval = "15m"
  ignore_protocol_stats = true

## CPU METRICS
[[inputs.cpu]]
  name_prefix = "prod.metrics."
  interval = "10m"
  percpu = false
  totalcpu = true
  collect_cpu_time = false
  report_active = false

## DISK METRICS
[[inputs.disk]]
  name_prefix = "prod.metrics."
  interval = "30m"

[[inputs.diskio]]
  name_prefix = "prod.metrics."
  interval = "5m"

## SYSTEM METRICS
[[inputs.system]]
  name_prefix = "prod.metrics."
  interval = "10m"

## MEMORY METRICS
 [[inputs.mem]]
   name_prefix = "prod.metrics."
   interval = "5m"
   fieldpass = ["active",
                "available",
                "buffered",
                "cached",
                "free",
                "inactive",
                "slab",
                "used",
                "available_percent",
                "used_percent",
                "wired",
                "commit_limit",
                "committed_as",
                "dirty",
                "high_free",
                "huge_pages_free",
                "low_free",
                "mapped",
                "page_tables",
                "shared",
                "swap_cached",
                "swap_free",
                "vmalloc_chunk",
                "vmalloc_used",
                "write_back",
                "write_back_tmp"]

[[inputs.mem]]
  name_prefix = "prod.metrics."
  interval = "60m"
  fieldpass = ["total","high_total","huge_page_size","huge_pages_total","low_total","swap_total","vmalloc_total"]

## SWAP METRICS
[[inputs.swap]]
  name_prefix = "prod.metrics."
  interval = "30m"
  fieldpass = ["free", "total","used", "used_percent"]

[[inputs.swap]]
  name_prefix = "prod.metrics."
  interval = "5m"
  fieldpass = ["in", "out"]

## TELEGRAF INTERNAL METRICS
[[inputs.internal]]
  interval = "60m"
  name_prefix = "prod.metrics."
  namepass = ["internal_gather*"]
    [inputs.internal.tagpass]
      input = ["internal"]

Logs from Telegraf

2023-07-17T15:26:21Z I! Loading config: /etc/telegraf/telegraf.d/monitor.conf
2023-07-17T15:26:21Z I! Starting Telegraf 1.27.2
2023-07-17T15:26:21Z I! Available plugins: 237 inputs, 9 aggregators, 28 processors, 23 parsers, 59 outputs, 4 secret-stores
2023-07-17T15:26:21Z I! Loaded inputs: cpu disk diskio exec (2x) internal mem (2x) net swap (2x) system
2023-07-17T15:26:21Z I! Loaded aggregators:
2023-07-17T15:26:21Z I! Loaded processors:
2023-07-17T15:26:21Z I! Loaded secretstores:
2023-07-17T15:26:21Z I! Loaded outputs: wavefront
2023-07-17T15:26:21Z I! Tags enabled: env=Production_Linux host=stuxsh03
2023-07-17T15:26:21Z I! [agent] Config: Interval:1m0s, Quiet:false, Hostname:"****", Flush Interval:10s
2023-07-17T15:27:53Z I! [agent] Hang on, flushing any cached metrics before shutdown
2023-07-17T15:27:53Z I! [agent] Stopping running outputs
2023-07-17T15:27:58Z I! Loading config: /etc/telegraf/telegraf.conf
2023-07-17T15:27:58Z I! Loading config: /etc/telegraf/telegraf.d/compute_services.conf
2023-07-17T15:27:58Z I! Loading config: /etc/telegraf/telegraf.d/monitor.conf
2023-07-17T15:27:58Z I! Starting Telegraf 1.27.2
2023-07-17T15:27:58Z I! Available plugins: 237 inputs, 9 aggregators, 28 processors, 23 parsers, 59 outputs, 4 secret-stores
2023-07-17T15:27:58Z I! Loaded inputs: cpu disk diskio exec (2x) internal mem (2x) net swap (2x) system
2023-07-17T15:27:58Z I! Loaded aggregators:
2023-07-17T15:27:58Z I! Loaded processors:
2023-07-17T15:27:58Z I! Loaded secretstores:
2023-07-17T15:27:58Z I! Loaded outputs: wavefront
2023-07-17T15:27:58Z I! Tags enabled: env=Production_Linux host**************
2023-07-17T15:27:58Z I! [agent] Config: Interval:1m0s, Quiet:false, Hostname:"***********", Flush Interval:10s
2023-07-17T15:27:58Z D! [agent] Initializing plugins
2023-07-17T15:27:58Z D! [agent] Connecting outputs
2023-07-17T15:27:58Z D! [agent] Attempting connection to [outputs.wavefront]
2023-07-17T15:27:58Z D! [outputs.wavefront] connecting over http/https using Url: ******************:2878
2023-07-17T15:27:58Z D! [agent] Successfully connected to outputs.wavefront
2023-07-17T15:27:58Z D! [agent] Starting service inputs
2023-07-17T15:28:14Z D! [outputs.wavefront] Flushing batch of 1 points
2023-07-17T15:28:14Z D! [outputs.wavefront] Wrote batch of 1 metrics in 63.547432ms
2023-07-17T15:28:14Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:28:30Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:28:46Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:29:03Z D! [outputs.wavefront] Flushing batch of 1 points
2023-07-17T15:29:03Z D! [outputs.wavefront] Wrote batch of 1 metrics in 33.405965ms
2023-07-17T15:29:03Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:29:16Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:29:31Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:29:47Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:30:00Z D! [inputs.disk] [SystemPS] => unable to get disk usage ("/var/named/chroot/etc/named"): permission denied
2023-07-17T15:30:00Z D! [inputs.disk] [SystemPS] => unable to get disk usage ("/var/named/chroot/var/named"): permission denied
2023-07-17T15:30:00Z D! [inputs.disk] [SystemPS] => unable to get disk usage ("/var/named/chroot/usr/lib64/bind"): permission denied
2023-07-17T15:30:04Z D! [outputs.wavefront] Error building tags: unexpected type: string, with value: 1 day,  2:37, for: prod.metrics.system.uptime_format
2023-07-17T15:30:04Z D! [outputs.wavefront] Flushing batch of 99 points
2023-07-17T15:30:04Z D! [outputs.wavefront] Wrote batch of 99 metrics in 69.526316ms
2023-07-17T15:30:04Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:30:14Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:30:24Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:30:36Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:30:48Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:31:07Z D! [outputs.wavefront] Flushing batch of 1 points
2023-07-17T15:31:07Z D! [outputs.wavefront] Wrote batch of 1 metrics in 33.711449ms
2023-07-17T15:31:07Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:31:21Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
2023-07-17T15:31:35Z D! [outputs.wavefront] Buffer fullness: 0 / 10000 metrics
stuxsh03.st9793:/root#

System info

telegraf-1.27.2 it's running in OS Linux 2.6.32-754.50.1.el6.x86_64

Docker

No response

Steps to reproduce

Reproducing has been tricky as it doesn't always appear to occur, but on systems that were impacted (hundreds+) reverting Telegraf to an earlier version, stopping the Telegraf service and removing the orphaned process, or performing the below actions resolved the issue.

What we have seen:
Upgrading the Telegraf version 1.14 to 1.25.2 on RHEL servers seems to create an issue where DBus generates many orphaned processes. This eventually causes the system to hit the ceiling of available PIDs. Rolling back to 1.14 seems to clear the problem.

Example from one of our systems:

ps -ef|grep dbus|grep -v grep|wc -l
1459

Based on the issue #13481 it was resolved in recent release telegraf-1.27.2 but we are experiencing the same issue with recent release aswell

Expected behavior

Telegraf works as expected.

Actual behavior

Telegraf inadvertantly creates thousands of orphaned DBus processes which eventually causes the available PID's to hit the maximum ceiling, which causes system degradation.

Additional info

No response

Tasks

Give feedback

No tasks being tracked yet.

Options

The text was updated successfully, but these errors were encountered:

crflanigan · 2023-07-17T15:43:53Z

@powersj
New issue created

crflanigan · 2023-07-17T15:46:37Z

As an aside, it looks like so far this issue appears to be absent from 1.24.2.

elangovanseshan · 2023-07-17T15:47:09Z

here are the dbus details which we are seeing it in server

root       336     1  0 07:03 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       353     1  0 06:16 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       372     1  0 03:43 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       385     1  0 08:38 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       426     1  0 07:03 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       427     1  0 05:21 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       458     1  0 08:39 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       495     1  0 09:25 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       569     1  0 04:30 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       602     1  0 07:04 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       643     1  0 08:39 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       701     1  0 09:26 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       823     1  0 09:26 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       840     1  0 06:17 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       884     1  0 04:31 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       896     1  0 02:47 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       936     1  0 05:21 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       940     1  0 07:04 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       951     1  0 08:40 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root       982     1  0 09:27 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1020     1  0 03:44 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1068     1  0 05:22 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1111     1  0 07:55 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1116     1  0 04:31 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1210     1  0 07:55 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1230     1  0 03:44 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1305     1  0 03:45 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1429     1  0 02:47 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1518     1  0 03:45 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1736     1  0 03:46 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1756     1  0 07:56 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1815     1  0 09:27 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1874     1  0 07:56 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1887     1  0 09:28 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1937     1  0 04:32 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      1963     1  0 03:46 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2022     1  0 08:41 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2029     1  0 04:32 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2044     1  0 09:28 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2112     1  0 07:57 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2118     1  0 09:29 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2154     1  0 05:22 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2156     1  0 04:33 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2184     1  0 09:29 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2211     1  0 02:48 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2242     1  0 07:57 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2253     1  0 08:41 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2272     1  0 04:33 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2298     1  0 05:23 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2330     1  0 09:30 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2353     1  0 07:58 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2365     1  0 04:34 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2408     1  0 07:58 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2418     1  0 07:05 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2461     1  0 05:23 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2480     1  0 07:59 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2602     1  0 07:05 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2623     1  0 04:34 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2643     1  0 07:59 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2678     1  0 03:47 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2681     1  0 08:42 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2690     1  0 05:24 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2704     1  0 04:35 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2748     1  0 04:35 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root      2788     1  0 07:06 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session

elangovanseshan · 2023-07-17T15:48:16Z

telegraf  5850     1  0 06:54 ?        00:00:02 /usr/bin/telegraf -pidfile /var/run/telegraf/telegraf.pid -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf telegraf.d
telegraf  5866     1  0 06:54 ?        00:00:00 /usr/bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root     11702 24282  0 07:34 pts/0    00:00:00 grep --color=auto -i telegraf

crflanigan · 2023-07-17T15:52:22Z

Is there a way to disable the secret store completely? We don't use it and some component related to it seems to be causing the issues.

powersj · 2023-07-17T17:28:31Z

Thanks for the issue and logs. Are you seeing this across RHEL 6, 7, and 8 this time? Or only RHEL 6? I have got a RHEL 7 VM up looping over telegraf with --once to see if I can see multiple dbus-daemon's starting. I am over 10k loops and nothing showing up yet.

Is there a way to disable the secret store completely?

Only with a custom build of Telegraf.

Assuming that the issue is with the same code of the secret store as last time, that dbus command runs in the init function of that library. Which means the function is run as soon as the library is imported, before we have any time to do anything else.

elangovanseshan · 2023-07-17T17:53:16Z

Thanks Joshua for your reply ,I could see this issue only from RHEL6 and i have the latest version deployed in RHEL7/8 and there i don't see any issue with DBUS.

ps -ef|grep -i telegraf
telegraf 14007     1 10 09:49 ?        00:00:01 /usr/bin/telegraf -pidfile /var/run/telegraf/telegraf.pid -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d
telegraf 14048     1  0 09:49 ?        00:00:00 /bin/dbus-daemon --fork --print-pid 4 --print-address 6 --session
root     14366  7614  0 09:49 pts/0    00:00:00 grep -i telegraf

powersj · 2023-07-17T20:07:44Z

@crflanigan, @elangovanseshan,

If you must have the newer version of Telegraf on RHEL 6, my suggestion then is to consider building telegraf with the custom builder. The result would provide you with a ~23Mb binary containing only the plugins you need and the secret store plugins would not be present.

git clone https://github.com/influxdata/telegraf
cd telegraf
go build -o ./tools/custom_builder/custom_builder ./tools/custom_builder
./tools/custom_builder/custom_builder --config <conf_file> --config-dir <conf_dir>

Would this be an option for you?

crflanigan · 2023-07-17T20:10:31Z

Hi @powersj,

We can look at that.
I had thought that the Secret Store is core to Telegraf irrespective of the configuration you use post release 1.25, is that right?

Is Telegraf not supported on RHEL 6, if so, when was the last release where it was supported?

Thanks!

elangovanseshan · 2023-07-17T20:12:15Z

Thank you @powersj ,Let me try custom builder without secret store plugins

powersj · 2023-07-17T20:27:46Z

I had thought that the Secret Store is core to Telegraf irrespective of the configuration you use post release 1.25, is that right?

Internally to Telegraf, the secret stores are treated like the other plugins, so that you could build telegraf without it.

Is Telegraf not supported on RHEL 6, if so, when was the last release where it was supported?

We have a published doc for supported platforms, which essentially says we support OSes that are under standard support. In line with that, RHEL 6 stopped being supported at the end of 2020. RHEL 7 will stop next June 2024.

While we will not go out of our way to break any previous releases, if we do make a change that breaks them we are less inclined to revert it nor will we continue to test it.

crflanigan · 2023-07-17T20:33:01Z

Ok @powersj ,

It sounds like patching this issue is unlikely since it's occuring on an unsupported OS, is that right?

Thanks!

powersj · 2023-07-18T13:28:56Z

It sounds like patching this issue is unlikely since it's occuring on an unsupported OS, is that right?

If you proposed a PR or an idea to get around this we would certainly consider it. We are not going to completely close the door on a fix.

crflanigan · 2023-07-18T14:52:08Z

@powersj,

Fair enough, thanks buddy!

elangovanseshan · 2023-07-18T19:23:36Z

@crflanigan, @elangovanseshan,

If you must have the newer version of Telegraf on RHEL 6, my suggestion then is to consider building telegraf with the custom builder. The result would provide you with a ~23Mb binary containing only the plugins you need and the secret store plugins would not be present.
git clone https://github.com/influxdata/telegraf
cd telegraf
go build -o ./tools/custom_builder/custom_builder ./tools/custom_builder
./tools/custom_builder/custom_builder --config <conf_file> --config-dir <conf_dir>
Would this be an option for you?

Thanks ! @powersj custom_builder is working fine for me. I passed the sample conf file to build the binary ,it contain the cpu disk diskio exec mem net swap system input plugins and it's working fine. We have multiple internal teams are using multiple input plugins other than i mentioned above so if we build the binary with limited input plugins, it will affect other internal customers, So we would like to build the custom binary with all input plugins but except secret store plugins . Is there any possible way to build it without passing conf file for each input plugins or can we build with dummy conf files without secret store plugins?

powersj · 2023-07-18T19:49:12Z

So now i would like to build the custom binary with all input plugins but except secret store plugins

You can get a list of all the input plugins by generating the default config and grep'ing out all the input headers:

make
./telegraf config > default.toml
grep "^# \[\[inputs.*\]\]" default.toml | cut -d' ' -f2 | sort | uniq

You could then add that to your example config or pass that as a second file to the custom builder.

You could also use the various build tags to build telegraf as the customization docs show using BUILDTAGS:

BUILDTAGS="custom,aggregators,inputs,outputs,parsers,processors,serializers" make

If you do start to go this route, please ensure you include everything you actually need ;) It is easy to forget or not realize you are using a serializer for example. This is why I like the custom builder + an actual config better.

elangovanseshan · 2023-07-27T19:22:59Z

@powersj our initial testing is working fine with custom Telegraf with limited input and output plugin and no evidence of dbus process .

also i would like to know that how can we add the serializers to custom build? I added the required input,output,aggregators,processors through the example conf but not sure about serializers .

Do we need to pass it through conf file or do we have any other option?

powersj · 2023-07-27T19:27:55Z

Do we need to pass it through conf file or do we have any other option?

You can reference any of the serializers the same way. For example, if you want only the JSON serialier you can add serializers.json to the build tags.

The way to determine these build tags is to look in each plugin's all folder and look at the build tags at the top of a file. This is the JSON all file and you can see that the JSON serializer is imported if this is not a custom build, if a user specifies serializers, which pulls in all serializers, or if they specify serializers.json.

Does that help?

elangovanseshan · 2023-07-27T19:34:59Z

Thank you @powersj let me try this out

one more thing for your information, initially i updated like dbus issue happening only in RHEL6 servers but we had an issue with RHEL7/8 as well .

So we are planning to go with custom telegraf with limited plugins .

powersj · 2024-03-19T18:22:40Z

@elangovanseshan, @crflanigan,

but we had an issue with RHEL7/8 as well .

Sorry I never responded to this. Looking at the mentioned gosnowflake issue it looks like a workaround is setting DBUS_SESSION_BUS_ADDRESS=$XDG_RUNTIME_DIR/bus in the environment as well.

For Telegraf, I am inclined to document this and link to the still open upstream issue. Thoughts?

crflanigan · 2024-03-28T20:50:14Z

Hi @powersj,

Sorry for the delayed response.

I actually commented on one of these issues for keyring and got a notification this morning that they may have resolved it? Seems like a lot of people use this library.

99designs/keyring#103

What do you think?

powersj · 2024-03-29T14:49:16Z

Hey @crflanigan,

Did someone delete their comment? Latest I see is from Apr 12, 2023.

Hipska · 2024-05-31T14:01:13Z

@powersj I think @crflanigan was referring to snowflakedb/gosnowflake#773 (comment)

BTW, I now have that message even when not using outputs.sql at all..

WARN[0000]log.go:244 gosnowflake.(*defaultLogger).Warn DBUS_SESSION_BUS_ADDRESS envvar looks to be not set, this can lead to runaway dbus-daemon processes. To avoid this, set envvar DBUS_SESSION_BUS_ADDRESS=$XDG_RUNTIME_DIR/bus (if it exists) or DBUS_SESSION_BUS_ADDRESS=/dev/null.
2024-05-31T14:00:14Z I! Loading config: test.toml
2024-05-31T14:00:14Z I! Starting Telegraf 1.31.0-35bff98f brought to you by InfluxData the makers of InfluxDB
2024-05-31T14:00:14Z I! Available plugins: 234 inputs, 9 aggregators, 32 processors, 26 parsers, 60 outputs, 6 secret-stores
2024-05-31T14:00:14Z I! Loaded inputs: snmp
2024-05-31T14:00:14Z I! Loaded aggregators:
2024-05-31T14:00:14Z I! Loaded processors:
2024-05-31T14:00:14Z I! Loaded secretstores:
2024-05-31T14:00:14Z W! Outputs are not used in testing mode!
2024-05-31T14:00:14Z I! Tags enabled:

This does not happen with telegraf 1.30.3

trauta · 2024-06-13T09:09:04Z

After upgrading telegraf to 1.31.0 all of our hosts seem to report the Warn DBUS_SESSION_BUS_ADDRESS log message. Here is an example log output from Telegraf 1.31.0 on a Ubuntu 24.04 hosts:

Jun 13 11:05:22 host.example.com systemd[1]: Starting telegraf.service - Telegraf...
Jun 13 11:05:22 host.example.com (telegraf)[495164]: telegraf.service: Referenced but unset environment variable evaluates to an empty string: TELEGRAF_OPTS
Jun 13 11:05:22 host.example.com telegraf[495164]: time="2024-06-13T11:05:22+02:00" level=warning msg="DBUS_SESSION_BUS_ADDRESS envvar looks to be not set, this can lead to runaway dbus-daemon processes. To avoid this, set envvar DBUS_SESSION_BUS_ADDRESS=$XDG_RUNTIME_DIR/bus (if it exists) or DBUS_SESSION_BUS_ADDRESS=/dev/null." func="gosnowflake.(*defaultLogger).Warn" file="log.go:244"
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Loading config: /etc/telegraf/telegraf.conf
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Starting Telegraf 1.31.0 brought to you by InfluxData the makers of InfluxDB
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Available plugins: 234 inputs, 9 aggregators, 32 processors, 26 parsers, 60 outputs, 6 secret-stores
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Loaded inputs: cpu disk diskio kernel mem net processes swap system
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Loaded aggregators:
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Loaded processors:
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Loaded secretstores:
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Loaded outputs: graphite
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! Tags enabled: host=host
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z I! [agent] Config: Interval:1m0s, Quiet:false, Hostname:"host", Flush Interval:10s
Jun 13 11:05:22 host.example.com telegraf[495164]: 2024-06-13T09:05:22Z W! DeprecationWarning: Value "false" for option "ignore_protocol_stats" of plugin "inputs.net" deprecated since version 1.27.3 and will be removed in 1.36.0: use the 'inputs.nstat' plugin instead for protocol stats
Jun 13 11:05:22 host.example.com systemd[1]: Started telegraf.service - Telegraf.

Hipska · 2024-06-13T09:12:37Z

@trauta Indeed, also seems to happen on RHEL and warned maintainers about it already 2 weeks ago: https://influxcommunity.slack.com/archives/C019JDRJAE7/p1717146621896149

powersj · 2024-06-17T19:30:41Z

When this was only on RHEL 6/7/8 I was less concerned about this especially given the upcoming EOL date. However, this is also appearing on newer releases as well (e.g. Ubuntu Noble).

The root cause is from the keyring dependency. We use this for secret store, but it appears the snowflake library we use also does. The keyring library has not been updated and does not appear to be planned to update anytime soon. Even if we moved to a fork, the snowflake library also uses it. I haven't played with the go replace enough, but it may be possible to use it?

The warning message from snowflake does tell you what to do to get rid of the message, so I don't consider this critical to fix, but it is something we are looking to figure out how to address.

l33 · 2024-07-08T10:51:13Z

I am seeing this on my setup on the current version of Debian (12, Bookworm), too. I don't see the warning about DBUS_SESSION_BUS_ADDRESS though. Just the stale dbus-daemon processes. One for every restart of telegraf.

srebhan · 2024-09-11T11:00:35Z

@l33, @crflanigan or @elangovanseshan could anyone please test the binary in #15860 to verify it solves this issue!?

srebhan · 2024-09-12T10:59:49Z

Closing this issue as it is likely solved by #15860. If I'm wrong, please reopen the issue or drop me a note so I can reopen it.

Hipska · 2024-09-12T11:02:25Z

That PR is not merged yet. Better would have been to add a resolves #13635 in its description, or manually link this issue.

pyksy · 2025-01-02T16:00:15Z

I just installed telegraf_1.33.0-1_amd64.deb (fresh install) from repos.influxdata.com on Debian unstable. Telegraf is still leaving behind dbus-daemon processes, one per telegraf (re)start.

No mention of dbus in debug log.

I'm new to Telegraf, but let me know if I can somehow help with debugging the issue.

srebhan · 2025-01-23T16:59:21Z

@pyksy do you have some idea on what exactly is opening DBUS? Could you post your (redacted) config?

pyksy · 2025-01-23T19:43:00Z

@srebhan unfortunately I have no idea why Telegraf spawns dbus-session processes and leaves them hanging around. I haven't really investigated much. Here's my uncommented telegraf.conf (nothing in telegraf.d)

[global_tags]`
  user = "influx"`
[agent]
  interval = "300s"
  round_interval = true
  metric_batch_size = 100
  metric_buffer_limit = 1000
  collection_jitter = "10s"
  flush_interval = "60s"
  flush_jitter = "10s"
  precision = "0s"
  debug = false
  logfile = ""
  logfile_rotation_interval = "7d"
  logfile_rotation_max_archives = 52
[[outputs.influxdb_v2]]
  urls = ["http://127.0.0.1:8086"]
  token = "..."
  organization = "hanuri"
  bucket = "telegraf"
[[inputs.cpu]]
  percpu = true
  totalcpu = true
  collect_cpu_time = false
  report_active = false
  core_tags = false
[[inputs.disk]]
  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"]
[[inputs.diskio]]
[[inputs.kernel]]
[[inputs.mem]]
[[inputs.processes]]
[[inputs.swap]]
[[inputs.system]]
[[inputs.apache]]
  urls = ["http://localhost/server-status?auto"]
[[inputs.bind]]
   urls = ["http://localhost:5380/xml/v3"]
[[inputs.conntrack]]
  collect = ["all", "percpu"]
[[inputs.dovecot]]
  servers = ["/var/run/dovecot/old-stats"]
[[inputs.interrupts]]
  cpu_as_tag = true
[[inputs.linux_cpu]]
  metrics = ["cpufreq"]
[[inputs.linux_sysctl_fs]]
[[inputs.mdstat]]
[[inputs.net]]
  interfaces = ["eth*", "enp0s[0-1]", "lo"]
  ignore_protocol_stats = true
[[inputs.netstat]]
[[inputs.nfsclient]]
[[inputs.postfix]]
  queue_directory = "/var/spool/postfix"
[[inputs.wireguard]]

Here's how the problem manifests:

root@hanuri ~ # ps aux|grep telegraf
root@hanuri ~ # service telegraf start
Starting the process telegraf [ OK ]
telegraf process was started [ OK ]
root@hanuri ~ # ps aux|grep telegraf
telegraf 233277  1.8  0.2 5705640 145012 pts/14 SLl  21:27   0:00 /usr/bin/telegraf -pidfile /var/run/telegraf/telegraf.pid -onfig /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d
telegraf 233301  0.0  0.0   7668  2036 ?        Ss   21:27   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session
root@hanuri ~ # service telegraf stop
telegraf process was stopped [ OK ]
root@hanuri ~ # ps aux|grep telegraf
telegraf 233301  0.0  0.0   7668  2036 ?        Ss   21:27   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session
root@hanuri ~ # service telegraf start
Starting the process telegraf [ OK ]
telegraf process was started [ OK ]
root@hanuri ~ # ps aux|grep telegraf
telegraf 233301  0.0  0.0   7668  2036 ?        Ss   21:27   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session
telegraf 233378  1.5  0.2 5705384 144352 pts/14 SLl  21:27   0:00 /usr/bin/telegraf -pidfile /var/run/telegraf/telegraf.pid -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d
telegraf 233402  0.0  0.0   7668  2104 ?        Ss   21:27   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session
root@hanuri ~ # service telegraf stop
telegraf process was stopped [ OK ]
root@hanuri ~ # ps aux|grep telegraf
telegraf 233301  0.0  0.0   7668  2036 ?        Ss   21:27   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session
telegraf 233402  0.0  0.0   7668  2104 ?        Ss   21:27   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session

srebhan · 2025-01-29T19:34:58Z

Just to be sure, does this also happen if you are starting Telegraf manually using the following command

# /usr/bin/telegraf -pidfile /var/run/telegraf/telegraf.pid -onfig /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d

i.e. without using the service?

pyksy · 2025-01-30T08:44:26Z

Thanks for the hint @srebhan. I did some more digging and it depends on whether the DBUS_SESSION_BUS_ADDRESS environment variable is set: launching the daemon as the telegraf user with the variable unset will leave a stray dbus-daemon (edit: namefix) process behind.

root@hanuri ~ # unset DBUS_SESSION_BUS_ADDRESS; su -c "/usr/bin/telegraf -pidfile /var/run/telegraf/telegraf.pid -config /etc/telegraf/telegraf.conf -config-directory /etc/telegraf/telegraf.d" -s /bin/bash telegraf
2025-01-30T08:24:43Z I! Loading config: /etc/telegraf/telegraf.conf
2025-01-30T08:24:43Z I! Starting Telegraf 1.33.1 brought to you by InfluxData the makers of InfluxDB
2025-01-30T08:24:43Z I! Available plugins: 236 inputs, 9 aggregators, 33 processors, 26 parsers, 63 outputs, 6 secret-stores
2025-01-30T08:24:43Z I! Loaded inputs: apache bind conntrack cpu disk diskio dovecot interrupts kernel linux_cpu linux_sysctl_fs mdstat mem net netstat nfsclient postfix processes swap system wireguard
2025-01-30T08:24:43Z I! Loaded aggregators:
2025-01-30T08:24:43Z I! Loaded processors:
2025-01-30T08:24:43Z I! Loaded secretstores:
2025-01-30T08:24:43Z I! Loaded outputs: influxdb_v2
2025-01-30T08:24:43Z I! Tags enabled: host=hanuri.dap user=influx
2025-01-30T08:24:43Z I! [agent] Config: Interval:5m0s, Quiet:false, Hostname:"hanuri", Flush Interval:1m0s
2025-01-30T08:24:43Z W! [agent] The default value of 'skip_processors_after_aggregators' will change to 'true' with Telegraf v1.40.0! If you need the current default behavior, please explicitly set the option to 'false'!
^C
Session terminated, killing shell...2025-01-30T08:24:48Z I! [agent] Hang on, flushing any cached metrics before shutdown
2025-01-30T08:24:48Z I! [agent] Stopping running outputs
 ...killed.

root@hanuri ~ # ps aux|grep telegraf
telegraf 108504  0.0  0.0   7660  2308 ?        Ss   10:24   0:00 /usr/bin/dbus-daemon --syslog --fork --print-pid 4 --print-address 6 --session

Running telegraf as root will not leave a stray dbus-daemon process.

I still run SysV init on this box. If I launch telegraf by running the init script directly (/etc/init.d/telegraf start) it does not launch the extra dbus-daemon, because the launching root shell has a valid DBUS_SESSION_BUS_ADDRESS environment variable set that gets passed to telegraf. if I use the service command (service telegraf start) then the extra dbus-daemon gets spawned because the service command runs the init script in a clean environment, with DBUS_SESSION_BUS_ADDRESS unset.

I guess not that many people run Debian with sysvinit nowadays, so the impact probably isn't that big. Might be a concern to folks running Devuan (the systemd-free Debian fork) but I haven't tested it personally.

srebhan · 2025-01-30T13:52:10Z

Would the workaround setting DBUS_SESSION_BUS_ADDRESS=/dev/null work for you?

I'm asking because digging into the code, it seems like solving this would involve replacing an (unmaintained) dependency which is a bigger issue...

pyksy · 2025-01-30T14:23:48Z

@srebhan yes, it does :) and could be a quick fix in scripts/init.sh. Thanks!

--- scripts/init.sh.orig        2025-01-30 16:15:21.174108939 +0200
+++ scripts/init.sh     2025-01-30 16:20:45.911103092 +0200
@@ -136,6 +136,10 @@
             log_failure_msg "set open file limit to $OPEN_FILE_LIMIT"
         fi
 
+        # Set DBUS_SESSION_BUS_ADDRESS if unset to prevent the daemon from
+        # spawning a stray dbus-session process
+        export DBUS_SESSION_BUS_ADDRESS="${DBUS_SESSION_BUS_ADDRESS:-/dev/null}"
+
         log_success_msg "Starting the process" "$name"
         if command -v startproc >/dev/null; then
             startproc -u "$USER" -g "$GROUP" -p "$pidfile" -q -- "$daemon" -pidfile "$pidfile" -config "$config" -config-directory "$confdir" $TELEGRAF_OPTS

srebhan · 2025-01-30T14:32:46Z

Could you please open a pull request with that fix!?

elangovanseshan added the bug label Jul 17, 2023

powersj added the waiting for response label Jul 17, 2023

telegraf-tiger bot removed the waiting for response label Jul 17, 2023

powersj added the waiting for response label Jul 17, 2023

telegraf-tiger bot removed the waiting for response label Jul 17, 2023

mihaitodor mentioned this issue Aug 29, 2023

Zombie dbus processes generated by github.com/99designs/keyring snowflakedb/gosnowflake#773

Closed

powersj added the waiting for response label Mar 19, 2024

telegraf-tiger bot removed the waiting for response label Mar 28, 2024

powersj added the upstream label May 10, 2024

Hipska mentioned this issue Jul 12, 2024

Translate option isn’t working #15617

Closed

srebhan self-assigned this Aug 28, 2024

Hipska mentioned this issue Sep 10, 2024

chore(deps): Bump github.com/snowflakedb/gosnowflake from 1.10.0 to 1.11.1 #15860

Merged

srebhan added the waiting for response label Sep 11, 2024

srebhan closed this as completed Sep 12, 2024

telegraf-tiger bot removed the waiting for response label Sep 12, 2024

srebhan reopened this Sep 12, 2024

DStrand1 closed this as completed in #15860 Sep 12, 2024

srebhan reopened this Jan 23, 2025

pyksy mentioned this issue Jan 30, 2025

fix(service): Set address to prevent orphaned dbus-session processes #16454

Merged

1 task

DStrand1 closed this as completed in #16454 Feb 4, 2025

Telegraf Generating Orphaned DBus Processes on RHEL Servers #2 #13635

Telegraf Generating Orphaned DBus Processes on RHEL Servers #2 #13635

Comments

elangovanseshan commented Jul 17, 2023 • edited Loading

Relevant telegraf.conf

Logs from Telegraf

System info

Docker

Steps to reproduce

Expected behavior

Actual behavior

Additional info

Tasks

crflanigan commented Jul 17, 2023

crflanigan commented Jul 17, 2023 • edited Loading

elangovanseshan commented Jul 17, 2023 • edited Loading

elangovanseshan commented Jul 17, 2023 • edited Loading

crflanigan commented Jul 17, 2023

powersj commented Jul 17, 2023

elangovanseshan commented Jul 17, 2023

powersj commented Jul 17, 2023

crflanigan commented Jul 17, 2023 • edited Loading

elangovanseshan commented Jul 17, 2023

powersj commented Jul 17, 2023

crflanigan commented Jul 17, 2023

powersj commented Jul 18, 2023

crflanigan commented Jul 18, 2023

elangovanseshan commented Jul 18, 2023 • edited Loading

powersj commented Jul 18, 2023 • edited Loading

elangovanseshan commented Jul 27, 2023

powersj commented Jul 27, 2023

elangovanseshan commented Jul 27, 2023

powersj commented Mar 19, 2024

crflanigan commented Mar 28, 2024

powersj commented Mar 29, 2024

Hipska commented May 31, 2024

trauta commented Jun 13, 2024

Hipska commented Jun 13, 2024

powersj commented Jun 17, 2024

l33 commented Jul 8, 2024 • edited Loading

srebhan commented Sep 11, 2024 • edited Loading

srebhan commented Sep 12, 2024

Hipska commented Sep 12, 2024

pyksy commented Jan 2, 2025

srebhan commented Jan 23, 2025

pyksy commented Jan 23, 2025 • edited by srebhan Loading

srebhan commented Jan 29, 2025

pyksy commented Jan 30, 2025 • edited Loading

srebhan commented Jan 30, 2025

pyksy commented Jan 30, 2025

srebhan commented Jan 30, 2025

elangovanseshan commented Jul 17, 2023 •

edited

Loading

crflanigan commented Jul 17, 2023 •

edited

Loading

elangovanseshan commented Jul 17, 2023 •

edited

Loading

elangovanseshan commented Jul 17, 2023 •

edited

Loading

crflanigan commented Jul 17, 2023 •

edited

Loading

elangovanseshan commented Jul 18, 2023 •

edited

Loading

powersj commented Jul 18, 2023 •

edited

Loading

l33 commented Jul 8, 2024 •

edited

Loading

srebhan commented Sep 11, 2024 •

edited

Loading

pyksy commented Jan 23, 2025 •

edited by srebhan

Loading

pyksy commented Jan 30, 2025 •

edited

Loading