-
Notifications
You must be signed in to change notification settings - Fork 0
/
taskXDF-timer
executable file
·161 lines (137 loc) · 7.84 KB
/
taskXDF-timer
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
#! /bin/bash
# This interval timer script executes taskXDF in the same folder.
# To avoid having to provide a password to edit the app_config.xml file,
# make yourself a member of the boinc permissions group. $ sudo usermod -a -G boinc yourusername
# and be sure that the boinc group has write permission to the app_config file:
# -rw-rw-r-- 1 boinc boinc 1523 Aug 23 08:02 /var/lib/boinc-client/projects/einstein.phys.uwm.edu/app_config.xml
# After adding yourself to the boinc group, a system reboot is needed for the new permissions to take effect.
# Running a timer script can sidestep the need for setting up a systemd .service and .timer,
# but you will need to manually launch this timer script after each reboot.
# TIP: Run taskXDF-timer on a 60 sec interval to get responsive task X changes.
time_arg=$1
# Need to read configuration settings and create .log in case it is not present.
. ./taskXDF.cfg
touch "$log_file"
target=taskXDF
script="$(basename -- $0)"
version='0.10.2'
if (( time_arg > 0 && time_arg < 60 )); then
time_arg=60
echo "Interval time parameter cannot be less than 60 seconds."
echo " Values between 0 and 60 will be converted to 60."
echo "TIP: A value of zero is an alternative to the --status argument."
fi
# Terminal font colors are defined in .cfg.
go=false
if [[ $time_arg == --header ]]; then
echo
echo "Report header:"
echo -n "date time │ queued │ VRAM% │ GTT% │ taskGB │ X │ "
printf "DFs: ${blue}[running]${nc} ${orng}[waiting]${nc} ${brn}[ready]${nc}\n"
echo "Datetime of $target script execution
————————————————│ queued: total # of tasks
—————————————————————————│ %'s: high card, max GPU memory in use
————————————————————————————————————————│ taskGB: high card, avg task VRAM GB used
—————————————————————————————————————————————————│ X: current task multiple
—————————————————————————————————————————————————————│ Delta Frequencies of GW tasks"
echo "NOTES:"
echo "'High card' is the card with the highest value among all active cards."
echo "DF values are listed from most progressed to least progessed tasks."
echo "When all running DFs are equal, taskGB is the GB requirement of that DF."
printf "${brn}'ready' is next task 'Ready to run', one listed for each active card.${nc}\n\n"
exit 0
elif [[ $time_arg == 0 ]] || [[ $time_arg == --status ]]; then
echo "
With an argument of $time_arg, $script is executed in status-only mode."
/bin/bash ./$target --status
exit 0
elif [[ $# != 1 ]] ||
[[ $time_arg =~ -help ]] ||
[[ $time_arg =~ [.] ]]; then
echo "
A time interval, in seconds, for running the taskXDF-timer script is required
to automate changes to task X.
e.g., $0 60
Values less than 60 will be converted to 60. Decimal seconds are invalid.
Optional arguments: --header, --status, or --help
--header, provides explanation of types of interval data reported, then exits.
--status, provides current BOINC and GPU metrics, then exits; does not change task X.
--help, brings up this message, then exits.
This timer script provides continuous timed runs of the $target bash script.
A time interval of 0 (zero) provides a one-off status report from $target,
otherwise a non-zero interval time provides interval reports and taskX adjustments.
A time interval of 60 will provide good script responsiveness.
To monitor task status on a regular interval without changing task X, edit the
taskXDF.cfg file to monitor_only=yes (default is 'no').
Supported AMD GPUs: Ellesmere, Polaris, Vega, or Navi that are running
Einstein@Home gravitational wave and gamma-ray GPU tasks.
Task X is changed only for the GW app.
$target may not properly if CPU tasks are queued or running.
A log file of Terminal output is created in the current folder.
Script run parameters can be configured in the $target.cfg text file.
Version: $version"
echo
exit 0
# Need to catch any unexpected argument that's made it this far...
elif [[ $time_arg =~ [^0-9] ]]; then
echo "Invalid parameter of \"$time_arg\". Try: $0 --help"
exit 0
else
# Flag to initiate the infinite while loop for interval timing.
go=true
fi
printf "\nStarting new instance of $script >>>>>>\n" >>$log_file
# $report_int (number of interval cycles between reports) is set in the .cfg file.
report_t=$(bc <<<"scale=1; $time_arg * $report_int / 60")
# Provide an initial information message when timer is started.
echo
echo "$script is executing $target every $time_arg seconds; $target reports every $report_t minutes.
The reporting interval is set in the $target.cfg file.
Running... (can be stopped with CTRL+C)" | tee -a $log_file
if [[ $monitor_only == yes ]]; then
echo "Running in monitor-only mode, as set in $target.cfg. Task X will not be changed,
but auto-suspend of long-running tasks is active if set in $target.cfg.
" | tee -a $log_file
fi
echo -n "date time │ queued │ VRAM% │ GTT% │ taskGB │ X │ " | tee -a $log_file
printf "DFs: ${blue}[running]${nc} ${orng}[waiting]${nc} ${brn}[ready]${nc}\n"
# Remove escape codes for log text.
echo "DFs: [running] [waiting] [ready]" >>$log_file
echo "————————————————│————————│———————│——————│————————│———│————————————————————————————————————————" | tee -a $log_file
# Run continuously until manually stopped. Runs the taskXDF (target) bash script each cycle.
# The --go argument is used as a tag so that time interval value can be passed to $target for time calculations.
# The cycle arg passes the cycle count to $target so that $report_int can be used.
# A reminder is provided every x cycles for how to stop the main script
# along with a table header and current script run conditions (from main script).
cycle=0
yikes=0
reminder_count=$((report_int * 20))
while [[ $go == true ]]
do
/bin/bash ./${target} --go${time_arg} $cycle
sleep $time_arg
((cycle++))
((yikes++))
if ((cycle == reminder_count)); then
echo ">> $script is executing $target every $time_arg seconds; $target reports every $report_t minutes. <<
(can be stopped with CTRL+C)" | tee -a $log_file
echo
echo -n "date time │ queued │ VRAM% │ GTT% │ taskGB │ X │ " | tee -a $log_file
printf "DFs: ${blue}[running]${nc} ${orng}[waiting]${nc} ${brn}[ready]${nc}\n"
# ..but need to remove font color escape codes from log text.
echo "DFs: [running] [waiting] [ready]" >>$log_file
echo -n "————————————————│————————│———————│——————│————————│———│" | tee -a $log_file
echo "————————————————————————————————————————" | tee -a $log_file
cycle=0
fi
# Set notice for ~28 days based on a 60 sec time_arg (86400 sec per day).
if (( yikes >= 4 * 10**4 )); then
days=$((time_arg * yikes / 86400))
days=$((86400 / time_arg ))
echo " **** YIKES! $script has been running for $yikes cycles, $days days. ****
Restart $script to get rid of this annoying reminder." | tee -a $log_file
# echo "Just in case you forgot about me, I will EXIT NOW...." | tee -a $log_file
echo
# exit 0
fi
done