57 |
static int check_checksum(Process_T); |
static int check_checksum(Process_T); |
58 |
static int checksum_helper(Process_T, char *, char *); |
static int checksum_helper(Process_T, char *, char *); |
59 |
static int check_timestamp(Process_T, Timestamp_T, char *); |
static int check_timestamp(Process_T, Timestamp_T, char *); |
60 |
|
static int check_device(Process_T, Device_T, char *); |
61 |
static void connection_timeout(int); |
static void connection_timeout(int); |
62 |
static void reset_resource_counter(Process_T); |
static void reset_resource_counter(Process_T); |
63 |
static void vlog(char * report, int n, Process_T p, char *m,...); |
static void vlog(char * report, int n, Process_T p, char *m,...); |
139 |
Port_T pp; |
Port_T pp; |
140 |
Resource_T pr; |
Resource_T pr; |
141 |
Timestamp_T tl; |
Timestamp_T tl; |
142 |
|
Device_T td; |
143 |
pid_t pid= -1; |
pid_t pid= -1; |
144 |
sigset_t ns,os; |
sigset_t ns,os; |
145 |
|
struct stat stat_buf; |
146 |
char report[STRLEN]; |
char report[STRLEN]; |
147 |
|
|
148 |
ASSERT(p); |
ASSERT(p); |
149 |
|
ASSERT(p->path); |
150 |
|
|
151 |
/* Obtain process action mutex */ |
/* Obtain process action mutex */ |
152 |
LOCK(p->mutex) |
LOCK(p->mutex) |
163 |
sigaddset(&ns, SIGTERM); |
sigaddset(&ns, SIGTERM); |
164 |
pthread_sigmask(SIG_BLOCK, &ns, &os); |
pthread_sigmask(SIG_BLOCK, &ns, &os); |
165 |
|
|
|
/* Test for running process */ |
|
|
if(!(pid= is_process_running(p))) { |
|
166 |
|
|
167 |
/* Reset the proc info object to prevent false data in the first run */ |
/** Context specific task actions */ |
168 |
memset(p->procinfo, 0, sizeof *(p->procinfo)); |
switch(p->task) { |
169 |
|
|
170 |
|
case TASK_PROCESS: |
171 |
|
|
172 |
|
/* Test for running process */ |
173 |
|
if(!(pid= is_process_running(p))) { |
174 |
|
|
175 |
|
/* Reset the proc info object to prevent false data in the first run */ |
176 |
|
memset(p->procinfo, 0, sizeof *(p->procinfo)); |
177 |
|
|
178 |
do_start(p, "Reason: Process is not running."); |
do_start(p, "Reason: Process is not running."); |
179 |
goto reinstall; |
goto reinstall; |
180 |
|
|
181 |
} else { |
} else { |
182 |
|
|
183 |
if(Run.debug) { |
if(Run.debug) { |
184 |
|
|
185 |
log("'%s' is running with pid %d\n", p->name, (int)pid); |
log("'%s' is running with pid %d\n", p->name, (int)pid); |
186 |
|
|
187 |
} |
} |
188 |
|
|
189 |
} |
} |
190 |
|
|
191 |
if(Run.doprocess) { |
if(Run.doprocess) { |
192 |
|
|
193 |
if(update_process_data(p, pt, treesize, pid)) { |
if(update_process_data(p, pt, treesize, pid)) { |
194 |
|
|
195 |
if(! check_process_state(p, report)) { |
if(! check_process_state(p, report)) { |
196 |
|
|
197 |
smtp_alert_resource(p, "Reason: %s\n", report); |
smtp_alert_resource(p, "Reason: %s\n", report); |
198 |
|
|
199 |
} else { |
} else { |
200 |
|
|
201 |
if(Run.debug) { |
if(Run.debug) { |
202 |
|
|
203 |
log("'%s' check_process_state() passed.\n", p->name); |
log("'%s' check_process_state() passed.\n", p->name); |
204 |
|
|
205 |
} |
} |
206 |
|
|
207 |
} |
} |
208 |
|
|
209 |
for(pr= p->resourcelist; pr; pr= pr->next) { |
for(pr= p->resourcelist; pr; pr= pr->next) { |
210 |
|
|
211 |
if(!check_resources(p, pr, report)) { |
if(!check_resources(p, pr, report)) { |
212 |
|
|
213 |
switch(pr->action) { |
switch(pr->action) { |
214 |
case ACTION_ALERT: |
case ACTION_ALERT: |
215 |
if(p->def_timeout) p->nstart++; |
if(p->def_timeout) p->nstart++; |
216 |
smtp_alert_resource(p, "Reason: %s\n", report); |
smtp_alert_resource(p, "Reason: %s\n", report); |
217 |
/* We are also interested in other alerts/stops/restarts! */ |
/* We are also interested in other alerts/stops/restarts! */ |
218 |
pr->cycle=0; |
pr->cycle=0; |
219 |
break; |
break; |
220 |
|
|
221 |
case ACTION_STOP: |
case ACTION_STOP: |
222 |
do_stop(p, "Reason: %s\n", report); |
do_stop(p, "Reason: %s\n", report); |
223 |
reset_resource_counter(p); |
reset_resource_counter(p); |
224 |
goto reinstall; |
goto reinstall; |
225 |
|
|
226 |
case ACTION_RESTART: |
case ACTION_RESTART: |
227 |
do_restart(p, "Reason: %s\n", report); |
do_restart(p, "Reason: %s\n", report); |
228 |
reset_resource_counter(p); |
reset_resource_counter(p); |
229 |
goto reinstall; |
goto reinstall; |
230 |
|
|
231 |
default: |
default: |
232 |
log("'%s' Unknow resource failure action.\n", p->name); |
log("'%s' Unknow resource failure action.\n", p->name); |
233 |
break; |
break; |
234 |
|
} |
235 |
|
|
236 |
} |
} |
237 |
|
|
238 |
|
} |
239 |
|
|
240 |
|
} else { |
241 |
|
|
242 |
} |
log("'%s' failed to get process data\n", p->name); |
243 |
|
|
244 |
} |
} |
|
|
|
|
} else { |
|
245 |
|
|
246 |
log("'%s' failed to get process data\n", p->name); |
} |
247 |
|
|
248 |
|
/* Test each host:port and protocol in the process's portlist */ |
249 |
|
for(pp= p->portlist; pp; pp= pp->next) { |
250 |
|
|
251 |
|
if(!check_connection(p, pp, report)) { |
252 |
|
|
253 |
|
do_restart(p, "Reason: %s\n", report); |
254 |
|
goto reinstall; |
255 |
|
|
256 |
|
} |
257 |
|
|
258 |
|
} |
259 |
|
break; |
260 |
|
|
261 |
|
case TASK_DEVICE: |
262 |
|
|
263 |
|
/* Test devices */ |
264 |
|
if(p->devicelist) { |
265 |
|
|
266 |
|
if(!get_fsusage(p->path, p->devinfo)) { |
267 |
|
do_restart(p, "Reason: unable to read device state\n"); |
268 |
|
goto reinstall; |
269 |
|
} |
270 |
|
|
271 |
|
for(td= p->devicelist; td; td= td->next) { |
272 |
|
|
273 |
|
if(!check_device(p, td, report)) { |
274 |
|
|
275 |
|
switch(td->action) { |
276 |
|
case ACTION_ALERT: |
277 |
|
smtp_alert_resource(p, "Reason: %s\n", report); |
278 |
|
break; |
279 |
|
|
280 |
|
case ACTION_STOP: |
281 |
|
do_stop(p, "Reason: %s\n", report); |
282 |
|
goto reinstall; |
283 |
|
|
284 |
|
case ACTION_RESTART: |
285 |
|
do_restart(p, "Reason: %s\n", report); |
286 |
|
goto reinstall; |
287 |
|
|
288 |
|
default: |
289 |
|
log("'%s' Unknow device failure action.\n", p->name); |
290 |
|
break; |
291 |
|
} |
292 |
|
|
293 |
|
} |
294 |
|
|
295 |
|
} |
296 |
|
|
297 |
} |
} |
298 |
|
break; |
299 |
|
|
300 |
|
case TASK_FILE: |
301 |
|
|
302 |
|
if(stat(p->path, &stat_buf) != 0) { |
303 |
|
log("'%s' doesn't exist\n", p->name); |
304 |
|
do_restart(p, "Reason: file doesn't exist\n"); |
305 |
|
goto reinstall; |
306 |
|
} |
307 |
|
|
308 |
|
if(!S_ISREG(stat_buf.st_mode)) { |
309 |
|
log("'%s' is not regular file\n", p->name); |
310 |
|
do_restart(p, "Reason: object is not file\n"); |
311 |
|
goto reinstall; |
312 |
|
} |
313 |
|
|
314 |
|
if( p->perm != -1 && (stat_buf.st_mode & 07777) != p->perm ) { |
315 |
|
log("'%s' permission test failed -- current permission: %o\n", |
316 |
|
p->name, stat_buf.st_mode&07777); |
317 |
|
do_restart(p, "Reason: permission test failed: %o\n", stat_buf.st_mode&07777); |
318 |
|
goto reinstall; |
319 |
|
} |
320 |
|
|
321 |
|
if(Run.debug) |
322 |
|
log("'%s' file permission check passed [current permission=%o]\n", |
323 |
|
p->name, stat_buf.st_mode&07777); |
324 |
|
|
325 |
|
break; |
326 |
|
|
327 |
|
case TASK_DIRECTORY: |
328 |
|
|
329 |
|
if(stat(p->path, &stat_buf) != 0) { |
330 |
|
log("'%s' doesn't exist\n", p->name); |
331 |
|
do_restart(p, "Reason: directory doesn't exist\n"); |
332 |
|
goto reinstall; |
333 |
|
} |
334 |
|
|
335 |
|
if(!S_ISDIR(stat_buf.st_mode)) { |
336 |
|
log("'%s' is not directory\n", p->name); |
337 |
|
do_restart(p, "Reason: object is not directory\n"); |
338 |
|
goto reinstall; |
339 |
|
} |
340 |
|
|
341 |
|
if( p->perm != -1 && (stat_buf.st_mode & 07777) != p->perm ) { |
342 |
|
log("'%s' permission test failed -- current permission: %o\n", |
343 |
|
p->name, stat_buf.st_mode&07777); |
344 |
|
do_restart(p, "Reason: permission test failed: %o\n", stat_buf.st_mode&07777); |
345 |
|
goto reinstall; |
346 |
|
} |
347 |
|
|
348 |
|
if(Run.debug) |
349 |
|
log("'%s' directory permission check passed [current permission=%o]\n", |
350 |
|
p->name, stat_buf.st_mode&07777); |
351 |
|
|
352 |
|
break; |
353 |
|
|
354 |
|
default: |
355 |
|
break; |
356 |
|
|
357 |
} |
} |
358 |
|
|
359 |
|
|
360 |
|
/** Common task actions */ |
361 |
for(tl= p->timestamplist; tl; tl= tl->next) { |
for(tl= p->timestamplist; tl; tl= tl->next) { |
362 |
|
|
363 |
if(!check_timestamp(p, tl, report)) { |
if(!check_timestamp(p, tl, report)) { |
386 |
|
|
387 |
} |
} |
388 |
|
|
|
/* Test each host:port and protocol in the process's portlist */ |
|
|
for(pp= p->portlist; pp; pp= pp->next) { |
|
|
|
|
|
if(!check_connection(p, pp, report)) { |
|
|
|
|
|
do_restart(p, "Reason: %s\n", report); |
|
|
goto reinstall; |
|
|
|
|
|
} |
|
|
|
|
|
} |
|
|
|
|
389 |
reinstall: |
reinstall: |
390 |
|
|
391 |
/* Remove the SIGTERM block */ |
/* Remove the SIGTERM block */ |
878 |
} |
} |
879 |
break; |
break; |
880 |
|
|
881 |
case RESOURCE_ID_CHILDREN: |
case RESOURCE_ID_CHILDREN: |
882 |
|
|
883 |
if(compare_value(pr->operator, pi->children, pr->limit)) { |
if(compare_value(pr->operator, pi->children, pr->limit)) { |
884 |
|
|
885 |
vlog(report, STRLEN, p, |
vlog(report, STRLEN, p, |
886 |
"children of %i matches resource limit " |
"children of %i matches resource limit " |
887 |
"[children%s%i]", |
"[children%s%i]", |
888 |
pi->children, operatorshortnames[pr->operator], |
pi->children, operatorshortnames[pr->operator], |
889 |
pr->limit); |
pr->limit); |
890 |
|
|
891 |
okay= FALSE; |
okay= FALSE; |
892 |
|
|
893 |
} else { |
} else { |
894 |
|
|
895 |
if(Run.debug) { |
if(Run.debug) { |
896 |
|
|
897 |
log("'%s' children check passed " |
log("'%s' children check passed " |
898 |
"[current children=%i]\n", |
"[current children=%i]\n", |
899 |
p->name, pi->children); |
p->name, pi->children); |
900 |
|
|
901 |
} |
} |
902 |
} |
} |
903 |
break; |
break; |
904 |
|
|
905 |
case RESOURCE_ID_TOTAL_MEM_KBYTE: |
case RESOURCE_ID_TOTAL_MEM_KBYTE: |
906 |
|
|
907 |
if(compare_value(pr->operator, pi->total_mem_kbyte, pr->limit)) { |
if(compare_value(pr->operator, pi->total_mem_kbyte, pr->limit)) { |
908 |
|
|
909 |
vlog(report, STRLEN, p, |
vlog(report, STRLEN, p, |
910 |
"total mem amount of %ldkB matches resource limit" |
"total mem amount of %ldkB matches resource limit" |
911 |
" [total mem amount%s%ldkB]", |
" [total mem amount%s%ldkB]", |
912 |
pi->total_mem_kbyte, operatorshortnames[pr->operator], |
pi->total_mem_kbyte, operatorshortnames[pr->operator], |
913 |
pr->limit); |
pr->limit); |
914 |
|
|
915 |
okay= FALSE; |
okay= FALSE; |
916 |
|
|
917 |
} else { |
} else { |
918 |
|
|
919 |
if(Run.debug) { |
if(Run.debug) { |
920 |
|
|
921 |
log("'%s' total mem amount check passed" |
log("'%s' total mem amount check passed" |
922 |
" [current total mem amount=%ldkB]\n", |
" [current total mem amount=%ldkB]\n", |
923 |
p->name, pi->total_mem_kbyte); |
p->name, pi->total_mem_kbyte); |
924 |
|
|
925 |
} |
} |
926 |
} |
} |
927 |
break; |
break; |
1152 |
|
|
1153 |
} |
} |
1154 |
|
|
1155 |
|
|
1156 |
|
/** |
1157 |
|
* Returns TRUE if the device test succeeded |
1158 |
|
* otherwise FALSE. |
1159 |
|
*/ |
1160 |
|
static int check_device(Process_T t, Device_T td, char *report) { |
1161 |
|
|
1162 |
|
ASSERT(t); |
1163 |
|
ASSERT(td); |
1164 |
|
|
1165 |
|
if( (td->limit_percent < 0) && (td->limit_absolute < 0) ) { |
1166 |
|
|
1167 |
|
log("'%s' error: device limit not set\n", t->name); |
1168 |
|
return FALSE; |
1169 |
|
|
1170 |
|
} |
1171 |
|
|
1172 |
|
switch(td->resource) { |
1173 |
|
|
1174 |
|
case RESOURCE_INODE: |
1175 |
|
|
1176 |
|
if(t->devinfo->f_files <= 0) { |
1177 |
|
|
1178 |
|
if(Run.debug) { |
1179 |
|
|
1180 |
|
log("'%s' filesystem doesn't support inodes\n", t->name); |
1181 |
|
|
1182 |
|
} |
1183 |
|
|
1184 |
|
return TRUE; |
1185 |
|
|
1186 |
|
} |
1187 |
|
|
1188 |
|
if(td->limit_percent >= 0) { |
1189 |
|
|
1190 |
|
if(compare_value( |
1191 |
|
td->operator, |
1192 |
|
100 * (t->devinfo->f_files - t->devinfo->f_filesfree) / t->devinfo->f_files, |
1193 |
|
td->limit_percent)) { |
1194 |
|
|
1195 |
|
vlog(report, STRLEN, t, |
1196 |
|
"inode usage %d%% matches resource limit [inode usage%s%d%%]", |
1197 |
|
100 * (t->devinfo->f_files - t->devinfo->f_filesfree) / t->devinfo->f_files, |
1198 |
|
operatorshortnames[td->operator], |
1199 |
|
td->limit_percent); |
1200 |
|
|
1201 |
|
return FALSE; |
1202 |
|
|
1203 |
|
} |
1204 |
|
|
1205 |
|
} else { |
1206 |
|
|
1207 |
|
if(compare_value(td->operator, t->devinfo->f_files - t->devinfo->f_filesfree, td->limit_absolute)) { |
1208 |
|
|
1209 |
|
vlog(report, STRLEN, t, |
1210 |
|
"inode usage %d matches resource limit [inode usage%s%d]", |
1211 |
|
t->devinfo->f_files - t->devinfo->f_filesfree, |
1212 |
|
operatorshortnames[td->operator], |
1213 |
|
td->limit_absolute); |
1214 |
|
|
1215 |
|
return FALSE; |
1216 |
|
|
1217 |
|
} |
1218 |
|
|
1219 |
|
} |
1220 |
|
|
1221 |
|
if(Run.debug) { |
1222 |
|
|
1223 |
|
log("'%s' inode usage check passed [current inode usage=%.1f%%]\n", |
1224 |
|
t->name, |
1225 |
|
(float) 100 * (t->devinfo->f_files - t->devinfo->f_filesfree) / t->devinfo->f_files); |
1226 |
|
|
1227 |
|
} |
1228 |
|
|
1229 |
|
return TRUE; |
1230 |
|
|
1231 |
|
case RESOURCE_SPACE: |
1232 |
|
|
1233 |
|
if(td->limit_percent >= 0) { |
1234 |
|
|
1235 |
|
if(compare_value( |
1236 |
|
td->operator, |
1237 |
|
100 * (t->devinfo->f_blocks - t->devinfo->f_blocksfree) / t->devinfo->f_blocks, |
1238 |
|
td->limit_percent)) { |
1239 |
|
|
1240 |
|
vlog(report, STRLEN, t, |
1241 |
|
"space usage %d%% matches resource limit [space usage%s%d%%]", |
1242 |
|
100 * (t->devinfo->f_blocks - t->devinfo->f_blocksfree) / t->devinfo->f_blocks, |
1243 |
|
operatorshortnames[td->operator], |
1244 |
|
td->limit_percent); |
1245 |
|
|
1246 |
|
return FALSE; |
1247 |
|
|
1248 |
|
} |
1249 |
|
|
1250 |
|
} else { |
1251 |
|
|
1252 |
|
if(compare_value(td->operator, t->devinfo->f_blocks - t->devinfo->f_blocksfree, td->limit_absolute)) { |
1253 |
|
|
1254 |
|
vlog(report, STRLEN, t, |
1255 |
|
"space usage %d blocks matches resource limit [space usage%s%d blocks]", |
1256 |
|
t->devinfo->f_blocks - t->devinfo->f_blocksfree, |
1257 |
|
operatorshortnames[td->operator], |
1258 |
|
td->limit_absolute); |
1259 |
|
|
1260 |
|
return FALSE; |
1261 |
|
|
1262 |
|
} |
1263 |
|
|
1264 |
|
} |
1265 |
|
|
1266 |
|
if(Run.debug) { |
1267 |
|
|
1268 |
|
log("'%s' space usage check passed [current space usage=%.1f%%]\n", |
1269 |
|
t->name, |
1270 |
|
(float) 100 * (t->devinfo->f_blocks - t->devinfo->f_blocksfree) / t->devinfo->f_blocks); |
1271 |
|
|
1272 |
|
} |
1273 |
|
|
1274 |
|
return TRUE; |
1275 |
|
|
1276 |
|
default: |
1277 |
|
|
1278 |
|
log("'%s' error: unknow resource type: [%d]\n", t->name, td->resource); |
1279 |
|
return FALSE; |
1280 |
|
|
1281 |
|
} |
1282 |
|
|
1283 |
|
} |
1284 |
|
|
1285 |
|
|
1286 |
/** |
/** |
1287 |
* Signal handler for connection timeout |
* Signal handler for connection timeout |