corosync 3.1.9
main.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2002-2006 MontaVista Software, Inc.
3 * Copyright (c) 2006-2021 Red Hat, Inc.
4 *
5 * All rights reserved.
6 *
7 * Author: Steven Dake (sdake@redhat.com)
8 *
9 * This software licensed under BSD license, the text of which follows:
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions are met:
13 *
14 * - Redistributions of source code must retain the above copyright notice,
15 * this list of conditions and the following disclaimer.
16 * - Redistributions in binary form must reproduce the above copyright notice,
17 * this list of conditions and the following disclaimer in the documentation
18 * and/or other materials provided with the distribution.
19 * - Neither the name of the MontaVista Software, Inc. nor the names of its
20 * contributors may be used to endorse or promote products derived from this
21 * software without specific prior written permission.
22 *
23 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
24 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
26 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
27 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
33 * THE POSSIBILITY OF SUCH DAMAGE.
34 */
35
72#include <config.h>
73
74#include <pthread.h>
75#include <assert.h>
76#include <sys/types.h>
77#include <sys/file.h>
78#include <sys/poll.h>
79#include <sys/uio.h>
80#include <sys/mman.h>
81#include <sys/socket.h>
82#include <sys/un.h>
83#include <sys/time.h>
84#include <sys/resource.h>
85#include <sys/stat.h>
86#include <netinet/in.h>
87#include <arpa/inet.h>
88#include <unistd.h>
89#include <fcntl.h>
90#include <stdlib.h>
91#include <stdio.h>
92#include <errno.h>
93#include <signal.h>
94#include <sched.h>
95#include <time.h>
96#include <semaphore.h>
97#include <string.h>
98
99#ifdef HAVE_LIBSYSTEMD
100#include <systemd/sd-daemon.h>
101#endif
102
103#include <qb/qbdefs.h>
104#include <qb/qblog.h>
105#include <qb/qbloop.h>
106#include <qb/qbutil.h>
107#include <qb/qbipcs.h>
108
109#include <corosync/swab.h>
110#include <corosync/corotypes.h>
111#include <corosync/corodefs.h>
113#include <corosync/logsys.h>
114#include <corosync/icmap.h>
115
116#include "quorum.h"
117#include "totemsrp.h"
118#include "logconfig.h"
119#include "totemconfig.h"
120#include "main.h"
121#include "sync.h"
122#include "timer.h"
123#include "util.h"
124#include "apidef.h"
125#include "service.h"
126#include "schedwrk.h"
127#include "ipcs_stats.h"
128#include "stats.h"
129
130#ifdef HAVE_SMALL_MEMORY_FOOTPRINT
131#define IPC_LOGSYS_SIZE 1024*64
132#else
133#define IPC_LOGSYS_SIZE 8192*128
134#endif
135
136/*
137 * LibQB adds default "*" syslog filter so we have to set syslog_priority as low
138 * as possible so filters applied later in _logsys_config_apply_per_file takes
139 * effect.
140 */
143 LOG_DAEMON,
144 LOG_EMERG);
145
147
148#define SERVER_BACKLOG 5
149
150static int sched_priority = 0;
151
152static unsigned int service_count = 32;
153
155
156static struct corosync_api_v1 *api = NULL;
157
158static int sync_in_process = 1;
159
160static qb_loop_t *corosync_poll_handle;
161
162struct sched_param global_sched_param;
163
164static corosync_timer_handle_t corosync_stats_timer_handle;
165
166static char corosync_lock_file[PATH_MAX + 1] = LOCALSTATEDIR"/run/corosync.pid";
167
168static char corosync_config_file[PATH_MAX + 1] = COROSYSCONFDIR "/corosync.conf";
169
170static int lockfile_fd = -1;
171
177
178qb_loop_t *cs_poll_handle_get (void)
179{
180 return (corosync_poll_handle);
181}
182
183int cs_poll_dispatch_add (qb_loop_t * handle,
184 int fd,
185 int events,
186 void *data,
187
188 int (*dispatch_fn) (int fd,
189 int revents,
190 void *data))
191{
192 return qb_loop_poll_add(handle, QB_LOOP_MED, fd, events, data,
193 dispatch_fn);
194}
195
196int cs_poll_dispatch_delete(qb_loop_t * handle, int fd)
197{
198 return qb_loop_poll_del(handle, fd);
199}
200
202{
203 int i;
204
205 for (i = 0; i < SERVICES_COUNT_MAX; i++) {
206 if (corosync_service[i] && corosync_service[i]->exec_dump_fn) {
208 }
209 }
210}
211
213{
214
215 return (corosync_config_file);
216}
217
218static void corosync_blackbox_write_to_file (void)
219{
220 char fname[PATH_MAX];
221 char fdata_fname[PATH_MAX];
222 char time_str[PATH_MAX];
223 struct tm cur_time_tm;
224 time_t cur_time_t;
225 ssize_t res;
226
227 cur_time_t = time(NULL);
228 localtime_r(&cur_time_t, &cur_time_tm);
229
230 strftime(time_str, PATH_MAX, "%Y-%m-%dT%H:%M:%S", &cur_time_tm);
231 if (snprintf(fname, PATH_MAX, "%s/fdata-%s-%lld",
233 time_str,
234 (long long int)getpid()) >= PATH_MAX) {
235 log_printf(LOGSYS_LEVEL_ERROR, "Can't snprintf blackbox file name");
236 return ;
237 }
238
239 if ((res = qb_log_blackbox_write_to_file(fname)) < 0) {
240 LOGSYS_PERROR(-res, LOGSYS_LEVEL_ERROR, "Can't store blackbox file");
241 return ;
242 }
243 snprintf(fdata_fname, sizeof(fdata_fname), "%s/fdata", get_state_dir());
244 unlink(fdata_fname);
245 if (symlink(fname, fdata_fname) == -1) {
246 log_printf(LOGSYS_LEVEL_ERROR, "Can't create symlink to '%s' for corosync blackbox file '%s'",
247 fname, fdata_fname);
248 }
249}
250
251static void unlink_all_completed (void)
252{
253 api->timer_delete (corosync_stats_timer_handle);
254 qb_loop_stop (corosync_poll_handle);
255 icmap_fini();
256}
257
259{
260 corosync_service_unlink_all (api, unlink_all_completed);
261}
262
263static int32_t sig_diag_handler (int num, void *data)
264{
266 return 0;
267}
268
269static int32_t sig_exit_handler (int num, void *data)
270{
271 log_printf(LOGSYS_LEVEL_NOTICE, "Node was shut down by a signal");
272 corosync_service_unlink_all (api, unlink_all_completed);
273 return 0;
274}
275
276static void sigsegv_handler (int num)
277{
278 (void)signal (num, SIG_DFL);
279 corosync_blackbox_write_to_file ();
280 qb_log_fini();
281 raise (num);
282}
283
284#define LOCALHOST_IP inet_addr("127.0.0.1")
285
286static void *corosync_group_handle;
287
288static struct totempg_group corosync_group = {
289 .group = "a",
290 .group_len = 1
291};
292
293static void serialize_lock (void)
294{
295}
296
297static void serialize_unlock (void)
298{
299}
300
301static void corosync_sync_completed (void)
302{
304 "Completed service synchronization, ready to provide service.");
305 sync_in_process = 0;
306
307 cs_ipcs_sync_state_changed(sync_in_process);
309 /*
310 * Inform totem to start using new message queue again
311 */
313
314#ifdef HAVE_LIBSYSTEMD
315 sd_notify (0, "READY=1");
316#endif
317}
318
319static int corosync_sync_callbacks_retrieve (
320 int service_id,
321 struct sync_callbacks *callbacks)
322{
323 if (corosync_service[service_id] == NULL) {
324 return (-1);
325 }
326
327 if (callbacks == NULL) {
328 return (0);
329 }
330
331 callbacks->name = corosync_service[service_id]->name;
332
333 callbacks->sync_init = corosync_service[service_id]->sync_init;
334 callbacks->sync_process = corosync_service[service_id]->sync_process;
335 callbacks->sync_activate = corosync_service[service_id]->sync_activate;
336 callbacks->sync_abort = corosync_service[service_id]->sync_abort;
337 return (0);
338}
339
340static struct memb_ring_id corosync_ring_id;
341
342static void member_object_joined (unsigned int nodeid)
343{
344 char member_ip[ICMAP_KEYNAME_MAXLEN];
345 char member_join_count[ICMAP_KEYNAME_MAXLEN];
346 char member_status[ICMAP_KEYNAME_MAXLEN];
347
348 snprintf(member_ip, ICMAP_KEYNAME_MAXLEN,
349 "runtime.members.%u.ip", nodeid);
350 snprintf(member_join_count, ICMAP_KEYNAME_MAXLEN,
351 "runtime.members.%u.join_count", nodeid);
352 snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
353 "runtime.members.%u.status", nodeid);
354
355 if (icmap_get(member_ip, NULL, NULL, NULL) == CS_OK) {
356 icmap_inc(member_join_count);
357 icmap_set_string(member_status, "joined");
358 } else {
359 icmap_set_string(member_ip, (char*)api->totem_ifaces_print (nodeid));
360 icmap_set_uint32(member_join_count, 1);
361 icmap_set_string(member_status, "joined");
362 }
363
365 "Member joined: %s", api->totem_ifaces_print (nodeid));
366}
367
368static void member_object_left (unsigned int nodeid)
369{
370 char member_status[ICMAP_KEYNAME_MAXLEN];
371
372 snprintf(member_status, ICMAP_KEYNAME_MAXLEN,
373 "runtime.members.%u.status", nodeid);
374 icmap_set_string(member_status, "left");
375
377 "Member left: %s", api->totem_ifaces_print (nodeid));
378}
379
380static void confchg_fn (
381 enum totem_configuration_type configuration_type,
382 const unsigned int *member_list, size_t member_list_entries,
383 const unsigned int *left_list, size_t left_list_entries,
384 const unsigned int *joined_list, size_t joined_list_entries,
385 const struct memb_ring_id *ring_id)
386{
387 int i;
388 int abort_activate = 0;
389
390 if (sync_in_process == 1) {
391 abort_activate = 1;
392 }
393 sync_in_process = 1;
394 cs_ipcs_sync_state_changed(sync_in_process);
395 memcpy (&corosync_ring_id, ring_id, sizeof (struct memb_ring_id));
396
397 for (i = 0; i < left_list_entries; i++) {
398 member_object_left (left_list[i]);
399 }
400 for (i = 0; i < joined_list_entries; i++) {
401 member_object_joined (joined_list[i]);
402 }
403 /*
404 * Call configuration change for all services
405 */
406 for (i = 0; i < service_count; i++) {
407 if (corosync_service[i] && corosync_service[i]->confchg_fn) {
408 corosync_service[i]->confchg_fn (configuration_type,
409 member_list, member_list_entries,
410 left_list, left_list_entries,
411 joined_list, joined_list_entries, ring_id);
412 }
413 }
414
415 if (abort_activate) {
416 sync_abort ();
417 }
418 if (configuration_type == TOTEM_CONFIGURATION_TRANSITIONAL) {
419 sync_save_transitional (member_list, member_list_entries, ring_id);
420 }
421 if (configuration_type == TOTEM_CONFIGURATION_REGULAR) {
422 sync_start (member_list, member_list_entries, ring_id);
423 }
424}
425
426static void priv_drop (void)
427{
428 return; /* TODO: we are still not dropping privs */
429}
430
431static void corosync_tty_detach (void)
432{
433 int devnull;
434
435 /*
436 * Disconnect from TTY if this is not a debug run
437 */
438
439 switch (fork ()) {
440 case -1:
442 break;
443 case 0:
444 /*
445 * child which is disconnected, run this process
446 */
447 break;
448 default:
449 exit (0);
450 break;
451 }
452
453 /* Create new session */
454 (void)setsid();
455
456 /*
457 * Map stdin/out/err to /dev/null.
458 */
459 devnull = open("/dev/null", O_RDWR);
460 if (devnull == -1) {
462 }
463
464 if (dup2(devnull, 0) < 0 || dup2(devnull, 1) < 0
465 || dup2(devnull, 2) < 0) {
466 close(devnull);
468 }
469 close(devnull);
470}
471
472static void corosync_mlockall (void)
473{
474 int res;
475 struct rlimit rlimit;
476
477 rlimit.rlim_cur = RLIM_INFINITY;
478 rlimit.rlim_max = RLIM_INFINITY;
479
480#ifndef RLIMIT_MEMLOCK
481#define RLIMIT_MEMLOCK RLIMIT_VMEM
482#endif
483
484 res = setrlimit (RLIMIT_MEMLOCK, &rlimit);
485 if (res == -1) {
487 "Could not increase RLIMIT_MEMLOCK, not locking memory");
488 return;
489 }
490
491 res = mlockall (MCL_CURRENT | MCL_FUTURE);
492 if (res == -1) {
494 "Could not lock memory of service to avoid page faults");
495 };
496}
497
498
499static void corosync_totem_stats_updater (void *data)
500{
501 totempg_stats_t * stats;
502 uint32_t total_mtt_rx_token;
503 uint32_t total_backlog_calc;
504 uint32_t total_token_holdtime;
505 int t, prev;
506 int32_t token_count;
507 const char *cstr;
508
509 stats = api->totem_get_stats();
510
511
513
516 cstr = "";
517
519 cstr = "number of multicast sendmsg failures is above threshold";
520 }
521
523 cstr = "totem is continuously in gather state";
524 }
525
527 "Totem is unable to form a cluster because of an "
528 "operating system or network fault (reason: %s). The most common "
529 "cause of this message is that the local firewall is "
530 "configured improperly.", cstr);
532 } else {
534 }
535
536 total_mtt_rx_token = 0;
537 total_token_holdtime = 0;
538 total_backlog_calc = 0;
539 token_count = 0;
540 t = stats->srp->latest_token;
541 while (1) {
542 if (t == 0)
543 prev = TOTEM_TOKEN_STATS_MAX - 1;
544 else
545 prev = t - 1;
546 if (prev == stats->srp->earliest_token)
547 break;
548 /* if tx == 0, then dropped token (not ours) */
549 if (stats->srp->token[t].tx != 0 ||
550 (stats->srp->token[t].rx - stats->srp->token[prev].rx) > 0 ) {
551 total_mtt_rx_token += (stats->srp->token[t].rx - stats->srp->token[prev].rx);
552 total_token_holdtime += (stats->srp->token[t].tx - stats->srp->token[t].rx);
553 total_backlog_calc += stats->srp->token[t].backlog_calc;
554 token_count++;
555 }
556 t = prev;
557 }
558 if (token_count) {
559 stats->srp->mtt_rx_token = (total_mtt_rx_token / token_count);
560 stats->srp->avg_token_workload = (total_token_holdtime / token_count);
561 stats->srp->avg_backlog_calc = (total_backlog_calc / token_count);
562 }
563
564 stats->srp->time_since_token_last_received = qb_util_nano_current_get () / QB_TIME_NS_IN_MSEC -
565 stats->srp->token[stats->srp->latest_token].rx;
566
568
569 api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
570 corosync_totem_stats_updater,
571 &corosync_stats_timer_handle);
572}
573
574static void corosync_totem_stats_init (void)
575{
576 /* start stats timer */
577 api->timer_add_duration (1500 * MILLI_2_NANO_SECONDS, NULL,
578 corosync_totem_stats_updater,
579 &corosync_stats_timer_handle);
580}
581
582static void deliver_fn (
583 unsigned int nodeid,
584 const void *msg,
585 unsigned int msg_len,
586 int endian_conversion_required)
587{
588 const struct qb_ipc_request_header *header;
589 int32_t service;
590 int32_t fn_id;
591 uint32_t id;
592
593 header = msg;
594 if (endian_conversion_required) {
595 id = swab32 (header->id);
596 } else {
597 id = header->id;
598 }
599
600 /*
601 * Call the proper executive handler
602 */
603 service = id >> 16;
604 fn_id = id & 0xffff;
605
606 if (!corosync_service[service]) {
607 return;
608 }
609 if (fn_id >= corosync_service[service]->exec_engine_count) {
610 log_printf(LOGSYS_LEVEL_WARNING, "discarded unknown message %d for service %d (max id %d)",
611 fn_id, service, corosync_service[service]->exec_engine_count);
612 return;
613 }
614
615 icmap_fast_inc(service_stats_rx[service][fn_id]);
616
617 if (endian_conversion_required) {
618 assert(corosync_service[service]->exec_engine[fn_id].exec_endian_convert_fn != NULL);
620 ((void *)msg);
621 }
622
624 (msg, nodeid);
625}
626
628 const struct iovec *iovec,
629 unsigned int iov_len,
630 unsigned int guarantee)
631{
632 const struct qb_ipc_request_header *req = iovec->iov_base;
633 int32_t service;
634 int32_t fn_id;
635
636 service = req->id >> 16;
637 fn_id = req->id & 0xffff;
638
639 if (corosync_service[service]) {
640 icmap_fast_inc(service_stats_tx[service][fn_id]);
641 }
642
643 return (totempg_groups_mcast_joined (corosync_group_handle, iovec, iov_len, guarantee));
644}
645
646static void corosync_ring_id_create_or_load (
648 unsigned int nodeid)
649{
650 int fd;
651 int res = 0;
652 char filename[PATH_MAX];
653
654 snprintf (filename, sizeof(filename), "%s/ringid_%u",
656 fd = open (filename, O_RDONLY);
657 /*
658 * If file can be opened and read, read the ring id
659 */
660 if (fd != -1) {
661 res = read (fd, &memb_ring_id->seq, sizeof (uint64_t));
662 close (fd);
663 }
664 /*
665 * If file could not be opened or read, create a new ring id
666 */
667 if ((fd == -1) || (res != sizeof (uint64_t))) {
668 memb_ring_id->seq = 0;
669 fd = creat (filename, 0600);
670 if (fd != -1) {
671 res = write (fd, &memb_ring_id->seq, sizeof (uint64_t));
672 close (fd);
673 if (res == -1) {
675 "Couldn't write ringid file '%s'", filename);
676
678 }
679 } else {
681 "Couldn't create ringid file '%s'", filename);
682
684 }
685 }
686
688}
689
690static void corosync_ring_id_store (
691 const struct memb_ring_id *memb_ring_id,
692 unsigned int nodeid)
693{
694 char filename[PATH_MAX];
695 int fd;
696 int res;
697
698 snprintf (filename, sizeof(filename), "%s/ringid_%u",
700
701 fd = creat (filename, 0600);
702 if (fd == -1) {
704 "Couldn't store new ring id " CS_PRI_RING_ID_SEQ " to stable storage",
706
708 }
710 "Storing new sequence id for ring " CS_PRI_RING_ID_SEQ, memb_ring_id->seq);
711 res = write (fd, &memb_ring_id->seq, sizeof(memb_ring_id->seq));
712 close (fd);
713 if (res != sizeof(memb_ring_id->seq)) {
715 "Couldn't store new ring id " CS_PRI_RING_ID_SEQ " to stable storage",
717
719 }
720}
721
722static qb_loop_timer_handle recheck_the_q_level_timer;
724{
725 totempg_check_q_level(corosync_group_handle);
727 qb_loop_timer_add(cs_poll_handle_get(), QB_LOOP_MED, 1*QB_TIME_NS_IN_MSEC,
728 NULL, corosync_recheck_the_q_level, &recheck_the_q_level_timer);
729 }
730}
731
735
736
738 unsigned int service,
739 unsigned int id,
740 const void *msg,
741 void *sending_allowed_private_data)
742{
744 (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
745 struct iovec reserve_iovec;
746 struct qb_ipc_request_header *header = (struct qb_ipc_request_header *)msg;
747 int sending_allowed;
748
749 reserve_iovec.iov_base = (char *)header;
750 reserve_iovec.iov_len = header->size;
751
753 corosync_group_handle,
754 &reserve_iovec, 1);
755 if (pd->reserved_msgs == -1) {
756 return -EINVAL;
757 }
758
759 /* Message ID out of range */
760 if (id >= corosync_service[service]->lib_engine_count) {
761 return -EINVAL;
762 }
763
764 sending_allowed = QB_FALSE;
765 if (corosync_quorum_is_quorate() == 1 ||
766 corosync_service[service]->allow_inquorate == CS_LIB_ALLOW_INQUORATE) {
767 // we are quorate
768 // now check flow control
769 if (corosync_service[service]->lib_engine[id].flow_control == CS_LIB_FLOW_CONTROL_NOT_REQUIRED) {
770 sending_allowed = QB_TRUE;
771 } else if (pd->reserved_msgs && sync_in_process == 0) {
772 sending_allowed = QB_TRUE;
773 } else if (pd->reserved_msgs == 0) {
774 return -ENOBUFS;
775 } else /* (sync_in_process) */ {
776 return -EINPROGRESS;
777 }
778 } else {
779 return -EHOSTUNREACH;
780 }
781
782 return (sending_allowed);
783}
784
785void corosync_sending_allowed_release (void *sending_allowed_private_data)
786{
788 (struct sending_allowed_private_data_struct *)sending_allowed_private_data;
789
790 if (pd->reserved_msgs == -1) {
791 return;
792 }
794}
795
797{
798 int ret = 0;
799
800 assert (source != NULL);
801 if (source->nodeid == totempg_my_nodeid_get ()) {
802 ret = 1;
803 }
804 return ret;
805}
806
808 mar_message_source_t *source,
809 void *conn)
810{
811 assert ((source != NULL) && (conn != NULL));
812 memset (source, 0, sizeof (mar_message_source_t));
813 source->nodeid = totempg_my_nodeid_get ();
814 source->conn = conn;
815}
816
819 qb_loop_timer_handle handle;
820 unsigned long long tv_prev;
821 unsigned long long max_tv_diff;
822};
823
824static void timer_function_scheduler_timeout (void *data)
825{
826 struct scheduler_pause_timeout_data *timeout_data = (struct scheduler_pause_timeout_data *)data;
827 unsigned long long tv_current;
828 unsigned long long tv_diff;
829 uint64_t schedmiss_event_tstamp;
830
831 tv_current = qb_util_nano_current_get ();
832
833 if (timeout_data->tv_prev == 0) {
834 /*
835 * Initial call -> just pretent everything is ok
836 */
837 timeout_data->tv_prev = tv_current;
838 timeout_data->max_tv_diff = 0;
839 }
840
841 tv_diff = tv_current - timeout_data->tv_prev;
842 timeout_data->tv_prev = tv_current;
843
844 if (tv_diff > timeout_data->max_tv_diff) {
845 schedmiss_event_tstamp = qb_util_nano_from_epoch_get() / QB_TIME_NS_IN_MSEC;
846
847 log_printf (LOGSYS_LEVEL_WARNING, "Corosync main process was not scheduled (@%" PRIu64 ") for %0.4f ms "
848 "(threshold is %0.4f ms). Consider token timeout increase.",
849 schedmiss_event_tstamp,
850 (float)tv_diff / QB_TIME_NS_IN_MSEC, (float)timeout_data->max_tv_diff / QB_TIME_NS_IN_MSEC);
851
852 stats_add_schedmiss_event(schedmiss_event_tstamp, (float)tv_diff / QB_TIME_NS_IN_MSEC);
853 }
854
855 /*
856 * Set next threshold, because token_timeout can change
857 */
858 timeout_data->max_tv_diff = timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC * 0.8;
859 qb_loop_timer_add (corosync_poll_handle,
860 QB_LOOP_MED,
861 timeout_data->totem_config->token_timeout * QB_TIME_NS_IN_MSEC / 3,
862 timeout_data,
863 timer_function_scheduler_timeout,
864 &timeout_data->handle);
865}
866
867
868/*
869 * Set main pid RR scheduler.
870 * silent: don't log sched_get_priority_max and sched_setscheduler errors
871 * Returns: 0 - success, -1 failure, -2 platform doesn't support SCHED_RR
872 */
873static int corosync_set_rr_scheduler (int silent)
874{
875 int ret_val = 0;
876
877#if defined(HAVE_PTHREAD_SETSCHEDPARAM) && defined(HAVE_SCHED_GET_PRIORITY_MAX) && defined(HAVE_SCHED_SETSCHEDULER)
878 int res;
879
880 sched_priority = sched_get_priority_max (SCHED_RR);
881 if (sched_priority != -1) {
882 global_sched_param.sched_priority = sched_priority;
883 res = sched_setscheduler (0, SCHED_RR, &global_sched_param);
884 if (res == -1) {
885 if (!silent) {
887 "Could not set SCHED_RR at priority %d",
888 global_sched_param.sched_priority);
889 }
890
891 global_sched_param.sched_priority = 0;
892#ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
893 qb_log_thread_priority_set (SCHED_OTHER, 0);
894#endif
895 ret_val = -1;
896 } else {
897
898 /*
899 * Turn on SCHED_RR in logsys system
900 */
901#ifdef HAVE_QB_LOG_THREAD_PRIORITY_SET
902 res = qb_log_thread_priority_set (SCHED_RR, sched_priority);
903#else
904 res = -1;
905#endif
906 if (res == -1) {
908 "Could not set logsys thread priority."
909 " Can't continue because of priority inversions.");
911 }
912 }
913 } else {
914 if (!silent) {
916 "Could not get maximum scheduler priority");
917 }
918 sched_priority = 0;
919 ret_val = -1;
920 }
921#else
923 "The Platform is missing process priority setting features. Leaving at default.");
924 ret_val = -2;
925#endif
926
927 return (ret_val);
928}
929
930
931/* The basename man page contains scary warnings about
932 thread-safety and portability, hence this */
933static const char *corosync_basename(const char *file_name)
934{
935 char *base;
936 base = strrchr (file_name, '/');
937 if (base) {
938 return base + 1;
939 }
940
941 return file_name;
942}
943
944static void
945_logsys_log_printf(int level, int subsys,
946 const char *function_name,
947 const char *file_name,
948 int file_line,
949 const char *format,
950 ...) __attribute__((format(printf, 6, 7)));
951
952static void
953_logsys_log_printf(int level, int subsys,
954 const char *function_name,
955 const char *file_name,
956 int file_line,
957 const char *format, ...)
958{
959 va_list ap;
960
961 va_start(ap, format);
962 qb_log_from_external_source_va(function_name, corosync_basename(file_name),
963 format, level, file_line,
964 subsys, ap);
965 va_end(ap);
966}
967
968static void fplay_key_change_notify_fn (
969 int32_t event,
970 const char *key_name,
971 struct icmap_notify_value new_val,
972 struct icmap_notify_value old_val,
973 void *user_data)
974{
975 if (strcmp(key_name, "runtime.blackbox.dump_flight_data") == 0) {
976 fprintf(stderr,"Writetofile\n");
977 corosync_blackbox_write_to_file ();
978 }
979 if (strcmp(key_name, "runtime.blackbox.dump_state") == 0) {
980 fprintf(stderr,"statefump\n");
982 }
983}
984
985static void corosync_fplay_control_init (void)
986{
987 icmap_track_t track = NULL;
988
989 icmap_set_string("runtime.blackbox.dump_flight_data", "no");
990 icmap_set_string("runtime.blackbox.dump_state", "no");
991
992 icmap_track_add("runtime.blackbox.dump_flight_data",
994 fplay_key_change_notify_fn,
995 NULL, &track);
996 icmap_track_add("runtime.blackbox.dump_state",
998 fplay_key_change_notify_fn,
999 NULL, &track);
1000}
1001
1002static void force_gather_notify_fn(
1003 int32_t event,
1004 const char *key_name,
1005 struct icmap_notify_value new_val,
1006 struct icmap_notify_value old_val,
1007 void *user_data)
1008{
1009 char *key_val;
1010
1011 if (icmap_get_string(key_name, &key_val) == CS_OK && strcmp(key_val, "no") == 0)
1012 goto out;
1013
1014 icmap_set_string("runtime.force_gather", "no");
1015
1016 if (strcmp(key_name, "runtime.force_gather") == 0) {
1017 log_printf(LOGSYS_LEVEL_ERROR, "Forcing into GATHER state\n");
1019 }
1020
1021out:
1022 free(key_val);
1023}
1024
1025static void corosync_force_gather_init (void)
1026{
1027 icmap_track_t track = NULL;
1028
1029 icmap_set_string("runtime.force_gather", "no");
1030
1031 icmap_track_add("runtime.force_gather",
1033 force_gather_notify_fn,
1034 NULL, &track);
1035}
1036
1037/*
1038 * Set RO flag for keys, which ether doesn't make sense to change by user (statistic)
1039 * or which when changed are not reflected by runtime (totem.crypto_cipher, ...).
1040 *
1041 * Also some RO keys cannot be determined in this stage, so they are set later in
1042 * other functions (like nodelist.local_node_pos, ...)
1043 */
1044static void set_icmap_ro_keys_flag (void)
1045{
1046 /*
1047 * Set RO flag for all keys of internal configuration and runtime statistics
1048 */
1049 icmap_set_ro_access("internal_configuration.", CS_TRUE, CS_TRUE);
1050 icmap_set_ro_access("runtime.services.", CS_TRUE, CS_TRUE);
1051 icmap_set_ro_access("runtime.config.", CS_TRUE, CS_TRUE);
1052 icmap_set_ro_access("runtime.totem.", CS_TRUE, CS_TRUE);
1053 icmap_set_ro_access("uidgid.config.", CS_TRUE, CS_TRUE);
1054 icmap_set_ro_access("system.", CS_TRUE, CS_TRUE);
1055 icmap_set_ro_access("nodelist.", CS_TRUE, CS_TRUE);
1056
1057 /*
1058 * Set RO flag for constrete keys of configuration which can't be changed
1059 * during runtime
1060 */
1061 icmap_set_ro_access("totem.crypto_cipher", CS_FALSE, CS_TRUE);
1062 icmap_set_ro_access("totem.crypto_hash", CS_FALSE, CS_TRUE);
1063 icmap_set_ro_access("totem.crypto_model", CS_FALSE, CS_TRUE);
1064 icmap_set_ro_access("totem.keyfile", CS_FALSE, CS_TRUE);
1065 icmap_set_ro_access("totem.key", CS_FALSE, CS_TRUE);
1066 icmap_set_ro_access("totem.secauth", CS_FALSE, CS_TRUE);
1067 icmap_set_ro_access("totem.ip_version", CS_FALSE, CS_TRUE);
1068 icmap_set_ro_access("totem.rrp_mode", CS_FALSE, CS_TRUE);
1069 icmap_set_ro_access("totem.transport", CS_FALSE, CS_TRUE);
1070 icmap_set_ro_access("totem.cluster_name", CS_FALSE, CS_TRUE);
1071 icmap_set_ro_access("totem.netmtu", CS_FALSE, CS_TRUE);
1072 icmap_set_ro_access("totem.threads", CS_FALSE, CS_TRUE);
1073 icmap_set_ro_access("totem.version", CS_FALSE, CS_TRUE);
1074 icmap_set_ro_access("totem.nodeid", CS_FALSE, CS_TRUE);
1075 icmap_set_ro_access("totem.clear_node_high_bit", CS_FALSE, CS_TRUE);
1076 icmap_set_ro_access("config.reload_in_progress", CS_FALSE, CS_TRUE);
1077 icmap_set_ro_access("config.totemconfig_reload_in_progress", CS_FALSE, CS_TRUE);
1078}
1079
1080static void main_service_ready (void)
1081{
1082 int res;
1083
1084 /*
1085 * This must occur after totempg is initialized because "this_ip" must be set
1086 */
1088 if (res == -1) {
1089 log_printf (LOGSYS_LEVEL_ERROR, "Could not initialize default services");
1091 }
1092 cs_ipcs_init();
1093 corosync_totem_stats_init ();
1094 corosync_fplay_control_init ();
1095 corosync_force_gather_init ();
1096
1097 sync_init (
1098 corosync_sync_callbacks_retrieve,
1099 corosync_sync_completed);
1100}
1101
1102static enum e_corosync_done corosync_flock (const char *lockfile, pid_t pid)
1103{
1104 struct flock lock;
1105 enum e_corosync_done err;
1106 char pid_s[17];
1107 int fd_flag;
1108
1109 err = COROSYNC_DONE_EXIT;
1110
1111 lockfile_fd = open (lockfile, O_WRONLY | O_CREAT, 0640);
1112 if (lockfile_fd == -1) {
1113 log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't create lock file.");
1115 }
1116
1117retry_fcntl:
1118 lock.l_type = F_WRLCK;
1119 lock.l_start = 0;
1120 lock.l_whence = SEEK_SET;
1121 lock.l_len = 0;
1122 if (fcntl (lockfile_fd, F_SETLK, &lock) == -1) {
1123 switch (errno) {
1124 case EINTR:
1125 goto retry_fcntl;
1126 break;
1127 case EAGAIN:
1128 case EACCES:
1129 log_printf (LOGSYS_LEVEL_ERROR, "Another Corosync instance is already running.");
1131 goto error_close;
1132 break;
1133 default:
1134 log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't acquire lock. Error was %s",
1135 strerror(errno));
1137 goto error_close;
1138 break;
1139 }
1140 }
1141
1142 if (ftruncate (lockfile_fd, 0) == -1) {
1143 log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't truncate lock file. Error was %s",
1144 strerror (errno));
1146 goto error_close_unlink;
1147 }
1148
1149 memset (pid_s, 0, sizeof (pid_s));
1150 snprintf (pid_s, sizeof (pid_s) - 1, "%u\n", pid);
1151
1152retry_write:
1153 if (write (lockfile_fd, pid_s, strlen (pid_s)) != strlen (pid_s)) {
1154 if (errno == EINTR) {
1155 goto retry_write;
1156 } else {
1157 log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't write pid to lock file. "
1158 "Error was %s", strerror (errno));
1160 goto error_close_unlink;
1161 }
1162 }
1163
1164 if ((fd_flag = fcntl (lockfile_fd, F_GETFD, 0)) == -1) {
1165 log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't get close-on-exec flag from lock file. "
1166 "Error was %s", strerror (errno));
1168 goto error_close_unlink;
1169 }
1170 fd_flag |= FD_CLOEXEC;
1171 if (fcntl (lockfile_fd, F_SETFD, fd_flag) == -1) {
1172 log_printf (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't set close-on-exec flag to lock file. "
1173 "Error was %s", strerror (errno));
1175 goto error_close_unlink;
1176 }
1177
1178 return (err);
1179
1180error_close_unlink:
1181 unlink (lockfile);
1182error_close:
1183 close (lockfile_fd);
1184
1185 return (err);
1186}
1187
1188static int corosync_move_to_root_cgroup(void) {
1189 FILE *f;
1190 int res = -1;
1191 const char *cgroup_task_fname = NULL;
1192
1193 /*
1194 * /sys/fs/cgroup is hardcoded, because most of Linux distributions are now
1195 * using systemd and systemd uses hardcoded path of cgroup mount point.
1196 *
1197 * This feature is expected to be removed as soon as systemd gets support
1198 * for managing RT configuration.
1199 */
1200 f = fopen("/sys/fs/cgroup/cpu/cpu.rt_runtime_us", "rt");
1201 if (f == NULL) {
1202 /*
1203 * Try cgroup v2
1204 */
1205 f = fopen("/sys/fs/cgroup/cgroup.procs", "rt");
1206 if (f == NULL) {
1207 log_printf(LOG_DEBUG, "cpu.rt_runtime_us or cgroup.procs doesn't exist -> "
1208 "system without cgroup or with disabled CONFIG_RT_GROUP_SCHED");
1209
1210 res = 0;
1211 goto exit_res;
1212 } else {
1213 log_printf(LOGSYS_LEVEL_DEBUG, "Moving main pid to cgroup v2 root cgroup");
1214
1215 cgroup_task_fname = "/sys/fs/cgroup/cgroup.procs";
1216 }
1217 } else {
1218 log_printf(LOGSYS_LEVEL_DEBUG, "Moving main pid to cgroup v1 root cgroup");
1219
1220 cgroup_task_fname = "/sys/fs/cgroup/cpu/tasks";
1221 }
1222 (void)fclose(f);
1223
1224 f = fopen(cgroup_task_fname, "w");
1225 if (f == NULL) {
1226 log_printf(LOGSYS_LEVEL_WARNING, "Can't open cgroups tasks file for writing");
1227
1228 goto exit_res;
1229 }
1230
1231 if (fprintf(f, "%jd\n", (intmax_t)getpid()) <= 0) {
1232 log_printf(LOGSYS_LEVEL_WARNING, "Can't write corosync pid into cgroups tasks file");
1233
1234 goto close_and_exit_res;
1235 }
1236
1237close_and_exit_res:
1238 if (fclose(f) != 0) {
1239 log_printf(LOGSYS_LEVEL_WARNING, "Can't close cgroups tasks file");
1240
1241 goto exit_res;
1242 }
1243
1244exit_res:
1245 return (res);
1246}
1247
1248static void show_version_info_crypto(void)
1249{
1250 const char *error_string;
1251 const char *list_str;
1252
1253 if (util_is_valid_knet_crypto_model(NULL, &list_str, 1, "", &error_string) != -1) {
1254 printf("Available crypto models: %s\n", list_str);
1255 } else {
1256 perror(error_string);
1257 }
1258}
1259
1260static void show_version_info_compress(void)
1261{
1262 const char *error_string;
1263 const char *list_str;
1264
1265 if (util_is_valid_knet_compress_model(NULL, &list_str, 1, "", &error_string) != -1) {
1266 printf("Available compression models: %s\n", list_str);
1267 } else {
1268 perror(error_string);
1269 }
1270}
1271
1272static void show_version_info(void)
1273{
1274
1275 printf ("Corosync Cluster Engine, version '%s'\n", VERSION);
1276 printf ("Copyright (c) 2006-2021 Red Hat, Inc.\n");
1277
1278 printf ("\nBuilt-in features:" PACKAGE_FEATURES "\n");
1279
1280 show_version_info_crypto();
1281 show_version_info_compress();
1282}
1283
1284int main (int argc, char **argv, char **envp)
1285{
1286 const char *error_string;
1288 int res, ch;
1289 int background, sched_rr, prio, testonly;
1290 enum move_to_root_cgroup_mode move_to_root_cgroup;
1291 enum e_corosync_done flock_err;
1292 uint64_t totem_config_warnings;
1294 long int tmpli;
1295 char *ep;
1296 char *tmp_str;
1297 int log_subsys_id_totem;
1298 int silent;
1299
1300 /* default configuration
1301 */
1302 background = 1;
1303 testonly = 0;
1304
1305 while ((ch = getopt (argc, argv, "c:l:ftv")) != EOF) {
1306
1307 switch (ch) {
1308 case 'c':
1309 res = snprintf(corosync_config_file, sizeof(corosync_config_file), "%s", optarg);
1310 if (res >= sizeof(corosync_config_file)) {
1311 fprintf (stderr, "Config file path too long.\n");
1312 syslog (LOGSYS_LEVEL_ERROR, "Config file path too long.");
1313
1315 return EXIT_FAILURE;
1316 }
1317 break;
1318 case 'l':
1319 res = snprintf(corosync_lock_file, sizeof(corosync_lock_file), "%s", optarg);
1320 if (res >= sizeof(corosync_lock_file)) {
1321 fprintf (stderr, "PID lock file path too long.\n");
1322 syslog (LOGSYS_LEVEL_ERROR, "PID lock file path too long.");
1323
1325 return EXIT_FAILURE;
1326 }
1327 break;
1328 case 'f':
1329 background = 0;
1330 break;
1331 case 't':
1332 testonly = 1;
1333 break;
1334 case 'v':
1335 show_version_info();
1337 return EXIT_SUCCESS;
1338
1339 break;
1340 default:
1341 fprintf(stderr, \
1342 "usage:\n"\
1343 " -c : Corosync config file path.\n"\
1344 " -l : Corosync pid lock file path.\n"\
1345 " -f : Start application in foreground.\n"\
1346 " -t : Test configuration and exit.\n"\
1347 " -v : Display version, git revision and some useful information about Corosync and exit.\n");
1349 return EXIT_FAILURE;
1350 }
1351 }
1352
1353
1354 /*
1355 * Other signals are registered later via qb_loop_signal_add
1356 */
1357 (void)signal (SIGSEGV, sigsegv_handler);
1358 (void)signal (SIGABRT, sigsegv_handler);
1359#if MSG_NOSIGNAL != 0
1360 (void)signal (SIGPIPE, SIG_IGN);
1361#endif
1362
1363 if (icmap_init() != CS_OK) {
1364 fprintf (stderr, "Corosync Executive couldn't initialize configuration component.\n");
1365 syslog (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize configuration component.");
1367 }
1368 set_icmap_ro_keys_flag();
1369
1370 /*
1371 * Initialize the corosync_api_v1 definition
1372 */
1373 api = apidef_get ();
1374
1375 res = coroparse_configparse(icmap_get_global_map(), &error_string);
1376 if (res == -1) {
1377 /*
1378 * Logsys can't log properly at this early stage, and we need to get this message out
1379 *
1380 */
1381 fprintf (stderr, "%s\n", error_string);
1382 syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1384 }
1385
1386 if (stats_map_init(api) != CS_OK) {
1387 fprintf (stderr, "Corosync Executive couldn't initialize statistics component.\n");
1388 syslog (LOGSYS_LEVEL_ERROR, "Corosync Executive couldn't initialize statistics component.");
1390 }
1391
1392 res = corosync_log_config_read (&error_string);
1393 if (res == -1) {
1394 /*
1395 * if we are here, we _must_ flush the logsys queue
1396 * and try to inform that we couldn't read the config.
1397 * this is a desperate attempt before certain death
1398 * and there is no guarantee that we can print to stderr
1399 * nor that logsys is sending the messages where we expect.
1400 */
1401 log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1402 fprintf(stderr, "%s", error_string);
1403 syslog (LOGSYS_LEVEL_ERROR, "%s", error_string);
1405 }
1406
1407 if (!testonly) {
1408 log_printf (LOGSYS_LEVEL_NOTICE, "Corosync Cluster Engine %s starting up", VERSION);
1409 log_printf (LOGSYS_LEVEL_INFO, "Corosync built-in features:" PACKAGE_FEATURES "");
1410 }
1411
1412 /*
1413 * Create totem logsys subsys before totem_config_read so log functions can be used
1414 */
1415 log_subsys_id_totem = _logsys_subsys_create("TOTEM", "totem,"
1416 "totemip.c,totemconfig.c,totemcrypto.c,totemsrp.c,"
1417 "totempg.c,totemudp.c,totemudpu.c,totemnet.c,totemknet.c");
1418
1419 res = chdir(get_state_dir());
1420 if (res == -1) {
1421 log_printf (LOGSYS_LEVEL_ERROR, "Cannot chdir to state directory %s. %s", get_state_dir(), strerror(errno));
1423 }
1424
1425 res = totem_config_read (&totem_config, &error_string, &totem_config_warnings);
1426 if (res == -1) {
1427 log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1429 }
1430
1431 if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_IGNORED) {
1432 log_printf (LOGSYS_LEVEL_WARNING, "member section is used together with nodelist. Members ignored.");
1433 }
1434
1435 if (totem_config_warnings & TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED) {
1436 log_printf (LOGSYS_LEVEL_WARNING, "member section is deprecated.");
1437 }
1438
1439 if (totem_config_warnings & TOTEM_CONFIG_WARNING_TOTEM_NODEID_SET) {
1440 log_printf (LOGSYS_LEVEL_WARNING, "nodeid in totem section is deprecated and ignored. "
1441 "Nodelist (or autogenerated) nodeid is going to be used.");
1442 }
1443
1444 if (totem_config_warnings & TOTEM_CONFIG_BINDNETADDR_NODELIST_SET) {
1445 log_printf (LOGSYS_LEVEL_WARNING, "interface section bindnetaddr is used together with nodelist. "
1446 "Nodelist one is going to be used.");
1447 }
1448
1449 if (totem_config_warnings != 0) {
1450 log_printf (LOGSYS_LEVEL_WARNING, "Please migrate config file to nodelist.");
1451 }
1452
1453 res = totem_config_validate (&totem_config, &error_string);
1454 if (res == -1) {
1455 log_printf (LOGSYS_LEVEL_ERROR, "%s", error_string);
1457 }
1458
1459 if (testonly) {
1461 }
1462
1463
1464 move_to_root_cgroup = MOVE_TO_ROOT_CGROUP_MODE_AUTO;
1465 if (icmap_get_string("system.move_to_root_cgroup", &tmp_str) == CS_OK) {
1466 /*
1467 * Validity of move_to_root_cgroup values checked in coroparse.c
1468 */
1469 if (strcmp(tmp_str, "yes") == 0) {
1470 move_to_root_cgroup = MOVE_TO_ROOT_CGROUP_MODE_ON;
1471 } else if (strcmp(tmp_str, "no") == 0) {
1472 move_to_root_cgroup = MOVE_TO_ROOT_CGROUP_MODE_OFF;
1473 }
1474 free(tmp_str);
1475 }
1476
1477
1478 sched_rr = 1;
1479 if (icmap_get_string("system.sched_rr", &tmp_str) == CS_OK) {
1480 if (strcmp(tmp_str, "yes") != 0) {
1481 sched_rr = 0;
1482 }
1483 free(tmp_str);
1484 }
1485
1486 prio = 0;
1487 if (icmap_get_string("system.priority", &tmp_str) == CS_OK) {
1488 if (strcmp(tmp_str, "max") == 0) {
1489 prio = INT_MIN;
1490 } else if (strcmp(tmp_str, "min") == 0) {
1491 prio = INT_MAX;
1492 } else {
1493 errno = 0;
1494
1495 tmpli = strtol(tmp_str, &ep, 10);
1496 if (errno != 0 || *ep != '\0' || tmpli > INT_MAX || tmpli < INT_MIN) {
1497 log_printf (LOGSYS_LEVEL_ERROR, "Priority value %s is invalid", tmp_str);
1499 }
1500
1501 prio = tmpli;
1502 }
1503
1504 free(tmp_str);
1505 }
1506
1507 if (move_to_root_cgroup == MOVE_TO_ROOT_CGROUP_MODE_ON) {
1508 /*
1509 * Try to move corosync into root cpu cgroup. Failure is not fatal and
1510 * error is deliberately ignored.
1511 */
1512 (void)corosync_move_to_root_cgroup();
1513 }
1514
1515 /*
1516 * Set round robin realtime scheduling with priority 99
1517 */
1518 if (sched_rr) {
1519 silent = (move_to_root_cgroup == MOVE_TO_ROOT_CGROUP_MODE_AUTO);
1520 res = corosync_set_rr_scheduler (silent);
1521
1522 if (res == -1 && move_to_root_cgroup == MOVE_TO_ROOT_CGROUP_MODE_AUTO) {
1523 /*
1524 * Try to move process to root cgroup and try set priority again
1525 */
1526 (void)corosync_move_to_root_cgroup();
1527
1528 res = corosync_set_rr_scheduler (0);
1529 }
1530
1531 if (res != 0) {
1532 prio = INT_MIN;
1533 } else {
1534 prio = 0;
1535 }
1536 }
1537
1538 if (prio != 0) {
1539 if (setpriority(PRIO_PGRP, 0, prio) != 0) {
1541 "Could not set priority %d", prio);
1542 }
1543 }
1544
1545 totem_config.totem_memb_ring_id_create_or_load = corosync_ring_id_create_or_load;
1546 totem_config.totem_memb_ring_id_store = corosync_ring_id_store;
1547
1550
1558
1560
1561 /*
1562 * Now we are fully initialized.
1563 */
1564 if (background) {
1566
1567 corosync_tty_detach ();
1568
1570
1571 log_printf (LOGSYS_LEVEL_DEBUG, "Corosync TTY detached");
1572 }
1573
1574 /*
1575 * Lock all memory to avoid page faults which may interrupt
1576 * application healthchecking
1577 */
1578 corosync_mlockall ();
1579
1580 corosync_poll_handle = qb_loop_create ();
1581
1584 timer_function_scheduler_timeout (&scheduler_pause_timeout_data);
1585
1586 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_LOW,
1587 SIGUSR2, NULL, sig_diag_handler, NULL);
1588 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1589 SIGINT, NULL, sig_exit_handler, NULL);
1590 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1591 SIGQUIT, NULL, sig_exit_handler, NULL);
1592 qb_loop_signal_add(corosync_poll_handle, QB_LOOP_HIGH,
1593 SIGTERM, NULL, sig_exit_handler, NULL);
1594
1595 if (logsys_thread_start() != 0) {
1596 log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize log thread");
1598 }
1599
1600 if ((flock_err = corosync_flock (corosync_lock_file, getpid ())) != COROSYNC_DONE_EXIT) {
1601 corosync_exit_error (flock_err);
1602 }
1603
1604 /*
1605 * if totempg_initialize doesn't have root priveleges, it cannot
1606 * bind to a specific interface. This only matters if
1607 * there is more then one interface in a system, so
1608 * in this case, only a warning is printed
1609 */
1610 /*
1611 * Join multicast group and setup delivery
1612 * and configuration change functions
1613 */
1614 if (totempg_initialize (
1615 corosync_poll_handle,
1616 &totem_config) != 0) {
1617
1618 log_printf (LOGSYS_LEVEL_ERROR, "Can't initialize TOTEM layer");
1620 }
1621
1623 main_service_ready);
1624
1626 &corosync_group_handle,
1627 deliver_fn,
1628 confchg_fn);
1629
1631 corosync_group_handle,
1632 &corosync_group,
1633 1);
1634
1635 /*
1636 * Drop root privleges to user 'corosync'
1637 * TODO: Don't really need full root capabilities;
1638 * needed capabilities are:
1639 * CAP_NET_RAW (bindtodevice)
1640 * CAP_SYS_NICE (setscheduler)
1641 * CAP_IPC_LOCK (mlockall)
1642 */
1643 priv_drop ();
1644
1646 serialize_lock,
1647 serialize_unlock);
1648
1649 /*
1650 * Start main processing loop
1651 */
1652 qb_loop_run (corosync_poll_handle);
1653
1654 /*
1655 * Exit was requested
1656 */
1658
1659 /*
1660 * free the loop resources
1661 */
1662 qb_loop_destroy (corosync_poll_handle);
1663
1664 /*
1665 * free up the icmap
1666 */
1667
1668 /*
1669 * Remove pid lock file
1670 */
1671 close (lockfile_fd);
1672 unlink (corosync_lock_file);
1673
1675
1676 return EXIT_SUCCESS;
1677}
struct corosync_api_v1 * apidef_get(void)
Definition apidef.c:146
#define VERSION
Definition config.h:520
#define PACKAGE_FEATURES
Definition config.h:388
#define LOCALSTATEDIR
Definition config.h:373
#define COROSYSCONFDIR
Definition config.h:8
totem_configuration_type
The totem_configuration_type enum.
Definition coroapi.h:132
@ TOTEM_CONFIGURATION_REGULAR
Definition coroapi.h:133
@ TOTEM_CONFIGURATION_TRANSITIONAL
Definition coroapi.h:134
@ CS_LIB_ALLOW_INQUORATE
Definition coroapi.h:164
qb_loop_timer_handle corosync_timer_handle_t
corosync_timer_handle_t
Definition coroapi.h:74
unsigned int nodeid
Definition coroapi.h:0
@ CS_LIB_FLOW_CONTROL_NOT_REQUIRED
Definition coroapi.h:153
#define SERVICES_COUNT_MAX
Definition coroapi.h:462
#define MILLI_2_NANO_SECONDS
Definition coroapi.h:105
int coroparse_configparse(icmap_map_t config_map, const char **error_string)
Definition coroparse.c:259
#define CS_PRI_RING_ID_SEQ
Definition corotypes.h:61
#define CS_FALSE
Definition corotypes.h:53
#define CS_TRUE
Definition corotypes.h:54
@ CS_OK
Definition corotypes.h:99
int corosync_quorum_is_quorate(void)
Definition exec/quorum.c:65
#define corosync_exit_error(err)
Definition exec/util.h:72
e_corosync_done
Definition exec/util.h:47
@ COROSYNC_DONE_INIT_SERVICES
Definition exec/util.h:54
@ COROSYNC_DONE_STATS
Definition exec/util.h:62
@ COROSYNC_DONE_ICMAP
Definition exec/util.h:53
@ COROSYNC_DONE_ALREADY_RUNNING
Definition exec/util.h:58
@ COROSYNC_DONE_FORK
Definition exec/util.h:49
@ COROSYNC_DONE_STORE_RINGID
Definition exec/util.h:61
@ COROSYNC_DONE_EXIT
Definition exec/util.h:48
@ COROSYNC_DONE_LOGSETUP
Definition exec/util.h:52
@ COROSYNC_DONE_FATAL_ERR
Definition exec/util.h:55
@ COROSYNC_DONE_LOGCONFIGREAD
Definition exec/util.h:50
@ COROSYNC_DONE_STD_TO_NULL_REDIR
Definition exec/util.h:59
@ COROSYNC_DONE_MAINCONFIGREAD
Definition exec/util.h:51
@ COROSYNC_DONE_ACQUIRE_LOCK
Definition exec/util.h:57
@ COROSYNC_DONE_DIR_NOT_PRESENT
Definition exec/util.h:56
#define ICMAP_TRACK_MODIFY
Definition icmap.h:78
cs_error_t icmap_set_ro_access(const char *key_name, int prefix, int ro_access)
Set read-only access for given key (key_name) or prefix, If prefix is set.
Definition icmap.c:1229
#define ICMAP_TRACK_DELETE
Definition icmap.h:77
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition icmap.c:1163
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition icmap.c:631
cs_error_t icmap_inc(const char *key_name)
Increase stored value by one.
Definition icmap.c:1053
cs_error_t icmap_fast_inc(const char *key_name)
Increase stored value by one.
Definition icmap.c:1073
icmap_map_t icmap_get_global_map(void)
Return global icmap.
Definition icmap.c:268
cs_error_t icmap_get(const char *key_name, void *value, size_t *value_len, icmap_value_types_t *type)
Retrieve value of key key_name and store it in user preallocated value pointer.
Definition icmap.c:729
cs_error_t icmap_init(void)
Initialize global icmap.
Definition icmap.c:212
#define ICMAP_KEYNAME_MAXLEN
Maximum length of key in icmap.
Definition icmap.h:48
void icmap_fini(void)
Finalize global icmap.
Definition icmap.c:251
cs_error_t icmap_set_uint32(const char *key_name, uint32_t value)
Definition icmap.c:601
#define ICMAP_TRACK_ADD
Definition icmap.h:76
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition icmap.c:860
void cs_ipcs_init(void)
Definition ipc_glue.c:818
void cs_ipcs_sync_state_changed(int32_t sync_in_process)
Definition ipc_glue.c:664
int32_t cs_ipcs_q_level_get(void)
Definition ipc_glue.c:601
void cs_ipc_allow_connections(int32_t allow)
Definition ipc_glue.c:159
int corosync_log_config_read(cmap_handle_t cmap_h, const char *default_logfile, const char **error_string)
Definition logconfig.c:719
#define LOGSYS_LEVEL_ERROR
Definition logsys.h:72
#define log_printf(level, format, args...)
Definition logsys.h:332
void logsys_blackbox_prefork(void)
Definition logsys.c:897
void logsys_system_fini(void)
logsys_system_fini
Definition logsys.c:286
#define LOGSYS_LEVEL_INFO
Definition logsys.h:75
#define LOGSYS_MODE_OUTPUT_STDERR
Definition logsys.h:61
#define LOGSYS_LEVEL_NOTICE
Definition logsys.h:74
void logsys_config_apply(void)
logsys_config_apply
Definition logsys.c:792
#define LOGSYS_PERROR(err_num, level, fmt, args...)
The LOGSYS_PERROR macro.
Definition logsys.h:326
int logsys_thread_start(void)
logsys_thread_start
Definition logsys.c:861
#define LOGSYS_DECLARE_SUBSYS(subsys)
The LOGSYS_DECLARE_SUBSYS macro.
Definition logsys.h:306
#define LOGSYS_LEVEL_WARNING
Definition logsys.h:73
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition logsys.c:435
void logsys_blackbox_postfork(void)
Definition logsys.c:903
#define LOGSYS_LEVEL_DEBUG
Definition logsys.h:76
#define LOGSYS_MODE_OUTPUT_SYSLOG
Definition logsys.h:62
#define LOGSYS_LEVEL_TRACE
Definition logsys.h:77
#define LOGSYS_DECLARE_SYSTEM(name, mode, syslog_facility, syslog_priority)
The LOGSYS_DECLARE_SYSTEM macro.
Definition logsys.h:290
void corosync_sending_allowed_release(void *sending_allowed_private_data)
Definition main.c:785
void corosync_state_dump(void)
Definition main.c:201
int cs_poll_dispatch_add(qb_loop_t *handle, int fd, int events, void *data, int(*dispatch_fn)(int fd, int revents, void *data))
Definition main.c:183
move_to_root_cgroup_mode
Definition main.c:172
@ MOVE_TO_ROOT_CGROUP_MODE_ON
Definition main.c:174
@ MOVE_TO_ROOT_CGROUP_MODE_OFF
Definition main.c:173
@ MOVE_TO_ROOT_CGROUP_MODE_AUTO
Definition main.c:175
#define RLIMIT_MEMLOCK
struct sched_param global_sched_param
Definition main.c:162
void message_source_set(mar_message_source_t *source, void *conn)
Definition main.c:807
int main(int argc, char **argv, char **envp)
Definition main.c:1284
int main_mcast(const struct iovec *iovec, unsigned int iov_len, unsigned int guarantee)
Definition main.c:627
qb_loop_t * cs_poll_handle_get(void)
Definition main.c:178
void corosync_recheck_the_q_level(void *data)
Definition main.c:723
const char * corosync_get_config_file(void)
Definition main.c:212
int message_source_is_local(const mar_message_source_t *source)
Definition main.c:796
int corosync_sending_allowed(unsigned int service, unsigned int id, const void *msg, void *sending_allowed_private_data)
Definition main.c:737
int cs_poll_dispatch_delete(qb_loop_t *handle, int fd)
Definition main.c:196
void corosync_shutdown_request(void)
Definition main.c:258
pthread_mutex_t lock
Definition sam.c:131
void * user_data
Definition sam.c:127
void schedwrk_init(void(*serialize_lock_fn)(void), void(*serialize_unlock_fn)(void))
Definition schedwrk.c:83
unsigned int corosync_service_defaults_link_and_init(struct corosync_api_v1 *corosync_api)
Load all of the default services.
Definition service.c:339
void corosync_service_unlink_all(struct corosync_api_v1 *api, void(*unlink_all_complete)(void))
Unlink and exit all corosync services.
Definition service.c:394
const char * service_stats_rx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition service.c:112
struct corosync_service_engine * corosync_service[SERVICES_COUNT_MAX]
Definition service.c:110
const char * service_stats_tx[SERVICES_COUNT_MAX][SERVICE_HANDLER_MAXIMUM_COUNT]
Definition service.c:113
void stats_add_schedmiss_event(uint64_t timestamp, float delay)
Definition stats.c:447
void stats_trigger_trackers()
Definition stats.c:561
cs_error_t stats_map_init(const struct corosync_api_v1 *corosync_api)
Definition stats.c:289
The corosync_api_v1 struct.
Definition coroapi.h:225
int(* timer_add_duration)(unsigned long long nanoseconds_in_future, void *data, void(*timer_nf)(void *data), corosync_timer_handle_t *handle)
Definition coroapi.h:229
void(* timer_delete)(corosync_timer_handle_t timer_handle)
Definition coroapi.h:241
const char *(* totem_ifaces_print)(unsigned int nodeid)
Definition coroapi.h:290
void *(* totem_get_stats)(void)
Definition coroapi.h:435
void(* exec_handler_fn)(const void *msg, unsigned int nodeid)
Definition coroapi.h:476
void(* exec_endian_convert_fn)(void *msg)
Definition coroapi.h:477
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition coroapi.h:515
const char * name
Definition coroapi.h:491
void(* sync_activate)(void)
Definition coroapi.h:522
void(* confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id)
Definition coroapi.h:509
void(* sync_abort)(void)
Definition coroapi.h:523
struct corosync_exec_handler * exec_engine
Definition coroapi.h:506
void(* exec_dump_fn)(void)
Definition coroapi.h:501
int(* sync_process)(void)
Definition coroapi.h:521
Structure passed as new_value and old_value in change callback.
Definition icmap.h:91
The mar_message_source_t struct.
Definition coroapi.h:50
The memb_ring_id struct.
Definition coroapi.h:122
unsigned long long seq
Definition coroapi.h:124
unsigned int rep
Definition totem.h:150
qb_loop_timer_handle handle
Definition main.c:819
unsigned long long max_tv_diff
Definition main.c:821
struct totem_config * totem_config
Definition main.c:818
unsigned long long tv_prev
Definition main.c:820
void(* sync_init)(const unsigned int *trans_list, size_t trans_list_entries, const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition sync.h:39
const char * name
Definition sync.h:48
int(* sync_process)(void)
Definition sync.h:45
void(* sync_activate)(void)
Definition sync.h:46
void(* sync_abort)(void)
Definition sync.h:47
unsigned int token_timeout
Definition totem.h:182
struct totem_logging_configuration totem_logging_configuration
Definition totem.h:208
void(* totem_memb_ring_id_create_or_load)(struct memb_ring_id *memb_ring_id, unsigned int nodeid)
Definition totem.h:250
void(* totem_memb_ring_id_store)(const struct memb_ring_id *memb_ring_id, unsigned int nodeid)
Definition totem.h:254
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition totem.h:101
void(*) in log_level_security)
Definition totem.h:110
const void * group
Definition totempg.h:56
totemsrp_stats_t * srp
Definition totemstats.h:96
uint32_t continuous_gather
Definition totemstats.h:78
uint32_t avg_backlog_calc
Definition totemstats.h:85
uint32_t continuous_sendmsg_failures
Definition totemstats.h:79
totemsrp_token_stats_t token[TOTEM_TOKEN_STATS_MAX]
Definition totemstats.h:90
uint8_t firewall_enabled_or_nic_failure
Definition totemstats.h:82
uint32_t avg_token_workload
Definition totemstats.h:84
uint32_t mtt_rx_token
Definition totemstats.h:83
uint64_t time_since_token_last_received
Definition totemstats.h:80
#define swab32(x)
The swab32 macro.
Definition swab.h:51
int sync_init(int(*sync_callbacks_retrieve)(int service_id, struct sync_callbacks *callbacks), void(*synchronization_completed)(void))
Definition sync.c:158
void sync_abort(void)
Definition sync.c:535
void sync_save_transitional(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition sync.c:524
void sync_start(const unsigned int *member_list, size_t member_list_entries, const struct memb_ring_id *ring_id)
Definition sync.c:512
typedef __attribute__
#define MAX_NO_CONT_SENDMSG_FAILURES
Definition totem.h:80
#define MAX_NO_CONT_GATHER
Maximum number of continuous gather states.
Definition totem.h:76
int totem_config_read(struct totem_config *totem_config, const char **error_string, uint64_t *warnings)
int totem_config_validate(struct totem_config *totem_config, const char **error_string)
#define TOTEM_CONFIG_BINDNETADDR_NODELIST_SET
Definition totemconfig.h:48
#define TOTEM_CONFIG_WARNING_TOTEM_NODEID_SET
Definition totemconfig.h:47
#define TOTEM_CONFIG_WARNING_MEMBERS_IGNORED
Definition totemconfig.h:45
#define TOTEM_CONFIG_WARNING_MEMBERS_DEPRECATED
Definition totemconfig.h:46
Totem Single Ring Protocol.
unsigned int totempg_my_nodeid_get(void)
Definition totempg.c:1532
void totempg_check_q_level(void *instance)
Definition totempg.c:1294
int totempg_groups_mcast_joined(void *instance, const struct iovec *iovec, unsigned int iov_len, int guarantee)
Definition totempg.c:1232
void totempg_trans_ack(void)
Definition totempg.c:1591
void totempg_force_gather(void)
Definition totempg.c:1596
int totempg_initialize(qb_loop_t *poll_handle, struct totem_config *totem_config)
Initialize the totem process groups abstraction.
Definition totempg.c:802
int totempg_groups_joined_release(int msg_count)
Definition totempg.c:1346
void totempg_service_ready_register(void(*totem_service_ready)(void))
Definition totempg.c:1541
int totempg_groups_joined_reserve(void *instance, const struct iovec *iovec, unsigned int iov_len)
Definition totempg.c:1302
int totempg_groups_join(void *instance, const struct totempg_group *groups, size_t group_cnt)
Definition totempg.c:1182
void totempg_finalize(void)
Definition totempg.c:855
@ TOTEM_Q_LEVEL_CRITICAL
Definition totempg.h:184
int totempg_groups_initialize(void **instance, void(*deliver_fn)(unsigned int nodeid, const void *msg, unsigned int msg_len, int endian_conversion_required), void(*confchg_fn)(enum totem_configuration_type configuration_type, const unsigned int *member_list, size_t member_list_entries, const unsigned int *left_list, size_t left_list_entries, const unsigned int *joined_list, size_t joined_list_entries, const struct memb_ring_id *ring_id))
Initialize a groups instance.
Definition totempg.c:1134
int guarantee
Definition totemsrp.c:6
struct memb_ring_id ring_id
Definition totemsrp.c:4
struct totem_message_header header
Definition totemsrp.c:0
Totem Single Ring Protocol.
#define TOTEM_TOKEN_STATS_MAX
Definition totemstats.h:89
int util_is_valid_knet_crypto_model(const char *val, const char **list_str, int machine_parseable_str, const char *error_string_prefix, const char **error_string)
Definition util.c:213
const char * get_state_dir(void)
Definition util.c:174
int util_is_valid_knet_compress_model(const char *val, const char **list_str, int machine_parseable_str, const char *error_string_prefix, const char **error_string)
Definition util.c:279