corosync 3.1.9
totemknet.c
Go to the documentation of this file.
1/*
2 * Copyright (c) 2016-2022 Red Hat, Inc.
3 *
4 * All rights reserved.
5 *
6 * Author: Christine Caulfield (ccaulfie@redhat.com)
7
8 * This software licensed under BSD license, the text of which follows:
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions are met:
12 *
13 * - Redistributions of source code must retain the above copyright notice,
14 * this list of conditions and the following disclaimer.
15 * - Redistributions in binary form must reproduce the above copyright notice,
16 * this list of conditions and the following disclaimer in the documentation
17 * and/or other materials provided with the distribution.
18 * - Neither the name of the MontaVista Software, Inc. nor the names of its
19 * contributors may be used to endorse or promote products derived from this
20 * software without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
23 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
26 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
32 * THE POSSIBILITY OF SUCH DAMAGE.
33 */
34
35#include <config.h>
36
37#include <assert.h>
38#include <sys/mman.h>
39#include <sys/types.h>
40#include <sys/stat.h>
41#include <sys/socket.h>
42#include <netdb.h>
43#include <sys/un.h>
44#include <sys/ioctl.h>
45#include <sys/param.h>
46#include <netinet/in.h>
47#include <net/ethernet.h>
48#include <arpa/inet.h>
49#include <unistd.h>
50#include <fcntl.h>
51#include <stdlib.h>
52#include <stdio.h>
53#include <errno.h>
54#include <pthread.h>
55#include <sched.h>
56#include <time.h>
57#include <sys/time.h>
58#include <sys/poll.h>
59#include <sys/uio.h>
60#include <limits.h>
61
62#include <qb/qbdefs.h>
63#include <qb/qbloop.h>
64#ifdef HAVE_LIBNOZZLE
65#include <libgen.h>
66#include <libnozzle.h>
67#endif
68
69#include <corosync/sq.h>
70#include <corosync/swab.h>
71#include <corosync/logsys.h>
72#include <corosync/icmap.h>
73#include <corosync/totem/totemip.h>
74#include "totemknet.h"
75
76#include "main.h"
77#include "util.h"
78
79#include <libknet.h>
80#include <corosync/totem/totemstats.h>
81
82#ifndef MSG_NOSIGNAL
83#define MSG_NOSIGNAL 0
84#endif
85
86#ifdef HAVE_LIBNOZZLE
87static int setup_nozzle(void *knet_context);
88#endif
89
90/* Should match that used by cfg */
91#define CFG_INTERFACE_STATUS_MAX_LEN 512
92
93struct totemknet_instance {
94 struct crypto_instance *crypto_inst;
95
96 struct knet_handle_crypto_cfg last_good_crypto_cfg;
97
98 qb_loop_t *poll_handle;
99
100 knet_handle_t knet_handle;
101
102 int link_mode;
103
104 void *context;
105
106 int (*totemknet_deliver_fn) (
107 void *context,
108 const void *msg,
109 unsigned int msg_len,
110 const struct sockaddr_storage *system_from);
111
112 int (*totemknet_iface_change_fn) (
113 void *context,
114 const struct totem_ip_address *iface_address,
115 unsigned int link_no);
116
117 void (*totemknet_mtu_changed) (
118 void *context,
119 int net_mtu);
120
121 void (*totemknet_target_set_completed) (void *context);
122
123 /*
124 * Function and data used to log messages
125 */
126 int totemknet_log_level_security;
127
128 int totemknet_log_level_error;
129
130 int totemknet_log_level_warning;
131
132 int totemknet_log_level_notice;
133
134 int totemknet_log_level_debug;
135
136 int totemknet_subsys_id;
137
138 int knet_subsys_id;
139
140 void (*totemknet_log_printf) (
141 int level,
142 int subsys,
143 const char *function,
144 const char *file,
145 int line,
146 const char *format,
147 ...)__attribute__((format(printf, 6, 7)));
148
149 void *knet_context;
150
151 char iov_buffer[KNET_MAX_PACKET_SIZE];
152
153 char *link_status[INTERFACE_MAX];
154
155 struct totem_ip_address my_ids[INTERFACE_MAX];
156
157 uint16_t ip_port[INTERFACE_MAX];
158
159 int our_nodeid;
160
161 int loopback_link;
162
163 struct totem_config *totem_config;
164
165 struct totem_ip_address token_target;
166
167 qb_loop_timer_handle timer_netif_check_timeout;
168
169 qb_loop_timer_handle timer_merge_detect_timeout;
170
171 int send_merge_detect_message;
172
173 unsigned int merge_detect_messages_sent_before_timeout;
174
175 int logpipes[2];
176 int knet_fd;
177
178 pthread_mutex_t log_mutex;
179#ifdef HAVE_LIBNOZZLE
180 char *nozzle_name;
181 char *nozzle_ipaddr;
182 char *nozzle_prefix;
183 char *nozzle_macaddr;
184 nozzle_t nozzle_handle;
185#endif
186};
187
188/* Awkward. But needed to get stats from knet */
189struct totemknet_instance *global_instance;
190
191struct work_item {
192 const void *msg;
193 unsigned int msg_len;
194 struct totemknet_instance *instance;
195};
196
197int totemknet_member_list_rebind_ip (
198 void *knet_context);
199
200
201static int totemknet_configure_compression (
202 struct totemknet_instance *instance,
203 struct totem_config *totem_config);
204
205static void totemknet_start_merge_detect_timeout(
206 void *knet_context);
207
208static void totemknet_stop_merge_detect_timeout(
209 void *knet_context);
210
211static void log_flush_messages (
212 void *knet_context);
213
214static void totemknet_instance_initialize (struct totemknet_instance *instance)
215{
216 int res;
217
218 memset (instance, 0, sizeof (struct totemknet_instance));
219 res = pthread_mutex_init(&instance->log_mutex, NULL);
220 /*
221 * There is not too much else what can be done.
222 */
223 assert(res == 0);
224}
225
226#define knet_log_printf_lock(level, subsys, function, file, line, format, args...) \
227do { \
228 (void)pthread_mutex_lock(&instance->log_mutex); \
229 instance->totemknet_log_printf ( \
230 level, subsys, function, file, line, \
231 (const char *)format, ##args); \
232 (void)pthread_mutex_unlock(&instance->log_mutex); \
233} while (0);
234
235#define knet_log_printf(level, format, args...) \
236do { \
237 knet_log_printf_lock ( \
238 level, instance->totemknet_subsys_id, \
239 __FUNCTION__, __FILE__, __LINE__, \
240 (const char *)format, ##args); \
241} while (0);
242
243#define libknet_log_printf(level, format, args...) \
244do { \
245 knet_log_printf_lock ( \
246 level, instance->knet_subsys_id, \
247 __FUNCTION__, "libknet.h", __LINE__, \
248 (const char *)format, ##args); \
249} while (0);
250
251#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...) \
252do { \
253 char _error_str[LOGSYS_MAX_PERROR_MSG_LEN]; \
254 const char *_error_ptr = qb_strerror_r(err_num, _error_str, sizeof(_error_str)); \
255 instance->totemknet_log_printf ( \
256 level, instance->totemknet_subsys_id, \
257 __FUNCTION__, __FILE__, __LINE__, \
258 fmt ": %s (%d)", ##args, _error_ptr, err_num); \
259 } while(0)
260
261
262#ifdef HAVE_LIBNOZZLE
263static inline int is_ether_addr_multicast(const uint8_t *addr)
264{
265 return (addr[0] & 0x01);
266}
267static inline int is_ether_addr_zero(const uint8_t *addr)
268{
269 return (!addr[0] && !addr[1] && !addr[2] && !addr[3] && !addr[4] && !addr[5]);
270}
271
272static int ether_host_filter_fn(void *private_data,
273 const unsigned char *outdata,
274 ssize_t outdata_len,
275 uint8_t tx_rx,
276 knet_node_id_t this_host_id,
277 knet_node_id_t src_host_id,
278 int8_t *channel,
279 knet_node_id_t *dst_host_ids,
280 size_t *dst_host_ids_entries)
281{
282 struct ether_header *eth_h = (struct ether_header *)outdata;
283 uint8_t *dst_mac = (uint8_t *)eth_h->ether_dhost;
284 uint16_t dst_host_id;
285
286 if (is_ether_addr_zero(dst_mac))
287 return -1;
288
289 if (is_ether_addr_multicast(dst_mac)) {
290 return 1;
291 }
292
293 memmove(&dst_host_id, &dst_mac[4], 2);
294
295 dst_host_ids[0] = ntohs(dst_host_id);
296 *dst_host_ids_entries = 1;
297
298 return 0;
299}
300#endif
301
302static int dst_host_filter_callback_fn(void *private_data,
303 const unsigned char *outdata,
304 ssize_t outdata_len,
305 uint8_t tx_rx,
306 knet_node_id_t this_host_id,
307 knet_node_id_t src_host_id,
308 int8_t *channel,
309 knet_node_id_t *dst_host_ids,
310 size_t *dst_host_ids_entries)
311{
312 struct totem_message_header *header = (struct totem_message_header *)outdata;
313 int res;
314
315#ifdef HAVE_LIBNOZZLE
316 if (*channel != 0) {
317 return ether_host_filter_fn(private_data,
318 outdata, outdata_len,
319 tx_rx,
320 this_host_id, src_host_id,
321 channel,
322 dst_host_ids,
323 dst_host_ids_entries);
324 }
325#endif
326 if (header->target_nodeid) {
327 dst_host_ids[0] = header->target_nodeid;
328 *dst_host_ids_entries = 1;
329 res = 0; /* unicast message */
330 }
331 else {
332 *dst_host_ids_entries = 0;
333 res = 1; /* multicast message */
334 }
335 return res;
336}
337
338static void socket_error_callback_fn(void *private_data, int datafd, int8_t channel, uint8_t tx_rx, int error, int errorno)
339{
340 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
341
342 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet socket ERROR notification called: txrx=%d, error=%d, errorno=%d", tx_rx, error, errorno);
343 if ((error == -1 && errorno != EAGAIN) || (error == 0)) {
344 knet_handle_remove_datafd(instance->knet_handle, datafd);
345 }
346}
347
348static void host_change_callback_fn(void *private_data, knet_node_id_t host_id, uint8_t reachable, uint8_t remote, uint8_t external)
349{
350 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
351
352 // TODO: what? if anything.
353 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet host change callback. nodeid: " CS_PRI_NODE_ID " reachable: %d", host_id, reachable);
354}
355
356static void pmtu_change_callback_fn(void *private_data, unsigned int data_mtu)
357{
358 struct totemknet_instance *instance = (struct totemknet_instance *)private_data;
359 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Knet pMTU change: %d", data_mtu);
360
361 /* We don't need to tell corosync the actual knet MTU */
362// instance->totemknet_mtu_changed(instance->context, data_mtu);
363}
364
365int totemknet_crypto_set (
366 void *knet_context,
367 const char *cipher_type,
368 const char *hash_type)
369{
370 return (0);
371}
372
373
374static inline void ucast_sendmsg (
375 struct totemknet_instance *instance,
376 struct totem_ip_address *system_to,
377 const void *msg,
378 unsigned int msg_len)
379{
380 int res = 0;
381 struct totem_message_header *header = (struct totem_message_header *)msg;
382 struct msghdr msg_ucast;
383 struct iovec iovec;
384
385 header->target_nodeid = system_to->nodeid;
386
387 iovec.iov_base = (void *)msg;
388 iovec.iov_len = msg_len;
389
390 /*
391 * Build unicast message
392 */
393 memset(&msg_ucast, 0, sizeof(msg_ucast));
394 msg_ucast.msg_iov = (void *)&iovec;
395 msg_ucast.msg_iovlen = 1;
396#ifdef HAVE_MSGHDR_CONTROL
397 msg_ucast.msg_control = 0;
398#endif
399#ifdef HAVE_MSGHDR_CONTROLLEN
400 msg_ucast.msg_controllen = 0;
401#endif
402#ifdef HAVE_MSGHDR_FLAGS
403 msg_ucast.msg_flags = 0;
404#endif
405#ifdef HAVE_MSGHDR_ACCRIGHTS
406 msg_ucast.msg_accrights = NULL;
407#endif
408#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
409 msg_ucast.msg_accrightslen = 0;
410#endif
411
412 /*
413 * Transmit unicast message
414 * An error here is recovered by totemsrp
415 */
416
417 res = sendmsg (instance->knet_fd, &msg_ucast, MSG_NOSIGNAL);
418 if (res < 0) {
419 KNET_LOGSYS_PERROR (errno, instance->totemknet_log_level_debug,
420 "sendmsg(ucast) failed (non-critical)");
421 }
422}
423
424static inline void mcast_sendmsg (
425 struct totemknet_instance *instance,
426 const void *msg,
427 unsigned int msg_len,
428 int only_active)
429{
430 int res;
431 struct totem_message_header *header = (struct totem_message_header *)msg;
432 struct msghdr msg_mcast;
433 struct iovec iovec;
434
435 iovec.iov_base = (void *)msg;
436 iovec.iov_len = msg_len;
437
438 header->target_nodeid = 0;
439
440 /*
441 * Build multicast message
442 */
443 memset(&msg_mcast, 0, sizeof(msg_mcast));
444 msg_mcast.msg_iov = (void *)&iovec;
445 msg_mcast.msg_iovlen = 1;
446#ifdef HAVE_MSGHDR_CONTROL
447 msg_mcast.msg_control = 0;
448#endif
449#ifdef HAVE_MSGHDR_CONTROLLEN
450 msg_mcast.msg_controllen = 0;
451#endif
452#ifdef HAVE_MSGHDR_FLAGS
453 msg_mcast.msg_flags = 0;
454#endif
455#ifdef HAVE_MSGHDR_ACCRIGHTS
456 msg_mcast.msg_accrights = NULL;
457#endif
458#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
459 msg_mcast.msg_accrightslen = 0;
460#endif
461
462
463// log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_sendmsg. only_active=%d, len=%d", only_active, msg_len);
464
465 res = sendmsg (instance->knet_fd, &msg_mcast, MSG_NOSIGNAL);
466 if (res < msg_len) {
467 knet_log_printf (LOGSYS_LEVEL_DEBUG, "totemknet: mcast_send sendmsg returned %d", res);
468 }
469
470 if (!only_active || instance->send_merge_detect_message) {
471 /*
472 * Current message was sent to all nodes
473 */
474 instance->merge_detect_messages_sent_before_timeout++;
475 instance->send_merge_detect_message = 0;
476 }
477}
478
479static int node_compare(const void *aptr, const void *bptr)
480{
481 uint16_t a,b;
482
483 a = *(uint16_t *)aptr;
484 b = *(uint16_t *)bptr;
485
486 return a > b;
487}
488
489#ifndef OWN_INDEX_NONE
490#define OWN_INDEX_NONE -1
491#endif
492
493int totemknet_nodestatus_get (
494 void *knet_context,
495 unsigned int nodeid,
496 struct totem_node_status *node_status)
497{
498 int i;
499 int res = 0;
500 struct knet_link_status link_status;
501 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
502 struct knet_host_status knet_host_status;
503 uint8_t link_list[KNET_MAX_LINK];
504 size_t num_links;
505
506 if (!instance->knet_handle) {
507 return CS_ERR_NOT_EXIST; /* Not using knet */
508 }
509
510 if (!node_status) {
511 return CS_ERR_INVALID_PARAM;
512 }
513
514 res = knet_host_get_status(instance->knet_handle,
515 nodeid,
516 &knet_host_status);
517 if (res) {
518 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_host_status(%d) failed: %d", nodeid, res);
519 return (-1);
520 }
521 node_status->nodeid = nodeid;
522 node_status->reachable = knet_host_status.reachable;
523 node_status->remote = knet_host_status.remote;
524 node_status->external = knet_host_status.external;
525
526#ifdef HAVE_KNET_ONWIRE_VER
527 res = knet_handle_get_onwire_ver(instance->knet_handle,
528 nodeid,
529 &node_status->onwire_min,
530 &node_status->onwire_max,
531 &node_status->onwire_ver);
532 if (res) {
533 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_handle_get_onwire_ver(%d) failed: %d", nodeid, res);
534 return (-1);
535 }
536#endif
537 /* Get link info */
538 res = knet_link_get_link_list(instance->knet_handle,
539 nodeid, link_list, &num_links);
540 if (res) {
541 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_list(%d) failed: %d", nodeid, res);
542 return (-1);
543 }
544
545 /* node_status[] has been zeroed for us in totempg.c */
546 for (i=0; i < num_links; i++) {
547 if (!instance->totem_config->interfaces[link_list[i]].configured) {
548 continue;
549 }
550 res = knet_link_get_status(instance->knet_handle,
551 nodeid,
552 link_list[i],
553 &link_status,
554 sizeof(link_status));
555 if (res == 0) {
556 node_status->link_status[link_list[i]].enabled = link_status.enabled;
557 node_status->link_status[link_list[i]].connected = link_status.connected;
558 node_status->link_status[link_list[i]].dynconnected = link_status.dynconnected;
559 node_status->link_status[link_list[i]].mtu = link_status.mtu;
560 memcpy(node_status->link_status[link_list[i]].src_ipaddr, link_status.src_ipaddr, KNET_MAX_HOST_LEN);
561 memcpy(node_status->link_status[link_list[i]].dst_ipaddr, link_status.dst_ipaddr, KNET_MAX_HOST_LEN);
562 } else {
563 knet_log_printf (LOGSYS_LEVEL_WARNING, "knet_link_get_link_status(%d, %d) failed: %d", nodeid, link_list[i], res);
564 }
565 }
566 return res;
567}
568
569
570
571int totemknet_ifaces_get (void *knet_context,
572 char ***status,
573 unsigned int *iface_count)
574{
575 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
576 struct knet_link_status link_status;
577 knet_node_id_t host_list[KNET_MAX_HOST];
578 uint8_t link_list[KNET_MAX_LINK];
579 size_t num_hosts;
580 size_t num_links;
581 size_t link_idx;
582 int i,j;
583 char *ptr;
584 int res = 0;
585
586 /*
587 * Don't do the whole 'link_info' bit if the caller just wants
588 * a count of interfaces.
589 */
590 if (status) {
591 int own_idx = OWN_INDEX_NONE;
592
593 res = knet_host_get_host_list(instance->knet_handle,
594 host_list, &num_hosts);
595 if (res) {
596 return (-1);
597 }
598 qsort(host_list, num_hosts, sizeof(uint16_t), node_compare);
599
600 for (j=0; j<num_hosts; j++) {
601 if (host_list[j] == instance->our_nodeid) {
602 own_idx = j;
603 break;
604 }
605 }
606
607 for (i=0; i<INTERFACE_MAX; i++) {
608 memset(instance->link_status[i], 'd', CFG_INTERFACE_STATUS_MAX_LEN-1);
609 if (own_idx != OWN_INDEX_NONE) {
610 instance->link_status[i][own_idx] = 'n';
611 }
612 instance->link_status[i][num_hosts] = '\0';
613 }
614
615 /* This is all a bit "inside-out" because "status" is a set of strings per link
616 * and knet orders things by host
617 */
618 for (j=0; j<num_hosts; j++) {
619 if (own_idx != OWN_INDEX_NONE && j == own_idx) {
620 continue ;
621 }
622
623 res = knet_link_get_link_list(instance->knet_handle,
624 host_list[j], link_list, &num_links);
625 if (res) {
626 return (-1);
627 }
628
629 link_idx = 0;
630 for (i=0; i < num_links; i++) {
631 /*
632 * Skip over links that are unconfigured to corosync. This is basically
633 * link0 if corosync isn't using it for comms, as we will still
634 * have it set up for loopback.
635 */
636 if (!instance->totem_config->interfaces[link_list[i]].configured) {
637 continue;
638 }
639 ptr = instance->link_status[link_idx++];
640
641 res = knet_link_get_status(instance->knet_handle,
642 host_list[j],
643 link_list[i],
644 &link_status,
645 sizeof(link_status));
646 if (res == 0) {
647 ptr[j] = '0' + (link_status.enabled |
648 link_status.connected<<1 |
649 link_status.dynconnected<<2);
650 }
651 else {
652 knet_log_printf (LOGSYS_LEVEL_ERROR,
653 "totemknet_ifaces_get: Cannot get link status: %s", strerror(errno));
654 ptr[j] = '?';
655 }
656 }
657 }
658 *status = instance->link_status;
659 }
660
661 *iface_count = INTERFACE_MAX;
662
663 return (res);
664}
665
666int totemknet_finalize (
667 void *knet_context)
668{
669 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
670 int res = 0;
671 int i,j;
672 static knet_node_id_t nodes[KNET_MAX_HOST]; /* static to save stack */
673 uint8_t links[KNET_MAX_LINK];
674 size_t num_nodes;
675 size_t num_links;
676
677 knet_log_printf(LOG_DEBUG, "totemknet: finalize");
678
679 qb_loop_poll_del (instance->poll_handle, instance->logpipes[0]);
680 qb_loop_poll_del (instance->poll_handle, instance->knet_fd);
681
682 /*
683 * Disable forwarding to make knet flush send queue. This ensures that the LEAVE message will be sent.
684 */
685 res = knet_handle_setfwd(instance->knet_handle, 0);
686 if (res) {
687 knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_setfwd failed: %s", strerror(errno));
688 }
689
690 res = knet_host_get_host_list(instance->knet_handle, nodes, &num_nodes);
691 if (res) {
692 knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet node list for shutdown: %s", strerror(errno));
693 /* Crash out anyway */
694 goto finalise_error;
695 }
696
697 /* Tidily shut down all nodes & links. */
698 for (i=0; i<num_nodes; i++) {
699
700 res = knet_link_get_link_list(instance->knet_handle, nodes[i], links, &num_links);
701 if (res) {
702 knet_log_printf (LOGSYS_LEVEL_ERROR, "Cannot get knet link list for node " CS_PRI_NODE_ID ": %s", nodes[i], strerror(errno));
703 goto finalise_error;
704 }
705 for (j=0; j<num_links; j++) {
706 res = knet_link_set_enable(instance->knet_handle, nodes[i], links[j], 0);
707 if (res) {
708 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_set_enable(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
709 }
710 res = knet_link_clear_config(instance->knet_handle, nodes[i], links[j]);
711 if (res) {
712 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_link_clear_config(node " CS_PRI_NODE_ID ", link %d) failed: %s", nodes[i], links[j], strerror(errno));
713 }
714 }
715 res = knet_host_remove(instance->knet_handle, nodes[i]);
716 if (res) {
717 knet_log_printf (LOGSYS_LEVEL_ERROR, "totemknet: knet_host_remove(node " CS_PRI_NODE_ID ") failed: %s", nodes[i], strerror(errno));
718 }
719 }
720
721finalise_error:
722 res = knet_handle_free(instance->knet_handle);
723 if (res) {
724 knet_log_printf (LOGSYS_LEVEL_CRIT, "totemknet: knet_handle_free failed: %s", strerror(errno));
725 }
726
727 totemknet_stop_merge_detect_timeout(instance);
728
729 log_flush_messages(instance);
730
731 /*
732 * Error is deliberately ignored
733 */
734 (void)pthread_mutex_destroy(&instance->log_mutex);
735
736 return (res);
737}
738
739static int log_deliver_fn (
740 int fd,
741 int revents,
742 void *data)
743{
744 struct totemknet_instance *instance = (struct totemknet_instance *)data;
745 char buffer[sizeof(struct knet_log_msg)*4];
746 char *bufptr = buffer;
747 int done = 0;
748 int len;
749
750 len = read(fd, buffer, sizeof(buffer));
751 while (done < len) {
752 struct knet_log_msg *msg = (struct knet_log_msg *)bufptr;
753 switch (msg->msglevel) {
754 case KNET_LOG_ERR:
755 libknet_log_printf (LOGSYS_LEVEL_ERROR, "%s: %s",
756 knet_log_get_subsystem_name(msg->subsystem),
757 msg->msg);
758 break;
759 case KNET_LOG_WARN:
760 libknet_log_printf (LOGSYS_LEVEL_WARNING, "%s: %s",
761 knet_log_get_subsystem_name(msg->subsystem),
762 msg->msg);
763 break;
764 case KNET_LOG_INFO:
765 libknet_log_printf (LOGSYS_LEVEL_INFO, "%s: %s",
766 knet_log_get_subsystem_name(msg->subsystem),
767 msg->msg);
768 break;
769 case KNET_LOG_DEBUG:
770 libknet_log_printf (LOGSYS_LEVEL_DEBUG, "%s: %s",
771 knet_log_get_subsystem_name(msg->subsystem),
772 msg->msg);
773 break;
774#ifdef KNET_LOG_TRACE
775 case KNET_LOG_TRACE:
776 libknet_log_printf (LOGSYS_LEVEL_TRACE, "%s: %s",
777 knet_log_get_subsystem_name(msg->subsystem),
778 msg->msg);
779 break;
780#endif
781 }
782 bufptr += sizeof(struct knet_log_msg);
783 done += sizeof(struct knet_log_msg);
784 }
785 return 0;
786}
787
788static int data_deliver_fn (
789 int fd,
790 int revents,
791 void *data)
792{
793 struct totemknet_instance *instance = (struct totemknet_instance *)data;
794 struct msghdr msg_hdr;
795 struct iovec iov_recv;
796 struct sockaddr_storage system_from;
797 ssize_t msg_len;
798 int truncated_packet;
799
800 iov_recv.iov_base = instance->iov_buffer;
801 iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
802
803 msg_hdr.msg_name = &system_from;
804 msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
805 msg_hdr.msg_iov = &iov_recv;
806 msg_hdr.msg_iovlen = 1;
807#ifdef HAVE_MSGHDR_CONTROL
808 msg_hdr.msg_control = 0;
809#endif
810#ifdef HAVE_MSGHDR_CONTROLLEN
811 msg_hdr.msg_controllen = 0;
812#endif
813#ifdef HAVE_MSGHDR_FLAGS
814 msg_hdr.msg_flags = 0;
815#endif
816#ifdef HAVE_MSGHDR_ACCRIGHTS
817 msg_hdr.msg_accrights = NULL;
818#endif
819#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
820 msg_hdr.msg_accrightslen = 0;
821#endif
822
823 msg_len = recvmsg (fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
824 if (msg_len <= 0) {
825 return (0);
826 }
827
828 truncated_packet = 0;
829
830#ifdef HAVE_MSGHDR_FLAGS
831 if (msg_hdr.msg_flags & MSG_TRUNC) {
832 truncated_packet = 1;
833 }
834#else
835 /*
836 * We don't have MSGHDR_FLAGS, but we can (hopefully) safely make assumption that
837 * if bytes_received == KNET_MAX_PACKET_SIZE then packet is truncated
838 */
839 if (bytes_received == KNET_MAX_PACKET_SIZE) {
840 truncated_packet = 1;
841 }
842#endif
843
844 if (truncated_packet) {
845 knet_log_printf(instance->totemknet_log_level_error,
846 "Received too big message. This may be because something bad is happening"
847 "on the network (attack?), or you tried join more nodes than corosync is"
848 "compiled with (%u) or bug in the code (bad estimation of "
849 "the KNET_MAX_PACKET_SIZE). Dropping packet.", PROCESSOR_COUNT_MAX);
850 return (0);
851 }
852
853 /*
854 * Handle incoming message
855 */
856 instance->totemknet_deliver_fn (
857 instance->context,
858 instance->iov_buffer,
859 msg_len,
860 &system_from);
861
862 return (0);
863}
864
865static void timer_function_netif_check_timeout (
866 void *data)
867{
868 struct totemknet_instance *instance = (struct totemknet_instance *)data;
869 int i;
870 int res = 0;
871
872 for (i=0; i < INTERFACE_MAX; i++) {
873 if (!instance->totem_config->interfaces[i].configured) {
874 continue;
875 }
876 res = instance->totemknet_iface_change_fn (instance->context,
877 &instance->my_ids[i],
878 i);
879 }
880 if (res != 0) {
881 /* This is only called at startup, so we can quit here.
882 Refresh takes a different path */
883 corosync_exit_error(COROSYNC_DONE_MAINCONFIGREAD);
884 }
885}
886
887static void knet_set_access_list_config(struct totemknet_instance *instance)
888{
889#ifdef HAVE_KNET_ACCESS_LIST
890 uint32_t value;
891 cs_error_t err;
892
893 value = instance->totem_config->block_unlisted_ips;
894 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_enable access list: %d", value);
895
896 err = knet_handle_enable_access_lists(instance->knet_handle, value);
897 if (err) {
898 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_access_lists failed");
899 }
900#endif
901}
902
903void totemknet_configure_log_level()
904{
905 int logsys_log_mode;
906 int knet_log_mode = KNET_LOG_INFO;
907 uint8_t s;
908 int err;
909
910 if (!global_instance || !global_instance->knet_handle) {
911 return;
912 }
913
914 /* Reconfigure logging level */
915 logsys_log_mode = logsys_config_debug_get("KNET");
916
917 switch (logsys_log_mode) {
918 case LOGSYS_DEBUG_OFF:
919 knet_log_mode = KNET_LOG_INFO;
920 break;
921 case LOGSYS_DEBUG_ON:
922 knet_log_mode = KNET_LOG_DEBUG;
923 break;
924 case LOGSYS_DEBUG_TRACE:
925#ifdef KNET_LOG_TRACE
926 knet_log_mode = KNET_LOG_TRACE;
927#else
928 knet_log_mode = KNET_LOG_DEBUG;
929#endif
930 break;
931 }
932 log_printf (LOGSYS_LEVEL_DEBUG, "totemknet setting log level %s", knet_log_get_loglevel_name(knet_log_mode));
933 err = 0;
934 for (s = 0; s<KNET_MAX_SUBSYSTEMS; s++) {
935 err = knet_log_set_loglevel(global_instance->knet_handle, s, knet_log_mode);
936 }
937
938 /* If one fails, they all fail. no point in issuing KNET_MAX_SUBSYSTEMS errors */
939 if (err) {
940 log_printf (LOGSYS_LEVEL_ERROR, "totemknet failed to set log level: %s", strerror(errno));
941 }
942}
943
944
945/* NOTE: this relies on the fact that totem_reload_notify() is called first */
946static void totemknet_refresh_config(
947 int32_t event,
948 const char *key_name,
949 struct icmap_notify_value new_val,
950 struct icmap_notify_value old_val,
951 void *user_data)
952{
953 uint8_t reloading;
954 int after_reload;
955 uint32_t link_no;
956 size_t num_nodes;
957 knet_node_id_t host_ids[KNET_MAX_HOST];
958 int i;
959 int err;
960 struct totemknet_instance *instance = (struct totemknet_instance *)user_data;
961
962 ENTER();
963
964 /*
965 * If a full reload is in progress then don't do anything until it's done and
966 * can reconfigure it all atomically
967 */
968 if (icmap_get_uint8("config.totemconfig_reload_in_progress", &reloading) == CS_OK && reloading) {
969 return;
970 }
971
972 after_reload = (strcmp(key_name, "config.totemconfig_reload_in_progress") == 0);
973
974 knet_set_access_list_config(instance);
975
976 if (strcmp(key_name, "totem.knet_pmtud_interval") == 0 || after_reload) {
977 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_pmtud_interval now %u",
978 instance->totem_config->knet_pmtud_interval);
979 err = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
980 if (err) {
981 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
982 }
983 }
984
985 if (strcmp(key_name, "totem.knet_mtu") == 0 || after_reload) {
986 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet_mtu now %u", instance->totem_config->knet_mtu);
987 err = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
988 if (err) {
989 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud failed");
990 }
991 }
992
993 /* Configure link parameters for each node */
994 err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_nodes);
995 if (err != 0) {
996 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list failed");
997 }
998
999 for (i=0; i<num_nodes; i++) {
1000 int linkerr = 0;
1001 for (link_no = 0; link_no < INTERFACE_MAX; link_no++) {
1002 if (host_ids[i] == instance->our_nodeid || !instance->totem_config->interfaces[link_no].configured) {
1003 continue;
1004 }
1005
1006 err = knet_link_set_ping_timers(instance->knet_handle, host_ids[i], link_no,
1007 instance->totem_config->interfaces[link_no].knet_ping_interval,
1008 instance->totem_config->interfaces[link_no].knet_ping_timeout,
1009 instance->totem_config->interfaces[link_no].knet_ping_precision);
1010 if (err) {
1011 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
1012 linkerr = err;
1013 }
1014 err = knet_link_set_pong_count(instance->knet_handle, host_ids[i], link_no,
1015 instance->totem_config->interfaces[link_no].knet_pong_count);
1016 if (err) {
1017 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for node " CS_PRI_NODE_ID " link %d failed",host_ids[i], link_no);
1018 linkerr = err;
1019 }
1020 err = knet_link_set_priority(instance->knet_handle, host_ids[i], link_no,
1021 instance->totem_config->interfaces[link_no].knet_link_priority);
1022 if (err) {
1023 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for node " CS_PRI_NODE_ID " link %d failed", host_ids[i], link_no);
1024 linkerr = err;
1025 }
1026
1027 }
1028 if (linkerr) {
1029 icmap_set_string("config.reload_error_message", "Failed to set knet ping timers(2)");
1030 }
1031 }
1032
1033 /* Log levels get reconfigured from logconfig.c as that happens last in the reload */
1034 LEAVE();
1035}
1036
1037static void totemknet_add_config_notifications(struct totemknet_instance *instance)
1038{
1039 icmap_track_t icmap_track_totem = NULL;
1040 icmap_track_t icmap_track_reload = NULL;
1041
1042 ENTER();
1043
1044 icmap_track_add("totem.",
1045 ICMAP_TRACK_ADD | ICMAP_TRACK_DELETE | ICMAP_TRACK_MODIFY | ICMAP_TRACK_PREFIX,
1046 totemknet_refresh_config,
1047 instance,
1048 &icmap_track_totem);
1049
1050 icmap_track_add("config.totemconfig_reload_in_progress",
1051 ICMAP_TRACK_ADD | ICMAP_TRACK_MODIFY,
1052 totemknet_refresh_config,
1053 instance,
1054 &icmap_track_reload);
1055
1056 LEAVE();
1057}
1058
1059static int totemknet_is_crypto_enabled(const struct totemknet_instance *instance)
1060{
1061
1062 return (!(strcmp(instance->totem_config->crypto_cipher_type, "none") == 0 &&
1063 strcmp(instance->totem_config->crypto_hash_type, "none") == 0));
1064
1065}
1066
1067static int totemknet_set_knet_crypto(struct totemknet_instance *instance)
1068{
1069 struct knet_handle_crypto_cfg crypto_cfg;
1070 int res;
1071
1072 /* These have already been validated */
1073 memcpy(crypto_cfg.crypto_model, instance->totem_config->crypto_model, sizeof(crypto_cfg.crypto_model));
1074 memcpy(crypto_cfg.crypto_cipher_type, instance->totem_config->crypto_cipher_type, sizeof(crypto_cfg.crypto_model));
1075 memcpy(crypto_cfg.crypto_hash_type, instance->totem_config->crypto_hash_type, sizeof(crypto_cfg.crypto_model));
1076 memcpy(crypto_cfg.private_key, instance->totem_config->private_key, instance->totem_config->private_key_len);
1077 crypto_cfg.private_key_len = instance->totem_config->private_key_len;
1078
1079#ifdef HAVE_KNET_CRYPTO_RECONF
1080
1081 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s on index %d",
1082 crypto_cfg.crypto_model,
1083 crypto_cfg.crypto_cipher_type,
1084 crypto_cfg.crypto_hash_type,
1085 instance->totem_config->crypto_index
1086 );
1087
1088 /* If crypto is being disabled we need to explicitly allow cleartext traffic in knet */
1089 if (!totemknet_is_crypto_enabled(instance)) {
1090 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1091 if (res) {
1092 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(ALLOW) failed %s", strerror(errno));
1093 }
1094 }
1095
1096 /* use_config will be called later when all nodes are synced */
1097 res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, instance->totem_config->crypto_index);
1098 if (res == 0) {
1099 /* Keep a copy in case it fails in future */
1100 memcpy(&instance->last_good_crypto_cfg, &crypto_cfg, sizeof(crypto_cfg));
1101 }
1102 if (res == -1) {
1103 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: %s", instance->totem_config->crypto_index, strerror(errno));
1104 goto exit_error;
1105 }
1106 if (res == -2) {
1107 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config (index %d) failed: -2", instance->totem_config->crypto_index);
1108 goto exit_error;
1109 }
1110#else
1111 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Configuring crypto %s/%s/%s",
1112 crypto_cfg.crypto_model,
1113 crypto_cfg.crypto_cipher_type,
1114 crypto_cfg.crypto_hash_type
1115 );
1116
1117 res = knet_handle_crypto(instance->knet_handle, &crypto_cfg);
1118 if (res == -1) {
1119 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: %s", strerror(errno));
1120 goto exit_error;
1121 }
1122 if (res == -2) {
1123 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto failed: -2");
1124 goto exit_error;
1125 }
1126#endif
1127
1128exit_error:
1129#ifdef HAVE_KNET_CRYPTO_RECONF
1130 if (res) {
1131 icmap_set_string("config.reload_error_message", "Failed to set crypto parameters");
1132
1133 /* Restore the old values in cmap & totem_config */
1134 icmap_set_string("totem.crypto_cipher", instance->last_good_crypto_cfg.crypto_cipher_type);
1135 icmap_set_string("totem.crypto_hash", instance->last_good_crypto_cfg.crypto_hash_type);
1136 icmap_set_string("totem.crypto_model", instance->last_good_crypto_cfg.crypto_model);
1137
1138 memcpy(instance->totem_config->crypto_hash_type, instance->last_good_crypto_cfg.crypto_hash_type,
1139 sizeof(instance->last_good_crypto_cfg.crypto_hash_type));
1140 memcpy(instance->totem_config->crypto_cipher_type, instance->last_good_crypto_cfg.crypto_cipher_type,
1141 sizeof(instance->last_good_crypto_cfg.crypto_cipher_type));
1142 memcpy(instance->totem_config->crypto_model, instance->last_good_crypto_cfg.crypto_model,
1143 sizeof(instance->last_good_crypto_cfg.crypto_model));
1144 }
1145#endif
1146 return res;
1147}
1148
1149/*
1150 * Create an instance
1151 */
1152int totemknet_initialize (
1153 qb_loop_t *poll_handle,
1154 void **knet_context,
1155 struct totem_config *totem_config,
1156 totemsrp_stats_t *stats,
1157 void *context,
1158
1159 int (*deliver_fn) (
1160 void *context,
1161 const void *msg,
1162 unsigned int msg_len,
1163 const struct sockaddr_storage *system_from),
1164
1165 int (*iface_change_fn) (
1166 void *context,
1167 const struct totem_ip_address *iface_address,
1168 unsigned int link_no),
1169
1170 void (*mtu_changed) (
1171 void *context,
1172 int net_mtu),
1173
1174 void (*target_set_completed) (
1175 void *context))
1176{
1177 struct totemknet_instance *instance;
1178 char *tmp_str;
1179 int8_t channel=0;
1180 int allow_knet_handle_fallback=0;
1181 int res;
1182 int i;
1183
1184 instance = malloc (sizeof (struct totemknet_instance));
1185 if (instance == NULL) {
1186 return (-1);
1187 }
1188
1189 totemknet_instance_initialize (instance);
1190
1191 instance->totem_config = totem_config;
1192
1193 /*
1194 * Configure logging
1195 */
1196 instance->totemknet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1197 instance->totemknet_log_level_error = totem_config->totem_logging_configuration.log_level_error;
1198 instance->totemknet_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
1199 instance->totemknet_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
1200 instance->totemknet_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
1201 instance->totemknet_subsys_id = totem_config->totem_logging_configuration.log_subsys_id;
1202 instance->totemknet_log_printf = totem_config->totem_logging_configuration.log_printf;
1203
1204 instance->knet_subsys_id = _logsys_subsys_create("KNET", "libknet.h");
1205
1206 /*
1207 * Initialize local variables for totemknet
1208 */
1209
1210 instance->our_nodeid = instance->totem_config->node_id;
1211
1212 for (i=0; i< INTERFACE_MAX; i++) {
1213 totemip_copy(&instance->my_ids[i], &totem_config->interfaces[i].bindnet);
1214 instance->my_ids[i].nodeid = instance->our_nodeid;
1215 instance->ip_port[i] = totem_config->interfaces[i].ip_port;
1216
1217 /* Needed for totemsrp */
1218 totem_config->interfaces[i].boundto.nodeid = instance->our_nodeid;
1219 }
1220
1221 instance->poll_handle = poll_handle;
1222
1223 instance->context = context;
1224 instance->totemknet_deliver_fn = deliver_fn;
1225
1226 instance->totemknet_iface_change_fn = iface_change_fn;
1227
1228 instance->totemknet_mtu_changed = mtu_changed;
1229
1230 instance->totemknet_target_set_completed = target_set_completed;
1231
1232 instance->loopback_link = 0;
1233
1234 res = pipe(instance->logpipes);
1235 if (res == -1) {
1236 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to create pipe for instance->logpipes");
1237 goto exit_error;
1238 }
1239 if (fcntl(instance->logpipes[0], F_SETFL, O_NONBLOCK) == -1 ||
1240 fcntl(instance->logpipes[1], F_SETFL, O_NONBLOCK) == -1) {
1241 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "failed to set O_NONBLOCK flag for instance->logpipes");
1242 goto exit_error;
1243 }
1244
1245 if (icmap_get_string("system.allow_knet_handle_fallback", &tmp_str) == CS_OK) {
1246 if (strcmp(tmp_str, "yes") == 0) {
1247 allow_knet_handle_fallback = 1;
1248 }
1249 free(tmp_str);
1250 }
1251
1252#if defined(KNET_API_VER) && (KNET_API_VER == 2)
1253 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, KNET_HANDLE_FLAG_PRIVILEGED);
1254#else
1255 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG);
1256#endif
1257
1258 if (allow_knet_handle_fallback && !instance->knet_handle && errno == ENAMETOOLONG) {
1259 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_new failed, trying unprivileged");
1260#if defined(KNET_API_VER) && (KNET_API_VER == 2)
1261 instance->knet_handle = knet_handle_new(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1262#else
1263 instance->knet_handle = knet_handle_new_ex(instance->totem_config->node_id, instance->logpipes[1], KNET_LOG_DEBUG, 0);
1264#endif
1265 }
1266
1267 if (!instance->knet_handle) {
1268 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_CRIT, "knet_handle_new failed");
1269 goto exit_error;
1270 }
1271
1272 knet_set_access_list_config(instance);
1273
1274 res = knet_handle_pmtud_setfreq(instance->knet_handle, instance->totem_config->knet_pmtud_interval);
1275 if (res) {
1276 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_setfreq failed");
1277 }
1278 res = knet_handle_pmtud_set(instance->knet_handle, instance->totem_config->knet_mtu);
1279 if (res) {
1280 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_pmtud_set failed");
1281 }
1282 res = knet_handle_enable_filter(instance->knet_handle, instance, dst_host_filter_callback_fn);
1283 if (res) {
1284 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_filter failed");
1285 }
1286 res = knet_handle_enable_sock_notify(instance->knet_handle, instance, socket_error_callback_fn);
1287 if (res) {
1288 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_sock_notify failed");
1289 }
1290 res = knet_host_enable_status_change_notify(instance->knet_handle, instance, host_change_callback_fn);
1291 if (res) {
1292 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_host_enable_status_change_notify failed");
1293 }
1294 res = knet_handle_enable_pmtud_notify(instance->knet_handle, instance, pmtu_change_callback_fn);
1295 if (res) {
1296 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_WARNING, "knet_handle_enable_pmtud_notify failed");
1297 }
1298 global_instance = instance;
1299
1300 /* Setup knet logging level */
1301 totemknet_configure_log_level();
1302
1303 /* Get an fd into knet */
1304 instance->knet_fd = 0;
1305 res = knet_handle_add_datafd(instance->knet_handle, &instance->knet_fd, &channel);
1306 if (res) {
1307 knet_log_printf(LOG_DEBUG, "knet_handle_add_datafd failed: %s", strerror(errno));
1308 goto exit_error;
1309 }
1310
1311 /* Enable crypto if requested */
1312#ifdef HAVE_KNET_CRYPTO_RECONF
1313 if (totemknet_is_crypto_enabled(instance)) {
1314 res = totemknet_set_knet_crypto(instance);
1315 if (res == 0) {
1316 res = knet_handle_crypto_use_config(instance->knet_handle, totem_config->crypto_index);
1317 if (res) {
1318 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_use_config failed: %s", strerror(errno));
1319 goto exit_error;
1320 }
1321 } else {
1322 knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1323 goto exit_error;
1324 }
1325 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1326 if (res) {
1327 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (DISALLOW) failed: %s", strerror(errno));
1328 goto exit_error;
1329 }
1330
1331 } else {
1332 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_ALLOW_CLEAR_TRAFFIC);
1333 if (res) {
1334 knet_log_printf(LOG_DEBUG, "knet_handle_crypto_rx_clear_traffic (ALLOW) failed: %s", strerror(errno));
1335 goto exit_error;
1336 }
1337 }
1338#else
1339 if (totemknet_is_crypto_enabled(instance)) {
1340 res = totemknet_set_knet_crypto(instance);
1341 if (res) {
1342 knet_log_printf(LOG_DEBUG, "Failed to set up knet crypto");
1343 goto exit_error;
1344 }
1345 }
1346#endif
1347
1348 /* Set up compression */
1349 if (strcmp(totem_config->knet_compression_model, "none") != 0) {
1350 /* Not fatal, but will log */
1351 (void)totemknet_configure_compression(instance, totem_config);
1352 }
1353
1354 knet_handle_setfwd(instance->knet_handle, 1);
1355
1356 instance->link_mode = KNET_LINK_POLICY_PASSIVE;
1357 if (strcmp(instance->totem_config->link_mode, "active")==0) {
1358 instance->link_mode = KNET_LINK_POLICY_ACTIVE;
1359 }
1360 if (strcmp(instance->totem_config->link_mode, "rr")==0) {
1361 instance->link_mode = KNET_LINK_POLICY_RR;
1362 }
1363
1364 for (i=0; i<INTERFACE_MAX; i++) {
1365 instance->link_status[i] = malloc(CFG_INTERFACE_STATUS_MAX_LEN);
1366 if (!instance->link_status[i]) {
1367 goto exit_error;
1368 }
1369 }
1370
1371 qb_loop_poll_add (instance->poll_handle,
1372 QB_LOOP_MED,
1373 instance->logpipes[0],
1374 POLLIN, instance, log_deliver_fn);
1375
1376 qb_loop_poll_add (instance->poll_handle,
1377 QB_LOOP_HIGH,
1378 instance->knet_fd,
1379 POLLIN, instance, data_deliver_fn);
1380
1381 /*
1382 * Upper layer isn't ready to receive message because it hasn't
1383 * initialized yet. Add short timer to check the interfaces.
1384 */
1385 qb_loop_timer_add (instance->poll_handle,
1386 QB_LOOP_MED,
1387 100*QB_TIME_NS_IN_MSEC,
1388 (void *)instance,
1389 timer_function_netif_check_timeout,
1390 &instance->timer_netif_check_timeout);
1391
1392 totemknet_start_merge_detect_timeout(instance);
1393
1394 /* Start listening for config changes */
1395 totemknet_add_config_notifications(instance);
1396
1397 /* Add stats keys to icmap */
1398 stats_knet_add_handle();
1399
1400 knet_log_printf (LOGSYS_LEVEL_INFO, "totemknet initialized");
1401 *knet_context = instance;
1402
1403 return (0);
1404
1405exit_error:
1406 log_flush_messages(instance);
1407 free(instance);
1408 return (-1);
1409}
1410
1411void *totemknet_buffer_alloc (void)
1412{
1413 /* Need to have space for a message AND a struct mcast in case of encapsulated messages */
1414 return malloc(KNET_MAX_PACKET_SIZE + 512);
1415}
1416
1417void totemknet_buffer_release (void *ptr)
1418{
1419 return free (ptr);
1420}
1421
1422int totemknet_processor_count_set (
1423 void *knet_context,
1424 int processor_count)
1425{
1426 return (0);
1427}
1428
1429int totemknet_recv_flush (void *knet_context)
1430{
1431 return (0);
1432}
1433
1434int totemknet_send_flush (void *knet_context)
1435{
1436 return (0);
1437}
1438
1439int totemknet_token_send (
1440 void *knet_context,
1441 const void *msg,
1442 unsigned int msg_len)
1443{
1444 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1445 int res = 0;
1446
1447 ucast_sendmsg (instance, &instance->token_target, msg, msg_len);
1448
1449 return (res);
1450}
1451int totemknet_mcast_flush_send (
1452 void *knet_context,
1453 const void *msg,
1454 unsigned int msg_len)
1455{
1456 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1457 int res = 0;
1458
1459 mcast_sendmsg (instance, msg, msg_len, 0);
1460
1461 return (res);
1462}
1463
1464int totemknet_mcast_noflush_send (
1465 void *knet_context,
1466 const void *msg,
1467 unsigned int msg_len)
1468{
1469 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1470 int res = 0;
1471
1472 mcast_sendmsg (instance, msg, msg_len, 1);
1473
1474 return (res);
1475}
1476
1477
1478extern int totemknet_iface_check (void *knet_context)
1479{
1480 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1481 int res = 0;
1482
1483 knet_log_printf(LOG_DEBUG, "totemknet: iface_check");
1484
1485 return (res);
1486}
1487
1488extern void totemknet_net_mtu_adjust (void *knet_context, struct totem_config *totem_config)
1489{
1490 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1491
1492 knet_log_printf(LOG_DEBUG, "totemknet: Returning MTU of %d", totem_config->net_mtu);
1493}
1494
1495int totemknet_token_target_set (
1496 void *knet_context,
1497 unsigned int nodeid)
1498{
1499 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1500 int res = 0;
1501
1502 instance->token_target.nodeid = nodeid;
1503
1504 instance->totemknet_target_set_completed (instance->context);
1505
1506 return (res);
1507}
1508
1509extern int totemknet_recv_mcast_empty (
1510 void *knet_context)
1511{
1512 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1513 unsigned int res;
1514 struct sockaddr_storage system_from;
1515 struct msghdr msg_hdr;
1516 struct iovec iov_recv;
1517 struct pollfd ufd;
1518 int nfds;
1519 int msg_processed = 0;
1520
1521 iov_recv.iov_base = instance->iov_buffer;
1522 iov_recv.iov_len = KNET_MAX_PACKET_SIZE;
1523
1524 msg_hdr.msg_name = &system_from;
1525 msg_hdr.msg_namelen = sizeof (struct sockaddr_storage);
1526 msg_hdr.msg_iov = &iov_recv;
1527 msg_hdr.msg_iovlen = 1;
1528#ifdef HAVE_MSGHDR_CONTROL
1529 msg_hdr.msg_control = 0;
1530#endif
1531#ifdef HAVE_MSGHDR_CONTROLLEN
1532 msg_hdr.msg_controllen = 0;
1533#endif
1534#ifdef HAVE_MSGHDR_FLAGS
1535 msg_hdr.msg_flags = 0;
1536#endif
1537#ifdef HAVE_MSGHDR_ACCRIGHTS
1538 msg_msg_hdr.msg_accrights = NULL;
1539#endif
1540#ifdef HAVE_MSGHDR_ACCRIGHTSLEN
1541 msg_msg_hdr.msg_accrightslen = 0;
1542#endif
1543
1544 do {
1545 ufd.fd = instance->knet_fd;
1546 ufd.events = POLLIN;
1547 nfds = poll (&ufd, 1, 0);
1548 if (nfds == 1 && ufd.revents & POLLIN) {
1549 res = recvmsg (instance->knet_fd, &msg_hdr, MSG_NOSIGNAL | MSG_DONTWAIT);
1550 if (res != -1) {
1551 msg_processed = 1;
1552 } else {
1553 msg_processed = -1;
1554 }
1555 }
1556 } while (nfds == 1);
1557
1558 return (msg_processed);
1559}
1560
1561int totemknet_iface_set (void *knet_context,
1562 const struct totem_ip_address *local_addr,
1563 unsigned short ip_port,
1564 unsigned int iface_no)
1565{
1566 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1567
1568 totemip_copy(&instance->my_ids[iface_no], local_addr);
1569
1570 knet_log_printf(LOG_INFO, "Configured link number %d: local addr: %s, port=%d", iface_no, totemip_print(local_addr), ip_port);
1571
1572 instance->ip_port[iface_no] = ip_port;
1573
1574 return 0;
1575}
1576
1577
1578int totemknet_member_add (
1579 void *knet_context,
1580 const struct totem_ip_address *local,
1581 const struct totem_ip_address *member,
1582 int link_no)
1583{
1584 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1585 int err;
1586 int port = instance->ip_port[link_no];
1587 struct sockaddr_storage remote_ss;
1588 struct sockaddr_storage local_ss;
1589 int addrlen;
1590 int i;
1591 int host_found = 0;
1592 knet_node_id_t host_ids[KNET_MAX_HOST];
1593 size_t num_host_ids;
1594
1595 /* Only create 1 loopback link and use link 0 */
1596 if (member->nodeid == instance->our_nodeid) {
1597 if (!instance->loopback_link) {
1598 link_no = 0;
1599 instance->loopback_link = 1;
1600 } else {
1601 /* Already done */
1602 return 0;
1603 }
1604 }
1605
1606 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: " CS_PRI_NODE_ID " (%s), link=%d", member->nodeid, totemip_print(member), link_no);
1607 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: local: " CS_PRI_NODE_ID " (%s)", local->nodeid, totemip_print(local));
1608
1609
1610 /* Only add the host if it doesn't already exist in knet */
1611 err = knet_host_get_host_list(instance->knet_handle, host_ids, &num_host_ids);
1612 if (err) {
1613 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_get_host_list");
1614 return -1;
1615 }
1616 for (i=0; i<num_host_ids; i++) {
1617 if (host_ids[i] == member->nodeid) {
1618 host_found = 1;
1619 }
1620 }
1621
1622 if (!host_found) {
1623 err = knet_host_add(instance->knet_handle, member->nodeid);
1624 if (err != 0 && errno != EEXIST) {
1625 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_host_add");
1626 return -1;
1627 }
1628 } else {
1629 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nodeid " CS_PRI_NODE_ID " already added", member->nodeid);
1630 }
1631
1632
1633 if (err == 0) {
1634 if (knet_host_set_policy(instance->knet_handle, member->nodeid, instance->link_mode)) {
1635 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_set_policy failed");
1636 return -1;
1637 }
1638 }
1639
1640 memset(&local_ss, 0, sizeof(local_ss));
1641 memset(&remote_ss, 0, sizeof(remote_ss));
1642 /* Casts to remove const */
1643 totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)member, port, &remote_ss, &addrlen);
1644 totemip_totemip_to_sockaddr_convert((struct totem_ip_address *)local, port, &local_ss, &addrlen);
1645
1646 if (member->nodeid == instance->our_nodeid) {
1647 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: loopback link is %d\n", link_no);
1648
1649 err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1650 KNET_TRANSPORT_LOOPBACK,
1651 &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1652 }
1653 else {
1654 err = knet_link_set_config(instance->knet_handle, member->nodeid, link_no,
1655 instance->totem_config->interfaces[link_no].knet_transport,
1656 &local_ss, &remote_ss, KNET_LINK_FLAG_TRAFFICHIPRIO);
1657 }
1658 if (err) {
1659 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_config failed");
1660 return -1;
1661 }
1662
1663 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_add: Setting link prio to %d",
1664 instance->totem_config->interfaces[link_no].knet_link_priority);
1665
1666 err = knet_link_set_priority(instance->knet_handle, member->nodeid, link_no,
1667 instance->totem_config->interfaces[link_no].knet_link_priority);
1668 if (err) {
1669 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_priority for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1670 }
1671
1672 /*
1673 * Ping timeouts may be 0 here for a newly added interface (on a reload),
1674 * so we leave this till later, it will get done in totemknet_refresh_config.
1675 * For the initial startup, we are all preset and ready to go from here.
1676 */
1677 if (instance->totem_config->interfaces[link_no].knet_ping_interval != 0) {
1678 err = knet_link_set_ping_timers(instance->knet_handle, member->nodeid, link_no,
1679 instance->totem_config->interfaces[link_no].knet_ping_interval,
1680 instance->totem_config->interfaces[link_no].knet_ping_timeout,
1681 instance->totem_config->interfaces[link_no].knet_ping_precision);
1682 if (err) {
1683 /* Flush logs before reporting this error so that the knet message prints before ours */
1684 int saved_errno = errno;
1685 log_flush_messages(instance);
1686 errno = saved_errno;
1687 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_ping_timers for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1688
1689 icmap_set_string("config.reload_error_message", "Failed to set knet ping timers");
1690
1691 return -1;
1692 }
1693 err = knet_link_set_pong_count(instance->knet_handle, member->nodeid, link_no,
1694 instance->totem_config->interfaces[link_no].knet_pong_count);
1695 if (err) {
1696 /* Flush logs before reporting this error so that the knet message prints before ours */
1697 int saved_errno = errno;
1698 log_flush_messages(instance);
1699 errno = saved_errno;
1700 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_pong_count for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1701 icmap_set_string("config.reload_error_message", "Failed to set knet pong count");
1702 return -1;
1703 }
1704 }
1705
1706 err = knet_link_set_enable(instance->knet_handle, member->nodeid, link_no, 1);
1707 if (err) {
1708 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set_enable for nodeid " CS_PRI_NODE_ID ", link %d failed", member->nodeid, link_no);
1709 return -1;
1710 }
1711
1712 /* register stats */
1713 stats_knet_add_member(member->nodeid, link_no);
1714 return (0);
1715}
1716
1717int totemknet_member_remove (
1718 void *knet_context,
1719 const struct totem_ip_address *token_target,
1720 int link_no)
1721{
1722 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1723 int res;
1724 uint8_t link_list[KNET_MAX_LINK];
1725 size_t num_links;
1726
1727 knet_log_printf (LOGSYS_LEVEL_DEBUG, "knet: member_remove: " CS_PRI_NODE_ID ", link=%d", token_target->nodeid, link_no);
1728
1729 /* Don't remove the link with the loopback on it until we shut down */
1730 if (token_target->nodeid == instance->our_nodeid) {
1731 return 0;
1732 }
1733
1734 /* Tidy stats */
1735 stats_knet_del_member(token_target->nodeid, link_no);
1736
1737 /* Remove the link first */
1738 res = knet_link_set_enable(instance->knet_handle, token_target->nodeid, link_no, 0);
1739 if (res != 0) {
1740 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_set enable(off) for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1741 return res;
1742 }
1743
1744 res = knet_link_clear_config(instance->knet_handle, token_target->nodeid, link_no);
1745 if (res != 0) {
1746 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_link_clear_config for nodeid " CS_PRI_NODE_ID ", link %d failed", token_target->nodeid, link_no);
1747 return res;
1748 }
1749
1750 /* If this is the last link, then remove the node */
1751 res = knet_link_get_link_list(instance->knet_handle,
1752 token_target->nodeid, link_list, &num_links);
1753 if (res) {
1754 return (0); /* not really failure */
1755 }
1756
1757 if (num_links == 0) {
1758 res = knet_host_remove(instance->knet_handle, token_target->nodeid);
1759 }
1760 return res;
1761}
1762
1763int totemknet_member_list_rebind_ip (
1764 void *knet_context)
1765{
1766 return (0);
1767}
1768
1769
1770static int totemknet_configure_compression (
1771 struct totemknet_instance *instance,
1772 struct totem_config *totem_config)
1773{
1774 struct knet_handle_compress_cfg compress_cfg;
1775 int res = 0;
1776
1777 assert(strlen(totem_config->knet_compression_model) < sizeof(compress_cfg.compress_model));
1778 strcpy(compress_cfg.compress_model, totem_config->knet_compression_model);
1779
1780 compress_cfg.compress_threshold = totem_config->knet_compression_threshold;
1781 compress_cfg.compress_level = totem_config->knet_compression_level;
1782
1783 res = knet_handle_compress(instance->knet_handle, &compress_cfg);
1784 if (res) {
1785 KNET_LOGSYS_PERROR(errno, LOGSYS_LEVEL_ERROR, "knet_handle_compress failed");
1786 }
1787 return res;
1788}
1789
1790int totemknet_reconfigure (
1791 void *knet_context,
1792 struct totem_config *totem_config)
1793{
1794 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1795 int res = 0;
1796
1797 (void)totemknet_configure_compression(instance, totem_config);
1798
1799#ifdef HAVE_LIBNOZZLE
1800 /* Set up nozzle device(s). Return code is ignored, because inability
1801 * configure nozzle is not fatal problem, errors are logged and
1802 * there is not much else we can do */
1803 (void)setup_nozzle(instance);
1804#endif
1805
1806 if (totem_config->crypto_changed) {
1807 /* Flip crypto_index */
1808 totem_config->crypto_index = 3-totem_config->crypto_index;
1809 res = totemknet_set_knet_crypto(instance);
1810 if (res == 0) {
1811 knet_log_printf(LOG_INFO, "kronosnet crypto reconfigured on index %d: %s/%s/%s", totem_config->crypto_index,
1812 totem_config->crypto_model,
1813 totem_config->crypto_cipher_type,
1814 totem_config->crypto_hash_type);
1815 } else {
1816 icmap_set_string("config.reload_error_message", "Failed to set knet crypto");
1817 }
1818 }
1819 return (res);
1820}
1821
1822
1823int totemknet_crypto_reconfigure_phase (
1824 void *knet_context,
1825 struct totem_config *totem_config,
1826 cfg_message_crypto_reconfig_phase_t phase)
1827{
1828#ifdef HAVE_KNET_CRYPTO_RECONF
1829 int res;
1830 int config_to_use;
1831 int config_to_clear;
1832 struct knet_handle_crypto_cfg crypto_cfg;
1833 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1834
1835 knet_log_printf(LOGSYS_LEVEL_DEBUG, "totemknet_crypto_reconfigure_phase %d, index=%d\n", phase, totem_config->crypto_index);
1836
1837 switch (phase) {
1838 case CRYPTO_RECONFIG_PHASE_ACTIVATE:
1839 config_to_use = totem_config->crypto_index;
1840 if (!totemknet_is_crypto_enabled(instance)) {
1841 config_to_use = 0; /* we are clearing it */
1842 }
1843
1844 /* Enable the new config on this node */
1845 res = knet_handle_crypto_use_config(instance->knet_handle, config_to_use);
1846 if (res == -1) {
1847 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_use_config %d failed: %s", config_to_use, strerror(errno));
1848 }
1849 break;
1850
1851 case CRYPTO_RECONFIG_PHASE_CLEANUP:
1852 /*
1853 * All nodes should now have the new config. clear the old one out
1854 * OR disable crypto entirely if that's what the new config insists on.
1855 */
1856 config_to_clear = 3-totem_config->crypto_index;
1857 knet_log_printf(LOGSYS_LEVEL_DEBUG, "Clearing old knet crypto config %d\n", config_to_clear);
1858
1859 strcpy(crypto_cfg.crypto_model, "none");
1860 strcpy(crypto_cfg.crypto_cipher_type, "none");
1861 strcpy(crypto_cfg.crypto_hash_type, "none");
1862 res = knet_handle_crypto_set_config(instance->knet_handle, &crypto_cfg, config_to_clear);
1863 if (res == -1) {
1864 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: %s", config_to_clear, strerror(errno));
1865 }
1866 if (res == -2) {
1867 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_set_config to clear index %d failed: -2", config_to_clear);
1868 }
1869
1870 /* If crypto is enabled then disable all cleartext reception */
1871 if (totemknet_is_crypto_enabled(instance)) {
1872 res = knet_handle_crypto_rx_clear_traffic(instance->knet_handle, KNET_CRYPTO_RX_DISALLOW_CLEAR_TRAFFIC);
1873 if (res) {
1874 knet_log_printf(LOGSYS_LEVEL_ERROR, "knet_handle_crypto_rx_clear_traffic(DISALLOW) failed %s", strerror(errno));
1875 }
1876 }
1877 }
1878#endif
1879 return 0;
1880}
1881
1889
1890/* For the stats module */
1891int totemknet_link_get_status (
1892 knet_node_id_t node, uint8_t link_no,
1893 struct knet_link_status *status)
1894{
1895 int res;
1896 int ret = CS_OK;
1897
1898 /* We are probably not using knet */
1899 if (!global_instance) {
1900 return CS_ERR_NOT_EXIST;
1901 }
1902
1903 if (link_no >= INTERFACE_MAX) {
1904 return CS_ERR_NOT_EXIST; /* Invalid link number */
1905 }
1906
1907 res = knet_link_get_status(global_instance->knet_handle, node, link_no, status, sizeof(struct knet_link_status));
1908 if (res) {
1909 switch (errno) {
1910 case EINVAL:
1911 ret = CS_ERR_INVALID_PARAM;
1912 break;
1913 case EBUSY:
1914 ret = CS_ERR_BUSY;
1915 break;
1916 case EDEADLK:
1917 ret = CS_ERR_TRY_AGAIN;
1918 break;
1919 default:
1920 ret = CS_ERR_LIBRARY;
1921 break;
1922 }
1923 }
1924
1925 return (ret);
1926}
1927
1928int totemknet_handle_get_stats (
1929 struct knet_handle_stats *stats)
1930{
1931 int res;
1932
1933 /* We are probably not using knet */
1934 if (!global_instance) {
1935 return CS_ERR_NOT_EXIST;
1936 }
1937
1938 res = knet_handle_get_stats(global_instance->knet_handle, stats, sizeof(struct knet_handle_stats));
1939 if (res != 0) {
1940 return (qb_to_cs_error(-errno));
1941 }
1942
1943 return CS_OK;
1944}
1945
1946static void timer_function_merge_detect_timeout (
1947 void *data)
1948{
1949 struct totemknet_instance *instance = (struct totemknet_instance *)data;
1950
1951 if (instance->merge_detect_messages_sent_before_timeout == 0) {
1952 instance->send_merge_detect_message = 1;
1953 }
1954
1955 instance->merge_detect_messages_sent_before_timeout = 0;
1956
1957 totemknet_start_merge_detect_timeout(instance);
1958}
1959
1960static void totemknet_start_merge_detect_timeout(
1961 void *knet_context)
1962{
1963 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1964
1965 qb_loop_timer_add(instance->poll_handle,
1966 QB_LOOP_MED,
1967 instance->totem_config->merge_timeout * 2 * QB_TIME_NS_IN_MSEC,
1968 (void *)instance,
1969 timer_function_merge_detect_timeout,
1970 &instance->timer_merge_detect_timeout);
1971
1972}
1973
1974static void totemknet_stop_merge_detect_timeout(
1975 void *knet_context)
1976{
1977 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1978
1979 qb_loop_timer_del(instance->poll_handle,
1980 instance->timer_merge_detect_timeout);
1981}
1982
1983static void log_flush_messages (void *knet_context)
1984{
1985 struct pollfd pfd;
1986 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
1987 int cont;
1988
1989 cont = 1;
1990
1991 while (cont) {
1992 pfd.fd = instance->logpipes[0];
1993 pfd.events = POLLIN;
1994 pfd.revents = 0;
1995
1996 if ((poll(&pfd, 1, 0) > 0) &&
1997 (pfd.revents & POLLIN) &&
1998 (log_deliver_fn(instance->logpipes[0], POLLIN, instance) == 0)) {
1999 cont = 1;
2000 } else {
2001 cont = 0;
2002 }
2003 }
2004}
2005
2006
2007#ifdef HAVE_LIBNOZZLE
2008#define NOZZLE_NAME "nozzle.name"
2009#define NOZZLE_IPADDR "nozzle.ipaddr"
2010#define NOZZLE_PREFIX "nozzle.ipprefix"
2011#define NOZZLE_MACADDR "nozzle.macaddr"
2012
2013#define NOZZLE_CHANNEL 1
2014
2015
2016static char *get_nozzle_script_dir(void *knet_context)
2017{
2018 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2019 char filename[PATH_MAX + FILENAME_MAX + 1];
2020 static char updown_dirname[PATH_MAX + FILENAME_MAX + 1];
2021 int res;
2022 const char *dirname_res;
2023
2024 /*
2025 * Build script directory based on corosync.conf file location
2026 */
2027 res = snprintf(filename, sizeof(filename), "%s",
2028 corosync_get_config_file());
2029 if (res >= sizeof(filename)) {
2030 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
2031 return NULL;
2032 }
2033
2034 dirname_res = dirname(filename);
2035
2036 res = snprintf(updown_dirname, sizeof(updown_dirname), "%s/%s",
2037 dirname_res, "updown.d");
2038 if (res >= sizeof(updown_dirname)) {
2039 knet_log_printf (LOGSYS_LEVEL_DEBUG, "nozzle up/down path too long");
2040 return NULL;
2041 }
2042 return updown_dirname;
2043}
2044
2045/*
2046 * Deliberately doesn't return the status as caller doesn't care.
2047 * The result will be logged though
2048 */
2049static void run_nozzle_script(struct totemknet_instance *instance, int type, const char *typename)
2050{
2051 int res;
2052 char *exec_string;
2053
2054 res = nozzle_run_updown(instance->nozzle_handle, type, &exec_string);
2055 if (res == -1 && errno != ENOENT) {
2056 knet_log_printf (LOGSYS_LEVEL_INFO, "exec nozzle %s script failed: %s", typename, strerror(errno));
2057 } else if (res == -2) {
2058 knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle %s script failed", typename);
2059 knet_log_printf (LOGSYS_LEVEL_INFO, "%s", exec_string);
2060 }
2061}
2062
2063/*
2064 * Reparse IP address to add in our node ID
2065 * IPv6 addresses must end in '::'
2066 * IPv4 addresses must just be valid
2067 * '/xx' lengths are optional for IPv6, mandatory for IPv4
2068 *
2069 * Returns the modified IP address as a string to pass into libnozzle
2070 */
2071static int reparse_nozzle_ip_address(struct totemknet_instance *instance,
2072 const char *input_addr,
2073 const char *prefix, int nodeid,
2074 char *output_addr, size_t output_len)
2075{
2076 char *coloncolon;
2077 int bits;
2078 int max_prefix = 64;
2079 uint32_t nodeid_mask;
2080 uint32_t addr_mask;
2081 uint32_t masked_nodeid;
2082 struct in_addr *addr;
2083 struct totem_ip_address totemip;
2084
2085 coloncolon = strstr(input_addr, "::");
2086 if (!coloncolon) {
2087 max_prefix = 30;
2088 }
2089
2090 bits = atoi(prefix);
2091 if (bits < 8 || bits > max_prefix) {
2092 knet_log_printf(LOGSYS_LEVEL_ERROR, "nozzle IP address prefix must be >= 8 and <= %d (got %d)", max_prefix, bits);
2093 return -1;
2094 }
2095
2096 /* IPv6 is easy */
2097 if (coloncolon) {
2098 memcpy(output_addr, input_addr, coloncolon-input_addr);
2099 sprintf(output_addr + (coloncolon-input_addr), "::%x", nodeid);
2100 return 0;
2101 }
2102
2103 /* For IPv4 we need to parse the address into binary, mask off the required bits,
2104 * add in the masked_nodeid and 'print' it out again
2105 */
2106 nodeid_mask = UINT32_MAX & ((1<<(32 - bits)) - 1);
2107 addr_mask = UINT32_MAX ^ nodeid_mask;
2108 masked_nodeid = nodeid & nodeid_mask;
2109
2110 if (totemip_parse(&totemip, input_addr, AF_INET)) {
2111 knet_log_printf(LOGSYS_LEVEL_ERROR, "Failed to parse IPv4 nozzle IP address");
2112 return -1;
2113 }
2114 addr = (struct in_addr *)&totemip.addr;
2115 addr->s_addr &= htonl(addr_mask);
2116 addr->s_addr |= htonl(masked_nodeid);
2117
2118 inet_ntop(AF_INET, addr, output_addr, output_len);
2119 return 0;
2120}
2121
2122static int create_nozzle_device(void *knet_context, const char *name,
2123 const char *ipaddr, const char *prefix,
2124 const char *macaddr)
2125{
2126 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2127 char device_name[IFNAMSIZ+1];
2128 size_t size = IFNAMSIZ;
2129 int8_t channel = NOZZLE_CHANNEL;
2130 nozzle_t nozzle_dev;
2131 int nozzle_fd;
2132 int res;
2133 char *updown_dir;
2134 char parsed_ipaddr[INET6_ADDRSTRLEN];
2135 char mac[19];
2136
2137 memset(device_name, 0, size);
2138 memset(&mac, 0, sizeof(mac));
2139 strncpy(device_name, name, size);
2140
2141 updown_dir = get_nozzle_script_dir(knet_context);
2142 knet_log_printf (LOGSYS_LEVEL_INFO, "nozzle script dir is %s", updown_dir);
2143
2144 nozzle_dev = nozzle_open(device_name, size, updown_dir);
2145 if (!nozzle_dev) {
2146 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to init nozzle device %s: %s", device_name, strerror(errno));
2147 return -1;
2148 }
2149 instance->nozzle_handle = nozzle_dev;
2150
2151 if (nozzle_set_mac(nozzle_dev, macaddr) < 0) {
2152 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle MAC to %s: %s", mac, strerror(errno));
2153 goto out_clean;
2154 }
2155
2156 if (reparse_nozzle_ip_address(instance, ipaddr, prefix, instance->our_nodeid, parsed_ipaddr, sizeof(parsed_ipaddr))) {
2157 /* Prints its own errors */
2158 goto out_clean;
2159 }
2160 knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle IP address is %s / %d", parsed_ipaddr, atoi(prefix));
2161 if (nozzle_add_ip(nozzle_dev, parsed_ipaddr, prefix) < 0) {
2162 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add set nozzle IP addr to %s/%s: %s", parsed_ipaddr, prefix, strerror(errno));
2163 goto out_clean;
2164 }
2165
2166 nozzle_fd = nozzle_get_fd(nozzle_dev);
2167 knet_log_printf (LOGSYS_LEVEL_INFO, "Opened '%s' on fd %d", device_name, nozzle_fd);
2168
2169 res = knet_handle_add_datafd(instance->knet_handle, &nozzle_fd, &channel);
2170 if (res != 0) {
2171 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to add nozzle FD to knet: %s", strerror(errno));
2172 goto out_clean;
2173 }
2174
2175 run_nozzle_script(instance, NOZZLE_PREUP, "pre-up");
2176
2177 res = nozzle_set_up(nozzle_dev);
2178 if (res != 0) {
2179 knet_log_printf (LOGSYS_LEVEL_ERROR, "Unable to set nozzle interface UP: %s", strerror(errno));
2180 goto out_clean;
2181 }
2182 run_nozzle_script(instance, NOZZLE_UP, "up");
2183
2184 return 0;
2185
2186out_clean:
2187 nozzle_close(nozzle_dev);
2188 return -1;
2189}
2190
2191static int remove_nozzle_device(void *knet_context)
2192{
2193 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2194 int res;
2195 int datafd;
2196
2197 res = knet_handle_get_datafd(instance->knet_handle, NOZZLE_CHANNEL, &datafd);
2198 if (res != 0) {
2199 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't find datafd for channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2200 return -1;
2201 }
2202
2203 res = knet_handle_remove_datafd(instance->knet_handle, datafd);
2204 if (res != 0) {
2205 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't remove datafd for nozzle channel %d: %s", NOZZLE_CHANNEL, strerror(errno));
2206 return -1;
2207 }
2208
2209 run_nozzle_script(instance, NOZZLE_DOWN, "pre-down");
2210 res = nozzle_set_down(instance->nozzle_handle);
2211 if (res != 0) {
2212 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't set nozzle device down: %s", strerror(errno));
2213 return -1;
2214 }
2215 run_nozzle_script(instance, NOZZLE_POSTDOWN, "post-down");
2216
2217 res = nozzle_close(instance->nozzle_handle);
2218 if (res != 0) {
2219 knet_log_printf (LOGSYS_LEVEL_ERROR, "Can't close nozzle device: %s", strerror(errno));
2220 return -1;
2221 }
2222 knet_log_printf (LOGSYS_LEVEL_INFO, "Removed nozzle device");
2223 return 0;
2224}
2225
2226static void free_nozzle(struct totemknet_instance *instance)
2227{
2228 free(instance->nozzle_name);
2229 free(instance->nozzle_ipaddr);
2230 free(instance->nozzle_prefix);
2231 free(instance->nozzle_macaddr);
2232
2233 instance->nozzle_name = instance->nozzle_ipaddr = instance->nozzle_prefix =
2234 instance->nozzle_macaddr = NULL;
2235}
2236
2237static int setup_nozzle(void *knet_context)
2238{
2239 struct totemknet_instance *instance = (struct totemknet_instance *)knet_context;
2240 char *ipaddr_str = NULL;
2241 char *name_str = NULL;
2242 char *prefix_str = NULL;
2243 char *macaddr_str = NULL;
2244 char mac[32];
2245 int name_res;
2246 int macaddr_res;
2247 int res = -1;
2248
2249 /*
2250 * Return value ignored on purpose. icmap_get_string changes
2251 * ipaddr_str/prefix_str only on success.
2252 */
2253 (void)icmap_get_string(NOZZLE_IPADDR, &ipaddr_str);
2254 (void)icmap_get_string(NOZZLE_PREFIX, &prefix_str);
2255 macaddr_res = icmap_get_string(NOZZLE_MACADDR, &macaddr_str);
2256 name_res = icmap_get_string(NOZZLE_NAME, &name_str);
2257
2258 /* Is is being removed? */
2259 if (name_res == CS_ERR_NOT_EXIST && instance->nozzle_handle) {
2260 remove_nozzle_device(instance);
2261 free_nozzle(instance);
2262 goto out_free;
2263 }
2264
2265 if (!name_str) {
2266 /* no nozzle */
2267 goto out_free;
2268 }
2269
2270 if (!ipaddr_str) {
2271 knet_log_printf (LOGSYS_LEVEL_ERROR, "No IP address supplied for Nozzle device");
2272 goto out_free;
2273 }
2274
2275 if (!prefix_str) {
2276 knet_log_printf (LOGSYS_LEVEL_ERROR, "No prefix supplied for Nozzle IP address");
2277 goto out_free;
2278 }
2279
2280 if (macaddr_str && strlen(macaddr_str) != 17) {
2281 knet_log_printf (LOGSYS_LEVEL_ERROR, "macaddr for nozzle device is not in the correct format '%s'", macaddr_str);
2282 goto out_free;
2283 }
2284 if (!macaddr_str) {
2285 macaddr_str = (char*)"54:54:01:00:00:00";
2286 }
2287
2288 if (instance->nozzle_name &&
2289 (strcmp(name_str, instance->nozzle_name) == 0) &&
2290 (strcmp(ipaddr_str, instance->nozzle_ipaddr) == 0) &&
2291 (strcmp(prefix_str, instance->nozzle_prefix) == 0) &&
2292 (instance->nozzle_macaddr == NULL ||
2293 strcmp(macaddr_str, instance->nozzle_macaddr) == 0)) {
2294 /* Nothing has changed */
2295 knet_log_printf (LOGSYS_LEVEL_DEBUG, "Nozzle device info not changed");
2296 goto out_free;
2297 }
2298
2299 /* Add nodeid into MAC address */
2300 memcpy(mac, macaddr_str, 12);
2301 snprintf(mac+12, sizeof(mac) - 13, "%02x:%02x",
2302 instance->our_nodeid >> 8,
2303 instance->our_nodeid & 0xFF);
2304 knet_log_printf (LOGSYS_LEVEL_INFO, "Local nozzle MAC address is %s", mac);
2305
2306 if (name_res == CS_OK && name_str) {
2307 /* Reconfigure */
2308 if (instance->nozzle_name) {
2309 remove_nozzle_device(instance);
2310 free_nozzle(instance);
2311 }
2312
2313 res = create_nozzle_device(knet_context, name_str, ipaddr_str, prefix_str,
2314 mac);
2315
2316 instance->nozzle_name = strdup(name_str);
2317 instance->nozzle_ipaddr = strdup(ipaddr_str);
2318 instance->nozzle_prefix = strdup(prefix_str);
2319 instance->nozzle_macaddr = strdup(macaddr_str);
2320 if (!instance->nozzle_name || !instance->nozzle_ipaddr ||
2321 !instance->nozzle_prefix) {
2322 knet_log_printf (LOGSYS_LEVEL_ERROR, "strdup failed in nozzle allocation");
2323 /*
2324 * This 'free' will cause a complete reconfigure of the device next time we reload
2325 * but will also let the the current device keep working until then.
2326 * remove_nozzle() only needs the, statically-allocated, nozzle_handle
2327 */
2328 free_nozzle(instance);
2329 }
2330 }
2331
2332out_free:
2333 free(name_str);
2334 free(ipaddr_str);
2335 free(prefix_str);
2336 if (macaddr_res == CS_OK) {
2337 free(macaddr_str);
2338 }
2339
2340 return res;
2341}
2342#endif // HAVE_LIBNOZZLE
INTERFACE_MAX
#define INTERFACE_MAX
Definition coroapi.h:88
nodeid
unsigned int nodeid
Definition coroapi.h:0
addr
unsigned char addr[TOTEMIP_ADDRLEN]
Definition coroapi.h:2
PROCESSOR_COUNT_MAX
#define PROCESSOR_COUNT_MAX
Definition coroapi.h:96
CS_PRI_NODE_ID
#define CS_PRI_NODE_ID
Definition corotypes.h:59
qb_to_cs_error
cs_error_t qb_to_cs_error(int result)
qb_to_cs_error
cs_error_t
cs_error_t
The cs_error_t enum.
Definition corotypes.h:98
CS_ERR_BUSY
@ CS_ERR_BUSY
Definition corotypes.h:108
CS_ERR_TRY_AGAIN
@ CS_ERR_TRY_AGAIN
Definition corotypes.h:104
CS_OK
@ CS_OK
Definition corotypes.h:99
CS_ERR_INVALID_PARAM
@ CS_ERR_INVALID_PARAM
Definition corotypes.h:105
CS_ERR_LIBRARY
@ CS_ERR_LIBRARY
Definition corotypes.h:100
CS_ERR_NOT_EXIST
@ CS_ERR_NOT_EXIST
Definition corotypes.h:110
corosync_exit_error
#define corosync_exit_error(err)
Definition exec/util.h:72
COROSYNC_DONE_MAINCONFIGREAD
@ COROSYNC_DONE_MAINCONFIGREAD
Definition exec/util.h:51
value
uint32_t value
Definition exec/votequorum.c:2
icmap_get_uint8
cs_error_t icmap_get_uint8(const char *key_name, uint8_t *u8)
Definition icmap.c:872
ICMAP_TRACK_MODIFY
#define ICMAP_TRACK_MODIFY
Definition icmap.h:78
ICMAP_TRACK_DELETE
#define ICMAP_TRACK_DELETE
Definition icmap.h:77
icmap_track_add
cs_error_t icmap_track_add(const char *key_name, int32_t track_type, icmap_notify_fn_t notify_fn, void *user_data, icmap_track_t *icmap_track)
Add tracking function for given key_name.
Definition icmap.c:1163
icmap_set_string
cs_error_t icmap_set_string(const char *key_name, const char *value)
Definition icmap.c:631
ICMAP_TRACK_PREFIX
#define ICMAP_TRACK_PREFIX
Whole prefix is tracked, instead of key only (so "totem." tracking means that "totem....
Definition icmap.h:85
ICMAP_TRACK_ADD
#define ICMAP_TRACK_ADD
Definition icmap.h:76
icmap_get_string
cs_error_t icmap_get_string(const char *key_name, char **str)
Shortcut for icmap_get for string type.
Definition icmap.c:860
LOGSYS_LEVEL_ERROR
#define LOGSYS_LEVEL_ERROR
Definition logsys.h:72
LEAVE
#define LEAVE
Definition logsys.h:334
log_printf
#define log_printf(level, format, args...)
Definition logsys.h:332
LOGSYS_LEVEL_INFO
#define LOGSYS_LEVEL_INFO
Definition logsys.h:75
LOGSYS_LEVEL_CRIT
#define LOGSYS_LEVEL_CRIT
Definition logsys.h:71
logsys_config_debug_get
int logsys_config_debug_get(const char *subsys)
Return the debug flag for this subsys.
Definition logsys.c:806
LOGSYS_DEBUG_OFF
#define LOGSYS_DEBUG_OFF
Definition logsys.h:92
LOGSYS_DEBUG_TRACE
#define LOGSYS_DEBUG_TRACE
Definition logsys.h:94
LOGSYS_DEBUG_ON
#define LOGSYS_DEBUG_ON
Definition logsys.h:93
LOGSYS_LEVEL_WARNING
#define LOGSYS_LEVEL_WARNING
Definition logsys.h:73
_logsys_subsys_create
int _logsys_subsys_create(const char *subsys, const char *filename)
_logsys_subsys_create
Definition logsys.c:435
LOGSYS_LEVEL_DEBUG
#define LOGSYS_LEVEL_DEBUG
Definition logsys.h:76
LOGSYS_LEVEL_TRACE
#define LOGSYS_LEVEL_TRACE
Definition logsys.h:77
ENTER
#define ENTER
Definition logsys.h:333
corosync_get_config_file
const char * corosync_get_config_file(void)
Definition main.c:212
user_data
void * user_data
Definition sam.c:127
icmap_notify_value
Structure passed as new_value and old_value in change callback.
Definition icmap.h:91
icmap_track
Definition icmap.c:61
totem_config
Definition totem.h:159
totem_config::crypto_model
char crypto_model[CONFIG_STRING_LEN_MAX]
Definition totem.h:224
totem_config::private_key_len
unsigned int private_key_len
Definition totem.h:177
totem_config::knet_mtu
unsigned int knet_mtu
Definition totem.h:170
totem_config::node_id
unsigned int node_id
Definition totem.h:167
totem_config::knet_compression_threshold
uint32_t knet_compression_threshold
Definition totem.h:236
totem_config::totem_logging_configuration
struct totem_logging_configuration totem_logging_configuration
Definition totem.h:208
totem_config::interfaces
struct totem_interface * interfaces
Definition totem.h:165
totem_config::crypto_changed
int crypto_changed
Definition totem.h:232
totem_config::merge_timeout
unsigned int merge_timeout
Definition totem.h:198
totem_config::knet_compression_level
int knet_compression_level
Definition totem.h:238
totem_config::net_mtu
unsigned int net_mtu
Definition totem.h:210
totem_config::knet_compression_model
char knet_compression_model[CONFIG_STRING_LEN_MAX]
Definition totem.h:234
totem_config::block_unlisted_ips
unsigned int block_unlisted_ips
Definition totem.h:246
totem_config::private_key
unsigned char private_key[TOTEM_PRIVATE_KEY_LEN_MAX]
Definition totem.h:175
totem_config::crypto_index
int crypto_index
Definition totem.h:230
totem_config::knet_pmtud_interval
unsigned int knet_pmtud_interval
Definition totem.h:169
totem_config::crypto_cipher_type
char crypto_cipher_type[CONFIG_STRING_LEN_MAX]
Definition totem.h:226
totem_config::link_mode
char link_mode[TOTEM_LINK_MODE_BYTES]
Definition totem.h:206
totem_config::crypto_hash_type
char crypto_hash_type[CONFIG_STRING_LEN_MAX]
Definition totem.h:228
totem_interface::knet_ping_timeout
int knet_ping_timeout
Definition totem.h:93
totem_interface::knet_link_priority
int knet_link_priority
Definition totem.h:91
totem_interface::boundto
struct totem_ip_address boundto
Definition totem.h:84
totem_interface::ip_port
uint16_t ip_port
Definition totem.h:87
totem_interface::knet_ping_interval
int knet_ping_interval
Definition totem.h:92
totem_interface::configured
uint8_t configured
Definition totem.h:89
totem_interface::knet_ping_precision
int knet_ping_precision
Definition totem.h:94
totem_interface::knet_pong_count
int knet_pong_count
Definition totem.h:95
totem_interface::knet_transport
int knet_transport
Definition totem.h:96
totem_interface::bindnet
struct totem_ip_address bindnet
Definition totem.h:83
totem_ip_address
The totem_ip_address struct.
Definition coroapi.h:111
totem_ip_address::nodeid
unsigned int nodeid
Definition coroapi.h:112
totem_logging_configuration::log_subsys_id
int log_subsys_id
Definition totem.h:116
totem_logging_configuration::log_level_error
int log_level_error
Definition totem.h:111
totem_logging_configuration::log_level_notice
int log_level_notice
Definition totem.h:113
totem_logging_configuration::log_level_debug
int log_level_debug
Definition totem.h:114
totem_logging_configuration::log_level_warning
int log_level_warning
Definition totem.h:112
totem_logging_configuration::log_printf
void(* log_printf)(int level, int subsys, const char *function_name, const char *file_name, int file_line, const char *format,...) __attribute__((format(printf
Definition totem.h:101
totem_message_header
Definition totem.h:126
totem_message_header::target_nodeid
unsigned int target_nodeid
Definition totem.h:132
totem_node_status
Definition totem.h:265
totemknet_instance
Definition totemknet.c:93
totemknet_instance::totemknet_subsys_id
int totemknet_subsys_id
Definition totemknet.c:136
totemknet_instance::totemknet_target_set_completed
void(* totemknet_target_set_completed)(void *context)
Definition totemknet.c:121
totemknet_instance::log_mutex
pthread_mutex_t log_mutex
Definition totemknet.c:178
totemknet_instance::logpipes
int logpipes[2]
Definition totemknet.c:175
totemknet_instance::totemknet_mtu_changed
void(* totemknet_mtu_changed)(void *context, int net_mtu)
Definition totemknet.c:117
totemknet_instance::crypto_inst
struct crypto_instance * crypto_inst
Definition totemknet.c:94
totemknet_instance::totem_config
struct totem_config * totem_config
Definition totemknet.c:163
totemknet_instance::timer_netif_check_timeout
qb_loop_timer_handle timer_netif_check_timeout
Definition totemknet.c:167
totemknet_instance::knet_subsys_id
int knet_subsys_id
Definition totemknet.c:138
totemknet_instance::link_status
char * link_status[INTERFACE_MAX]
Definition totemknet.c:153
totemknet_instance::knet_fd
int knet_fd
Definition totemknet.c:176
totemknet_instance::totemknet_log_printf
void(* totemknet_log_printf)(int level, int subsys, const char *function, const char *file, int line, const char *format,...) __attribute__((format(printf
Definition totemknet.c:140
totemknet_instance::knet_handle
knet_handle_t knet_handle
Definition totemknet.c:100
totemknet_instance::ip_port
uint16_t ip_port[INTERFACE_MAX]
Definition totemknet.c:157
totemknet_instance::totemknet_iface_change_fn
int(* totemknet_iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no)
Definition totemknet.c:112
totemknet_instance::totemknet_log_level_debug
int totemknet_log_level_debug
Definition totemknet.c:134
totemknet_instance::last_good_crypto_cfg
struct knet_handle_crypto_cfg last_good_crypto_cfg
Definition totemknet.c:96
totemknet_instance::loopback_link
int loopback_link
Definition totemknet.c:161
totemknet_instance::token_target
struct totem_ip_address token_target
Definition totemknet.c:165
totemknet_instance::our_nodeid
int our_nodeid
Definition totemknet.c:159
totemknet_instance::timer_merge_detect_timeout
qb_loop_timer_handle timer_merge_detect_timeout
Definition totemknet.c:169
totemknet_instance::knet_context
void(*) void knet_context)
Definition totemknet.c:149
totemknet_instance::totemknet_log_level_warning
int totemknet_log_level_warning
Definition totemknet.c:130
totemknet_instance::my_ids
struct totem_ip_address my_ids[INTERFACE_MAX]
Definition totemknet.c:155
totemknet_instance::totemknet_deliver_fn
int(* totemknet_deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from)
Definition totemknet.c:106
totemknet_instance::iov_buffer
char iov_buffer[KNET_MAX_PACKET_SIZE]
Definition totemknet.c:151
totemknet_instance::merge_detect_messages_sent_before_timeout
unsigned int merge_detect_messages_sent_before_timeout
Definition totemknet.c:173
totemknet_instance::send_merge_detect_message
int send_merge_detect_message
Definition totemknet.c:171
totemknet_instance::totemknet_log_level_error
int totemknet_log_level_error
Definition totemknet.c:128
totemknet_instance::totemknet_log_level_security
int totemknet_log_level_security
Definition totemknet.c:126
totemknet_instance::poll_handle
qb_loop_t * poll_handle
Definition totemknet.c:98
totemknet_instance::link_mode
int link_mode
Definition totemknet.c:102
totemknet_instance::totemknet_log_level_notice
int totemknet_log_level_notice
Definition totemknet.c:132
totemknet_instance::context
void * context
Definition totemknet.c:104
totemsrp_stats_t
Definition totemstats.h:53
work_item
Definition totemknet.c:191
work_item::msg
const void * msg
Definition totemknet.c:192
work_item::msg_len
unsigned int msg_len
Definition totemknet.c:193
work_item::instance
struct totemknet_instance * instance
Definition totemknet.c:194
__attribute__
typedef __attribute__
cfg_message_crypto_reconfig_phase_t
cfg_message_crypto_reconfig_phase_t
Definition totem.h:154
CRYPTO_RECONFIG_PHASE_CLEANUP
@ CRYPTO_RECONFIG_PHASE_CLEANUP
Definition totem.h:156
CRYPTO_RECONFIG_PHASE_ACTIVATE
@ CRYPTO_RECONFIG_PHASE_ACTIVATE
Definition totem.h:155
type
char type
Definition totem.h:2
totemip_parse
int totemip_parse(struct totem_ip_address *totemip, const char *addr, enum totem_ip_version_enum ip_version)
Definition totemip.c:306
totemip_copy
void totemip_copy(struct totem_ip_address *addr1, const struct totem_ip_address *addr2)
Definition totemip.c:123
totemip_totemip_to_sockaddr_convert
int totemip_totemip_to_sockaddr_convert(struct totem_ip_address *ip_addr, uint16_t port, struct sockaddr_storage *saddr, int *addrlen)
Definition totemip.c:264
totemip_print
const char * totemip_print(const struct totem_ip_address *addr)
Definition totemip.c:256
totemknet_mcast_flush_send
int totemknet_mcast_flush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition totemknet.c:1451
totemknet_iface_set
int totemknet_iface_set(void *knet_context, const struct totem_ip_address *local_addr, unsigned short ip_port, unsigned int iface_no)
Definition totemknet.c:1561
totemknet_finalize
int totemknet_finalize(void *knet_context)
Definition totemknet.c:666
totemknet_recv_flush
int totemknet_recv_flush(void *knet_context)
Definition totemknet.c:1429
totemknet_member_list_rebind_ip
int totemknet_member_list_rebind_ip(void *knet_context)
Definition totemknet.c:1763
totemknet_buffer_alloc
void * totemknet_buffer_alloc(void)
Definition totemknet.c:1411
totemknet_processor_count_set
int totemknet_processor_count_set(void *knet_context, int processor_count)
Definition totemknet.c:1422
totemknet_mcast_noflush_send
int totemknet_mcast_noflush_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition totemknet.c:1464
global_instance
struct totemknet_instance * global_instance
Definition totemknet.c:189
totemknet_buffer_release
void totemknet_buffer_release(void *ptr)
Definition totemknet.c:1417
totemknet_initialize
int totemknet_initialize(qb_loop_t *poll_handle, void **knet_context, struct totem_config *totem_config, totemsrp_stats_t *stats, void *context, int(*deliver_fn)(void *context, const void *msg, unsigned int msg_len, const struct sockaddr_storage *system_from), int(*iface_change_fn)(void *context, const struct totem_ip_address *iface_address, unsigned int link_no), void(*mtu_changed)(void *context, int net_mtu), void(*target_set_completed)(void *context))
Definition totemknet.c:1152
totemknet_ifaces_get
int totemknet_ifaces_get(void *knet_context, char ***status, unsigned int *iface_count)
Definition totemknet.c:571
totemknet_member_add
int totemknet_member_add(void *knet_context, const struct totem_ip_address *local, const struct totem_ip_address *member, int link_no)
Definition totemknet.c:1578
totemknet_crypto_set
int totemknet_crypto_set(void *knet_context, const char *cipher_type, const char *hash_type)
Definition totemknet.c:365
totemknet_member_remove
int totemknet_member_remove(void *knet_context, const struct totem_ip_address *token_target, int link_no)
Definition totemknet.c:1717
totemknet_token_send
int totemknet_token_send(void *knet_context, const void *msg, unsigned int msg_len)
Definition totemknet.c:1439
CFG_INTERFACE_STATUS_MAX_LEN
#define CFG_INTERFACE_STATUS_MAX_LEN
Definition totemknet.c:91
totemknet_link_get_status
int totemknet_link_get_status(knet_node_id_t node, uint8_t link_no, struct knet_link_status *status)
Definition totemknet.c:1891
OWN_INDEX_NONE
#define OWN_INDEX_NONE
Definition totemknet.c:490
totemknet_configure_log_level
void totemknet_configure_log_level()
Definition totemknet.c:903
totemknet_nodestatus_get
int totemknet_nodestatus_get(void *knet_context, unsigned int nodeid, struct totem_node_status *node_status)
Definition totemknet.c:493
totemknet_handle_get_stats
int totemknet_handle_get_stats(struct knet_handle_stats *stats)
Definition totemknet.c:1928
MSG_NOSIGNAL
#define MSG_NOSIGNAL
Definition totemknet.c:83
totemknet_stats_clear
void totemknet_stats_clear(void *knet_context)
Definition totemknet.c:1882
totemknet_send_flush
int totemknet_send_flush(void *knet_context)
Definition totemknet.c:1434
totemknet_net_mtu_adjust
void totemknet_net_mtu_adjust(void *knet_context, struct totem_config *totem_config)
Definition totemknet.c:1488
knet_log_printf
#define knet_log_printf(level, format, args...)
Definition totemknet.c:235
totemknet_token_target_set
int totemknet_token_target_set(void *knet_context, unsigned int nodeid)
Definition totemknet.c:1495
KNET_LOGSYS_PERROR
#define KNET_LOGSYS_PERROR(err_num, level, fmt, args...)
Definition totemknet.c:251
totemknet_reconfigure
int totemknet_reconfigure(void *knet_context, struct totem_config *totem_config)
Definition totemknet.c:1790
totemknet_crypto_reconfigure_phase
int totemknet_crypto_reconfigure_phase(void *knet_context, struct totem_config *totem_config, cfg_message_crypto_reconfig_phase_t phase)
Definition totemknet.c:1823
libknet_log_printf
#define libknet_log_printf(level, format, args...)
Definition totemknet.c:243
totemknet_recv_mcast_empty
int totemknet_recv_mcast_empty(void *knet_context)
Definition totemknet.c:1509
totemknet_iface_check
int totemknet_iface_check(void *knet_context)
Definition totemknet.c:1478
header
struct totem_message_header header
Definition totemsrp.c:0
system_from
struct srp_addr system_from
Definition totemsrp.c:1
stats_knet_add_handle
void stats_knet_add_handle(void)
Definition stats.c:753
stats_knet_del_member
void stats_knet_del_member(knet_node_id_t nodeid, uint8_t link)
Definition stats.c:740
stats_knet_add_member
void stats_knet_add_member(knet_node_id_t nodeid, uint8_t link)
Definition stats.c:730
Generated on Sat Jun 21 2025 09:54:36 for corosync by doxygen 1.9.8