1 /*
2 * Copyright (c) 2015-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /* File: kern/mach_node.h
29 * Author: Dean Reece
30 * Date: 2016
31 *
32 * Implementation of mach node support.
33 * This is the basis for flipc, which provides inter-node communication.
34 */
35
36
37 #include <mach/mach_types.h>
38 #include <mach/boolean.h>
39 #include <mach/kern_return.h>
40
41 #include <kern/kern_types.h>
42 #include <kern/assert.h>
43
44 #include <kern/host.h>
45 #include <kern/kalloc.h>
46 #include <kern/mach_node_link.h>
47 #include <kern/mach_node.h>
48 #include <kern/ipc_mig.h> // mach_msg_send_from_kernel_proper()
49
50 #include <ipc/port.h>
51 #include <ipc/ipc_types.h>
52 #include <ipc/ipc_init.h>
53 #include <ipc/ipc_kmsg.h>
54 #include <ipc/ipc_port.h>
55 #include <ipc/ipc_pset.h>
56 #include <ipc/ipc_table.h>
57 #include <ipc/ipc_entry.h>
58
59 #include <ipc/flipc.h>
60
61 #include <libkern/OSAtomic.h> // OSAddAtomic64(), OSCompareAndSwap()
62 #include <libkern/OSByteOrder.h> // OSHostByteOrder()
63
64 #pragma pack(4)
65
66 #define MNL_NAME_TABLE_SIZE (256) // Hash is evenly distributed, so ^2 is ok
67 #define MNL_NAME_HASH(name) (name % MNL_NAME_TABLE_SIZE)
68
69 /*** Visible outside mach_node layer ***/
70 mach_node_id_t localnode_id = -1; // This node's FLIPC id.
71 #if MACH_FLIPC
72 mach_node_t localnode; // This node's mach_node_t struct
73
74
75 /*** Private to mach_node layer ***/
76 static int mach_nodes_to_publish;
77 static mach_node_t mach_node_table[MACH_NODES_MAX];
78 static LCK_SPIN_DECLARE_ATTR(mach_node_table_lock_data,
79 &ipc_lck_grp, &ipc_lck_attr);
80 #define MACH_NODE_TABLE_LOCK() lck_spin_lock(&mach_node_table_lock_data)
81 #define MACH_NODE_TABLE_UNLOCK() lck_spin_unlock(&mach_node_table_lock_data)
82
83 static volatile SInt64 mnl_name_next;
84 static queue_head_t mnl_name_table[MNL_NAME_TABLE_SIZE];
85 static LCK_SPIN_DECLARE_ATTR(mnl_name_table_lock_data,
86 &ipc_lck_grp, &ipc_lck_attr);
87 #define MNL_NAME_TABLE_LOCK() lck_spin_lock(&mnl_name_table_lock_data)
88 #define MNL_NAME_TABLE_UNLOCK() lck_spin_unlock(&mnl_name_table_lock_data)
89
90 static void mach_node_init(void);
91 static void mnl_name_table_init(void);
92 static void mach_node_table_init(void);
93 static void mach_node_publish(mach_node_t node);
94
95 static mach_node_t mach_node_alloc_init(mach_node_id_t node_id);
96 static kern_return_t mach_node_register(mach_node_t node);
97
98
99 /* mach_node_init() is run lazily when a node link driver registers
100 * or the node special port is set.
101 * The variable localnode_id is used to determine if init has already run.
102 */
103 void
mach_node_init(void)104 mach_node_init(void)
105 {
106 mach_node_id_t node_id = 0; // TODO: Read from device tree?
107 if (OSCompareAndSwap((UInt32)(HOST_LOCAL_NODE),
108 (UInt32)node_id,
109 &localnode_id)) {
110 printf("mach_node_init(): localnode_id=%d of %d\n",
111 localnode_id, MACH_NODES_MAX);
112 mach_node_table_init();
113 mnl_name_table_init();
114 } // TODO: else block until init is finished (init completion race)
115 }
116
117 void
mach_node_table_init(void)118 mach_node_table_init(void)
119 {
120 MACH_NODE_TABLE_LOCK();
121
122 /* Start with an enpty node table. */
123 bzero(mach_node_table, sizeof(mach_node_t) * MACH_NODES_MAX);
124 mach_nodes_to_publish = 0;
125
126 /* Allocate localnode's struct */
127 localnode = mach_node_for_id_locked(localnode_id, 1, 1);
128 assert(MACH_NODE_VALID(localnode));
129
130 MACH_NODE_TABLE_UNLOCK();
131
132 /* Set up localnode's struct */
133 bzero(localnode, sizeof(*localnode));
134 localnode->info.datamodel = LOCAL_DATA_MODEL;
135 localnode->info.byteorder = OSHostByteOrder();
136 localnode->info.proto_vers_min = MNL_PROTOCOL_V1;
137 localnode->info.proto_vers_max = MNL_PROTOCOL_V1;
138 localnode->proto_vers = MNL_PROTOCOL_V1;
139 localnode->published = 0;
140 localnode->active = 1;
141
142 MACH_NODE_UNLOCK(localnode);
143 }
144
145 /* Sends a publication message to the local node's bootstrap server.
146 * This function is smart and will only send a notification if one as really
147 * needed - it can be called speculatively on any node at any time.
148 *
149 * Note: MUST be called with the node table lock held.
150 */
151
152 void
mach_node_publish(mach_node_t node)153 mach_node_publish(mach_node_t node)
154 {
155 kern_return_t kr;
156
157 if (!MACH_NODE_VALID(node) || (!node->active) || (node->published)) {
158 return; // node is invalid or not suitable for publication
159 }
160 ipc_port_t bs_port = localnode->bootstrap_port;
161 if (!IP_VALID(bs_port)) {
162 return; // No bootstrap server to notify!
163 }
164 /* Node is suitable and server is present, so make registration message */
165 struct mach_node_server_register_msg msg;
166
167 msg.node_header.header.msgh_remote_port = bs_port;
168 msg.node_header.header.msgh_size = sizeof(msg);
169 msg.node_header.header.msgh_local_port = MACH_PORT_NULL;
170 msg.node_header.header.msgh_voucher_port = MACH_PORT_NULL;
171 msg.node_header.header.msgh_id = MACH_NODE_SERVER_MSG_ID;
172 msg.node_header.node_id = node->info.node_id;
173 msg.node_header.options = 0;
174 msg.datamodel = node->info.datamodel;
175 msg.byteorder = node->info.byteorder;
176
177 if (node == localnode) {
178 msg.node_header.identifier = MACH_NODE_SM_REG_LOCAL;
179 msg.node_header.header.msgh_bits =
180 MACH_MSGH_BITS_SET(MACH_MSG_TYPE_COPY_SEND, 0, 0, 0);
181 } else {
182 msg.node_header.identifier = MACH_NODE_SM_REG_REMOTE;
183 msg.node_header.header.msgh_local_port = node->bootstrap_port;
184 msg.node_header.header.msgh_bits = MACH_MSGH_BITS_SET
185 (MACH_MSG_TYPE_COPY_SEND, MACH_MSG_TYPE_MAKE_SEND, 0, 0);
186 }
187
188 kr = mach_msg_send_from_kernel_proper(&msg.node_header.header,
189 sizeof(msg));
190 if (kr == KERN_SUCCESS) {
191 node->published = 1;
192 mach_nodes_to_publish--;
193 }
194 printf("mach_node_publish(%d)=%d\n", node->info.node_id, kr);
195 }
196
197 /* Called whenever the node special port changes */
198 void
mach_node_port_changed(void)199 mach_node_port_changed(void)
200 {
201 ipc_port_t bs_port;
202
203 mach_node_init(); // Lazy init of mach_node layer
204
205 /* Cleanup previous bootstrap port if necessary */
206 MACH_NODE_LOCK(localnode);
207 flipc_node_retire(localnode);
208 bs_port = localnode->bootstrap_port;
209 if (IP_VALID(bs_port)) {
210 localnode->bootstrap_port = IP_NULL;
211 // TODO: destroy send right to outgoing bs_port
212 }
213
214 kernel_get_special_port(host_priv_self(), HOST_NODE_PORT, &bs_port);
215 assert(IP_VALID(bs_port));
216 localnode->bootstrap_port = bs_port;
217 flipc_node_prepare(localnode);
218 MACH_NODE_UNLOCK(localnode);
219
220 /* Cleanup the publication state of all nodes in the table */
221 MACH_NODE_TABLE_LOCK();
222 // TODO: Signup for bootstrap port death notifications
223 localnode->active = 1;
224
225 mach_nodes_to_publish = 0;
226
227 int n;
228 for (n = 0; n < MACH_NODES_MAX; n++) {
229 mach_node_t np = mach_node_table[n];
230 // Publish all active nodes (except the local node)
231 if (!MACH_NODE_VALID(np)) {
232 continue;
233 }
234 np->published = 0;
235 if (np->active == 1) {
236 mach_nodes_to_publish++;
237 }
238 }
239
240 mach_node_publish(localnode); // Always publish local node first
241
242 for (n = 0; n < MACH_NODES_MAX; n++) {
243 mach_node_publish(mach_node_table[n]);
244 }
245
246 MACH_NODE_TABLE_UNLOCK();
247
248 // TODO: notify all active nodes we are bootstrapped
249 }
250
251 /* Allocate/init a mach_node struct and fill in the node_id field.
252 * This does NOT insert the node struct into the node table.
253 */
254 mach_node_t
mach_node_alloc_init(mach_node_id_t node_id)255 mach_node_alloc_init(mach_node_id_t node_id)
256 {
257 mach_node_t node = MACH_NODE_ALLOC();
258 if (MACH_NODE_VALID(node)) {
259 bzero(node, sizeof(struct mach_node));
260 MACH_NODE_LOCK_INIT(node);
261 node->info.node_id = node_id;
262 }
263 return node;
264 }
265
266
267 /* This function takes a mach_node struct with a completed info field and
268 * registers it with the mach_node and flipc (if flipc is enabled) layers.
269 */
270 kern_return_t
mach_node_register(mach_node_t node)271 mach_node_register(mach_node_t node)
272 {
273 assert(MACH_NODE_VALID(node));
274 mach_node_id_t nid = node->info.node_id;
275 assert(MACH_NODE_ID_VALID(nid));
276
277 kern_return_t kr;
278 ipc_space_t proxy_space = IS_NULL;
279 ipc_pset_t pp_set = IPS_NULL; // pset for proxy ports
280 ipc_port_t bs_port = MACH_PORT_NULL;
281 ipc_port_t ack_port = MACH_PORT_NULL;
282
283 printf("mach_node_register(%d)\n", nid);
284
285 /* TODO: Support non-native byte order and data models */
286 if ((node->info.byteorder != OSHostByteOrder()) ||
287 (node->info.datamodel != LOCAL_DATA_MODEL)) {
288 printf("mach_node_register: unsupported byte order (%d) or width (%d)",
289 node->info.byteorder, node->info.datamodel);
290 return KERN_INVALID_ARGUMENT;
291 }
292
293 /* Create the space that holds all local rights assigned to <nid> */
294 kr = ipc_space_create_special(&proxy_space);
295 if (kr != KERN_SUCCESS) {
296 goto out;
297 }
298 proxy_space->is_node_id = nid;
299
300 /* Create the bootstrap proxy port for this remote node */
301 bs_port = ipc_port_alloc_special(proxy_space, IPC_PORT_INIT_MESSAGE_QUEUE);
302 if (bs_port == MACH_PORT_NULL) {
303 kr = KERN_RESOURCE_SHORTAGE;
304 goto out;
305 }
306
307 /* Create the control (ack) port for this remote node */
308 ack_port = ipc_port_alloc_special(proxy_space, IPC_PORT_INIT_MESSAGE_QUEUE);
309 if (ack_port == MACH_PORT_NULL) {
310 kr = KERN_RESOURCE_SHORTAGE;
311 goto out;
312 }
313
314 /* Create the set that holds all proxy ports for this remote node */
315 pp_set = ipc_pset_alloc_special(proxy_space);
316 if (pp_set == IPS_NULL) {
317 kr = KERN_RESOURCE_SHORTAGE;
318 goto out;
319 }
320
321 waitq_set_lazy_init_link(&pp_set->ips_wqset);
322 /* Add the bootstrap port to the proxy port set */
323 uint64_t wq_link_id = waitq_link_reserve(NULL);
324 uint64_t wq_reserved_prepost = waitq_prepost_reserve(NULL, 10,
325 WAITQ_DONT_LOCK);
326 ips_mq_lock(pp_set); // Revisit the lock when enabling flipc
327 ip_mq_lock(bs_port);
328 ipc_pset_add_unlock(pp_set,
329 bs_port,
330 &wq_link_id,
331 &wq_reserved_prepost);
332 ips_mq_unlock(pp_set);
333
334 waitq_link_release(wq_link_id);
335 waitq_prepost_release_reserve(wq_reserved_prepost);
336
337 /* Add the control port to the proxy port set */
338 wq_link_id = waitq_link_reserve(NULL);
339 wq_reserved_prepost = waitq_prepost_reserve(NULL, 10,
340 WAITQ_DONT_LOCK);
341 ips_mq_lock(pp_set); // Revisit the lock when enabling flipc
342 ip_mq_lock(ack_port);
343 ipc_pset_add_unlock(pp_set,
344 ack_port,
345 &wq_link_id,
346 &wq_reserved_prepost);
347 ips_mq_unlock(pp_set);
348
349 waitq_link_release(wq_link_id);
350 waitq_prepost_release_reserve(wq_reserved_prepost);
351
352 // Setup mach_node struct
353 node->published = 0;
354 node->active = 1;
355 node->proxy_space = proxy_space;
356 node->proxy_port_set = pp_set;
357 node->bootstrap_port = bs_port;
358 node->proto_vers = node->info.proto_vers_max;
359 node->control_port = ack_port;
360
361 // Place new mach_node struct into node table
362 MACH_NODE_TABLE_LOCK();
363
364 mach_node_t old_node = mach_node_table[nid];
365 if (!MACH_NODE_VALID(old_node) || (old_node->dead)) {
366 node->antecedent = old_node;
367 flipc_node_prepare(node);
368 mach_node_table[nid] = node;
369 mach_nodes_to_publish++;
370 mach_node_publish(node);
371 kr = KERN_SUCCESS;
372 } else {
373 printf("mach_node_register: id %d already active!", nid);
374 kr = KERN_FAILURE;
375 }
376 MACH_NODE_TABLE_UNLOCK();
377
378 out:
379 if (kr != KERN_SUCCESS) { // Dispose of whatever we allocated
380 if (pp_set) {
381 ips_mq_lock(pp_set);
382 ipc_pset_destroy(proxy_space, pp_set);
383 }
384
385 if (bs_port) {
386 ipc_port_dealloc_special(bs_port, proxy_space);
387 }
388
389 if (ack_port) {
390 ipc_port_dealloc_special(ack_port, proxy_space);
391 }
392
393 if (proxy_space) {
394 ipc_space_terminate(proxy_space);
395 }
396 }
397
398 return kr;
399 }
400
401
402 /* Gets or allocates a locked mach_node struct for the specified <node_id>.
403 * The current node is locked and returned if it is not dead, or if it is dead
404 * and <alloc_if_dead> is false. A new node struct is allocated, locked and
405 * returned if the node is dead and <alloc_if_dead> is true, or if the node
406 * is absent and <alloc_if_absent> is true. MACH_NODE_NULL is returned if
407 * the node is absent and <alloc_if_absent> is false. MACH_NODE_NULL is also
408 * returned if a new node structure was not able to be allocated.
409 *
410 * Note: This function must be called with the node table lock held!
411 */
412 mach_node_t
mach_node_for_id_locked(mach_node_id_t node_id,boolean_t alloc_if_dead,boolean_t alloc_if_absent)413 mach_node_for_id_locked(mach_node_id_t node_id,
414 boolean_t alloc_if_dead,
415 boolean_t alloc_if_absent)
416 {
417 if ((node_id < 0) || (node_id >= MACH_NODES_MAX)) {
418 return MACH_NODE_NULL;
419 }
420
421 mach_node_t node = mach_node_table[node_id];
422
423 if ((!MACH_NODE_VALID(node) && alloc_if_absent) ||
424 (MACH_NODE_VALID(node) && node->dead && alloc_if_dead)) {
425 node = mach_node_alloc_init(node_id);
426 if (MACH_NODE_VALID(node)) {
427 node->antecedent = mach_node_table[node_id];
428 mach_node_table[node_id] = node;
429 }
430 }
431
432 if (MACH_NODE_VALID(node)) {
433 MACH_NODE_LOCK(node);
434 }
435
436 return node;
437 }
438
439
440
441 /*** Mach Node Link Name and Hash Table Implementation ***/
442
443 /* Allocate a new unique name and return it.
444 * Dispose of this with mnl_name_free().
445 * Returns MNL_NAME_NULL on failure.
446 */
447 mnl_name_t
mnl_name_alloc(void)448 mnl_name_alloc(void)
449 {
450 return (mnl_name_t)OSAddAtomic64(MACH_NODES_MAX, &mnl_name_next);
451 }
452
453
454 /* Deallocate a unique name that was allocated via mnl_name_alloc().
455 */
456 void
mnl_name_free(mnl_name_t name __unused)457 mnl_name_free(mnl_name_t name __unused)
458 {
459 ; // Nothing to do for now since we don't recycle mnl names.
460 }
461
462
463 /* Called once from mach_node_init(), this sets up the hash table structures.
464 */
465 void
mnl_name_table_init(void)466 mnl_name_table_init(void)
467 {
468 MNL_NAME_TABLE_LOCK();
469
470 // Set the first name to this node's bootstrap name
471 mnl_name_next = localnode_id + MACH_NODES_MAX;
472
473 for (int i = 0; i < MNL_NAME_TABLE_SIZE; i++) {
474 queue_head_init(mnl_name_table[i]);
475 }
476
477 MNL_NAME_TABLE_UNLOCK();
478 }
479
480
481 /* Initialize the data structures in the mnl_obj structure at the head of the
482 * provided object. This should be called on an object before it is passed to
483 * any other mnl_obj* routine.
484 */
485 void
mnl_obj_init(mnl_obj_t obj)486 mnl_obj_init(mnl_obj_t obj)
487 {
488 queue_chain_init(obj->links);
489 obj->name = MNL_NAME_NULL;
490 }
491
492
493 /* Search the local node's hash table for the object associated with a
494 * mnl_name_t and return it. Returns MNL_NAME_NULL on failure.
495 */
496 mnl_obj_t
mnl_obj_lookup(mnl_name_t name)497 mnl_obj_lookup(mnl_name_t name)
498 {
499 mnl_obj_t obj = MNL_OBJ_NULL;
500
501 if (name != MNL_NAME_NULL) {
502 qe_foreach_element(obj, &mnl_name_table[MNL_NAME_HASH(name)], links) {
503 if (obj->name == name) {
504 break;
505 }
506 }
507 }
508 return obj;
509 }
510
511
512 /* Search the local node's hash table for the object associated with a
513 * mnl_name_t and remove it. The pointer to the removed object is returned so
514 * that the caller can appropriately dispose of the object.
515 * Returns MNL_NAME_NULL on failure.
516 */
517 mnl_obj_t
mnl_obj_remove(mnl_name_t name)518 mnl_obj_remove(mnl_name_t name)
519 {
520 mnl_obj_t obj = MNL_OBJ_NULL;
521
522 if (name != MNL_NAME_NULL) {
523 qe_foreach_element_safe(obj, &mnl_name_table[MNL_NAME_HASH(name)], links) {
524 if (obj->name == name) {
525 remqueue(&obj->links);
526 }
527 }
528 }
529 return obj;
530 }
531
532
533 /* Insert an object into the local node's hash table. If the name of the
534 * provided object is MNL_NAME_NULL then a new mnl_name is allocated and
535 * assigned to the object.
536 * Returns KERN_SUCCESS if obj was added to hash table
537 * Returns KERN_INVALID_ARGUMENT if obj is invalid
538 * Returns KERN_NAME_EXISTS if obj's name already exists in hash table
539 */
540 kern_return_t
mnl_obj_insert(mnl_obj_t obj)541 mnl_obj_insert(mnl_obj_t obj)
542 {
543 if (!MNL_OBJ_VALID(obj)) {
544 return KERN_INVALID_ARGUMENT;
545 }
546
547 MNL_NAME_TABLE_LOCK();
548
549 if (!MNL_NAME_VALID(obj->name)) {
550 // obj is unnammed, so lets allocate a fresh one
551 obj->name = mnl_name_alloc();
552 }
553
554 enqueue(&mnl_name_table[MNL_NAME_HASH(obj->name)], &obj->links);
555 MNL_NAME_TABLE_UNLOCK();
556
557 if (obj->name >= (MACH_NODES_MAX << 1)) {
558 panic("Unexpected MNL_NAME %lld in obj %p", obj->name, obj);
559 }
560
561 return KERN_SUCCESS;
562 }
563
564
565 /*** Mach Node Link Driver Interface Implementation ***/
566
567 /* Allocate a mnl_msg struct plus additional payload. Link drivers are not
568 * required to use this to allocate messages; any wired and mapped kernel
569 * memory is acceptable.
570 *
571 * Arguments:
572 * payload Number of additional bytes to allocate for message payload
573 * flags Currently unused; 0 should be passed
574 *
575 * Return values:
576 * MNL_MSG_NULL: Allocation failed
577 * *: Pointer to new mnl_msg struct of requested size
578 */
579 mnl_msg_t
mnl_msg_alloc(int payload,uint32_t flags __unused)580 mnl_msg_alloc(int payload,
581 uint32_t flags __unused)
582 {
583 mnl_msg_t msg = kalloc(MNL_MSG_SIZE + payload);
584
585 if (MNL_MSG_VALID(msg)) {
586 bzero(msg, MNL_MSG_SIZE); // Only zero the header
587 msg->size = payload;
588 }
589
590 return msg;
591 }
592
593
594 /* Free a mnl_msg struct allocated by mnl_msg_alloc().
595 *
596 * Arguments:
597 * msg Pointer to the message buffer to be freed
598 * flags Currently unused; 0 should be passed
599 */
600 void
mnl_msg_free(mnl_msg_t msg,uint32_t flags __unused)601 mnl_msg_free(mnl_msg_t msg,
602 uint32_t flags __unused)
603 {
604 if (MNL_MSG_VALID(msg)) {
605 kfree(msg, MNL_MSG_SIZE + msg->size);
606 }
607 }
608
609
610 /* The link driver calls this to setup a new (or restarted) node, and to get
611 * an mnl_node_info struct for use as a parameter to other mnl functions.
612 * If MNL_NODE_NULL is returned, the operation failed. Otherwise, a pointer
613 * to a new mnl_node struct is returned. The caller should set all fields
614 * in the structure, then call mnl_register() to complete node registration.
615 *
616 * Arguments:
617 * nid The id of the node to be instantiated
618 * flags Currently unused; 0 should be passed
619 *
620 * Return values:
621 * MNL_NODE_NULL: Operation failed
622 * *: Pointer to a new mnl_node struct
623 */
624 mnl_node_info_t
mnl_instantiate(mach_node_id_t nid,uint32_t flags __unused)625 mnl_instantiate(mach_node_id_t nid,
626 uint32_t flags __unused)
627 {
628 mach_node_init(); // Lazy init of mach_node layer
629
630 if ((nid == localnode_id) || !MACH_NODE_ID_VALID(nid)) {
631 return MNL_NODE_NULL;
632 }
633
634 return (mnl_node_info_t)mach_node_alloc_init(nid);
635 }
636
637 /* The link driver calls mnl_register() to complete the node registration
638 * process. KERN_SUCCESS is returned if registration succeeded, otherwise
639 * an error is returned.
640 *
641 * Arguments:
642 * node Pointer to the node's mnl_node structure
643 * flags Currently unused; 0 should be passed
644 *
645 * Return values:
646 * KERN_SUCCESS: Registration succeeded
647 * KERN_INVALID_ARGUMENT: Field(s) in <node> contained unacceptable values
648 * KERN_*: Values returned from underlying functions
649 */
650 kern_return_t
mnl_register(mnl_node_info_t node,uint32_t flags __unused)651 mnl_register(mnl_node_info_t node,
652 uint32_t flags __unused)
653 {
654 if (MNL_NODE_VALID(node) && (node->node_id != localnode_id)) {
655 return mach_node_register((mach_node_t)node);
656 }
657
658 return KERN_INVALID_ARGUMENT;
659 }
660
661
662 /* The link driver calls this to report that the link has been raised in one
663 * or both directions. If the link is two uni-directional channels, each link
664 * driver will independently call this function, each only raising the link
665 * they are responsible for. The mach_node layer will not communicate with
666 * the remote node until both rx and tx links are up.
667 *
668 * Arguments:
669 * node Pointer to the node's mnl_node structure
670 * link Indicates which link(s) are up (see MNL_LINK_* defines)
671 * flags Currently unused; 0 should be passed
672 *
673 * Return values:
674 * KERN_SUCCESS: Link state changed successfully.
675 * KERN_INVALID_ARGUMENT: An argument value was not allowed.
676 * KERN_*: Values returned from underlying functions.
677 */
678 kern_return_t
mnl_set_link_state(mnl_node_info_t node,int link,uint32_t flags __unused)679 mnl_set_link_state(mnl_node_info_t node,
680 int link,
681 uint32_t flags __unused)
682 {
683 kern_return_t kr;
684 mach_node_t mnode = (mach_node_t)node;
685
686 if (!MACH_NODE_VALID(mnode) || !(link & MNL_LINK_UP) || (link & mnode->link)) {
687 return KERN_INVALID_ARGUMENT; // bad node, or bad link argument
688 }
689 MACH_NODE_LOCK(mnode);
690
691 if (mnode->dead) {
692 kr = KERN_NODE_DOWN;
693 } else {
694 mnode->link |= link;
695 kr = KERN_SUCCESS;
696 }
697
698 MACH_NODE_UNLOCK(mnode);
699
700 return kr;
701 }
702
703 /* The link driver calls this to indicate a node has terminated and is no
704 * longer available for messaging. This may be due to a crash or an orderly
705 * shutdown, but either way the remote node no longer retains any state about
706 * the remaining nodes. References held on behalf of the terminated node
707 * will be cleaned up. After this is called, both the rx and tx links are
708 * marked as down. If the remote node restarts, the link driver can bring
709 * up the link using mnl_instantiate() again.
710 *
711 * Arguments:
712 * node Pointer to the node's mnl_node structure
713 * flags Currently unused; 0 should be passed
714 *
715 * Return values:
716 * KERN_SUCCESS: Node was terminated.
717 * KERN_INVALID_ARGUMENT: Node id was invalid or non-existant.
718 * KERN_*: Values returned from underlying functions.
719 */
720 kern_return_t
mnl_terminate(mnl_node_info_t node,uint32_t flags __unused)721 mnl_terminate(mnl_node_info_t node,
722 uint32_t flags __unused)
723 {
724 kern_return_t kr = KERN_SUCCESS;
725 mach_node_t mnode = (mach_node_t)node;
726
727 if (!MACH_NODE_VALID(mnode)) {
728 return KERN_INVALID_ARGUMENT; // bad node
729 }
730 MACH_NODE_LOCK(mnode);
731 if (mnode->dead) {
732 kr = KERN_NODE_DOWN; // node is already terminated
733 goto unlock;
734 }
735
736 mnode->link = MNL_LINK_DOWN;
737 mnode->active = 0;
738 mnode->suspended = 0;
739 mnode->dead = 1;
740
741 flipc_node_retire(mnode);
742
743 // Wake any threads sleeping on the proxy port set
744 if (mnode->proxy_port_set != IPS_NULL) {
745 ips_mq_lock(mnode->proxy_port_set);
746 ipc_pset_destroy(mnode->proxy_space, mnode->proxy_port_set);
747 mnode->proxy_port_set = IPS_NULL;
748 }
749
750 // TODO: Inform node name server (if registered) of termination
751
752 unlock:
753 MACH_NODE_UNLOCK(mnode);
754 return kr;
755 }
756
757
758 /* The link driver calls this to deliver an incoming message. Note that the
759 * link driver must dispose of the memory pointed to by <msg> after the
760 * function call returns.
761 *
762 * Arguments:
763 * node Pointer to the node's mnl_node structure
764 * msg Pointer to the message buffer
765 * flags Currently unused; 0 should be passed
766 */
767 void
mnl_msg_from_node(mnl_node_info_t node __unused,mnl_msg_t msg,uint32_t flags __unused)768 mnl_msg_from_node(mnl_node_info_t node __unused,
769 mnl_msg_t msg,
770 uint32_t flags __unused)
771 {
772 assert(MNL_MSG_VALID(msg));
773 assert(MACH_NODE_ID_VALID(msg->node_id));
774 assert(MNL_NODE_VALID(node));
775
776 /* If node message forwarding is supported, the from_node_id arg may not
777 * match fmsg->info.node_id. The former is the node from which we received
778 * the message; the latter is the node that generated the message originally.
779 * We always use fmsg->info.node_id, which is where the ack needs to go.
780 */
781
782 switch (msg->sub) {
783 case MACH_NODE_SUB_FLIPC:
784 flipc_msg_from_node((mach_node_t)node, msg, flags);
785 break;
786
787 default:
788 #if DEBUG
789 PE_enter_debugger("mnl_msg_from_node(): Invalid subsystem");
790 #endif
791 break;
792 }
793 }
794
795
796 /* The link driver calls this to fetch the next message to transmit.
797 * This function will block until a message is available, or will return
798 * FLIPC_MSG_NULL if the link is to be terminated. After the caller has
799 * completed the transmission and no longer needs the msg buffer, it should
800 * call mnl_msg_complete().
801 *
802 * Arguments:
803 * node Pointer to the node's mnl_node structure
804 * flags Currently unused; 0 should be passed
805 */
806 mnl_msg_t
mnl_msg_to_node(mnl_node_info_t node __unused,uint32_t flags __unused)807 mnl_msg_to_node(mnl_node_info_t node __unused,
808 uint32_t flags __unused)
809 {
810 assert(MNL_NODE_VALID(node));
811
812 #if DEBUG
813 thread_set_thread_name(current_thread(), "MNL_Link");
814 #endif
815
816 return flipc_msg_to_remote_node((mach_node_t)node, 0);
817 }
818
819
820 /* The link driver calls this to indicate that the specified msg buffer has
821 * been sent over the link and can be deallocated.
822 *
823 * Arguments:
824 * node Pointer to the node's mnl_node structure
825 * msg Pointer to the message buffer
826 * flags Currently unused; 0 should be passed
827 */
828 void
mnl_msg_complete(mnl_node_info_t node __unused,mnl_msg_t msg,uint32_t flags)829 mnl_msg_complete(mnl_node_info_t node __unused,
830 mnl_msg_t msg,
831 uint32_t flags)
832 {
833 switch (msg->sub) {
834 case MACH_NODE_SUB_NODE:
835 mnl_msg_free(msg, flags);
836 break;
837
838 case MACH_NODE_SUB_FLIPC:
839 flipc_msg_free(msg, flags);
840 break;
841
842 default:
843 #if DEBUG
844 PE_enter_debugger("mnl_msg_complete(): Invalid subsystem");
845 #endif
846 break;
847 }
848 }
849
850 #else // MACH_FLIPC not configured, so provide KPI stubs
851
852 mnl_msg_t
mnl_msg_alloc(int payload __unused,uint32_t flags __unused)853 mnl_msg_alloc(int payload __unused, uint32_t flags __unused)
854 {
855 return MNL_MSG_NULL;
856 }
857
858 void
mnl_msg_free(mnl_msg_t msg __unused,uint32_t flags __unused)859 mnl_msg_free(mnl_msg_t msg __unused, uint32_t flags __unused)
860 {
861 return;
862 }
863
864 mnl_node_info_t
mnl_instantiate(mach_node_id_t nid __unused,uint32_t flags __unused)865 mnl_instantiate(mach_node_id_t nid __unused, uint32_t flags __unused)
866 {
867 return MNL_NODE_NULL;
868 }
869
870 kern_return_t
mnl_register(mnl_node_info_t node __unused,uint32_t flags __unused)871 mnl_register(mnl_node_info_t node __unused, uint32_t flags __unused)
872 {
873 return KERN_FAILURE;
874 }
875
876 kern_return_t
mnl_set_link_state(mnl_node_info_t node __unused,int link __unused,uint32_t flags __unused)877 mnl_set_link_state(mnl_node_info_t node __unused,
878 int link __unused,
879 uint32_t flags __unused)
880 {
881 return KERN_FAILURE;
882 }
883
884 kern_return_t
mnl_terminate(mnl_node_info_t node __unused,uint32_t flags __unused)885 mnl_terminate(mnl_node_info_t node __unused, uint32_t flags __unused)
886 {
887 return KERN_FAILURE;
888 }
889
890 void
mnl_msg_from_node(mnl_node_info_t node __unused,mnl_msg_t msg __unused,uint32_t flags __unused)891 mnl_msg_from_node(mnl_node_info_t node __unused,
892 mnl_msg_t msg __unused,
893 uint32_t flags __unused)
894 {
895 return;
896 }
897
898 mnl_msg_t
mnl_msg_to_node(mnl_node_info_t node __unused,uint32_t flags __unused)899 mnl_msg_to_node(mnl_node_info_t node __unused, uint32_t flags __unused)
900 {
901 return MNL_MSG_NULL;
902 }
903
904 void
mnl_msg_complete(mnl_node_info_t node __unused,mnl_msg_t msg __unused,uint32_t flags __unused)905 mnl_msg_complete(mnl_node_info_t node __unused,
906 mnl_msg_t msg __unused,
907 uint32_t flags __unused)
908 {
909 return;
910 }
911
912 #endif // MACH_FLIPC
913