1 /*
2 * Copyright (c) 2011-2014 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <nfs/nfs_conf.h>
30 #if CONFIG_NFS_SERVER
31
32 #include <stdint.h>
33 #include <sys/param.h>
34 #include <sys/mount_internal.h>
35 #include <sys/malloc.h>
36 #include <sys/queue.h>
37
38 #include <libkern/libkern.h>
39 #include <libkern/OSAtomic.h>
40 #include <kern/debug.h>
41 #include <kern/thread.h>
42
43 #include <nfs/rpcv2.h>
44 #include <nfs/nfsproto.h>
45 #include <nfs/nfs.h>
46
47 #ifdef NFS_UC_DEBUG
48 #define DPRINT(fmt, ...) printf(fmt,## __VA_ARGS__)
49 #else
50 #define DPRINT(fmt, ...)
51 #endif
52
53 struct nfsrv_uc_arg {
54 TAILQ_ENTRY(nfsrv_uc_arg) nua_svcq;
55 socket_t nua_so;
56 struct nfsrv_sock *nua_slp;
57 int nua_waitflag; /* Should always be MBUF_DONTWAIT */
58 uint32_t nua_flags;
59 uint32_t nua_qi;
60 };
61 #define NFS_UC_QUEUED 0x0001
62
63 #define NFS_UC_HASH_SZ 7
64 #define NFS_UC_HASH(x) ((((uint32_t)(uintptr_t)(x)) >> 3) % nfsrv_uc_thread_count)
65
66 TAILQ_HEAD(nfsrv_uc_q, nfsrv_uc_arg);
67
68 static struct nfsrv_uc_queue {
69 lck_mtx_t ucq_lock;
70 struct nfsrv_uc_q ucq_queue[1];
71 thread_t ucq_thd;
72 uint32_t ucq_flags;
73 } nfsrv_uc_queue_tbl[NFS_UC_HASH_SZ];
74 #define NFS_UC_QUEUE_SLEEPING 0x0001
75
76 static LCK_GRP_DECLARE(nfsrv_uc_group, "nfs_upcall_locks");
77 static LCK_MTX_DECLARE(nfsrv_uc_shutdown_lock, &nfsrv_uc_group);
78 static volatile int nfsrv_uc_shutdown = 0;
79 static int32_t nfsrv_uc_thread_count;
80
81 extern kern_return_t thread_terminate(thread_t);
82
83 #ifdef NFS_UC_Q_DEBUG
84 int nfsrv_uc_use_proxy = 1;
85 uint32_t nfsrv_uc_queue_limit;
86 uint32_t nfsrv_uc_queue_max_seen;
87 volatile uint32_t nfsrv_uc_queue_count;
88 #endif
89
90 /*
91 * Thread that dequeues up-calls and runs the nfsrv_rcv routine
92 */
93 static void
nfsrv_uc_thread(void * arg,wait_result_t wr __unused)94 nfsrv_uc_thread(void *arg, wait_result_t wr __unused)
95 {
96 int qi = (int)(uintptr_t)arg;
97 int error;
98 struct nfsrv_uc_arg *ep = NULL;
99 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
100
101 DPRINT("nfsrv_uc_thread %d started\n", qi);
102 while (!nfsrv_uc_shutdown) {
103 lck_mtx_lock(&myqueue->ucq_lock);
104
105 while (!nfsrv_uc_shutdown && TAILQ_EMPTY(myqueue->ucq_queue)) {
106 myqueue->ucq_flags |= NFS_UC_QUEUE_SLEEPING;
107 error = msleep(myqueue, &myqueue->ucq_lock, PSOCK, "nfsd_upcall_handler", NULL);
108 myqueue->ucq_flags &= ~NFS_UC_QUEUE_SLEEPING;
109 if (error) {
110 printf("nfsrv_uc_thread received error %d\n", error);
111 }
112 }
113 if (nfsrv_uc_shutdown) {
114 lck_mtx_unlock(&myqueue->ucq_lock);
115 break;
116 }
117
118
119 ep = TAILQ_FIRST(myqueue->ucq_queue);
120 DPRINT("nfsrv_uc_thread:%d dequeue %p from %p\n", qi, ep, myqueue);
121
122 TAILQ_REMOVE(myqueue->ucq_queue, ep, nua_svcq);
123
124 ep->nua_flags &= ~NFS_UC_QUEUED;
125
126 lck_mtx_unlock(&myqueue->ucq_lock);
127
128 #ifdef NFS_UC_Q_DEBUG
129 OSDecrementAtomic(&nfsrv_uc_queue_count);
130 #endif
131
132 DPRINT("calling nfsrv_rcv for %p\n", (void *)ep->nua_slp);
133 nfsrv_rcv(ep->nua_so, (void *)ep->nua_slp, ep->nua_waitflag);
134 }
135
136 lck_mtx_lock(&nfsrv_uc_shutdown_lock);
137 nfsrv_uc_thread_count--;
138 wakeup(&nfsrv_uc_thread_count);
139 lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
140
141 thread_terminate(current_thread());
142 }
143
144 /*
145 * Dequeue a closed nfsrv_sock if needed from the up-call queue.
146 * Call from nfsrv_zapsock
147 */
148 void
nfsrv_uc_dequeue(struct nfsrv_sock * slp)149 nfsrv_uc_dequeue(struct nfsrv_sock *slp)
150 {
151 struct nfsrv_uc_arg *ap = slp->ns_ua;
152 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[ap->nua_qi];
153
154 /*
155 * We assume that the socket up-calls have been stop and the socket
156 * is shutting down so no need for acquiring the lock to check that
157 * the flag is cleared.
158 */
159 if (ap == NULL || (ap->nua_flags & NFS_UC_QUEUED) == 0) {
160 return;
161 }
162 /* If we're queued we might race with nfsrv_uc_thread */
163 lck_mtx_lock(&myqueue->ucq_lock);
164 if (ap->nua_flags & NFS_UC_QUEUED) {
165 printf("nfsrv_uc_dequeue remove %p\n", ap);
166 TAILQ_REMOVE(myqueue->ucq_queue, ap, nua_svcq);
167 ap->nua_flags &= ~NFS_UC_QUEUED;
168 #ifdef NFS_UC_Q_DEBUG
169 OSDecrementAtomic(&nfsrv_uc_queue_count);
170 #endif
171 }
172 kfree_type(struct nfsrv_uc_arg, slp->ns_ua);
173 slp->ns_ua = NULL;
174 lck_mtx_unlock(&myqueue->ucq_lock);
175 }
176
177 /*
178 * Allocate and initialize globals for nfsrv_sock up-call support.
179 */
180 void
nfsrv_uc_init(void)181 nfsrv_uc_init(void)
182 {
183 for (int i = 0; i < NFS_UC_HASH_SZ; i++) {
184 TAILQ_INIT(nfsrv_uc_queue_tbl[i].ucq_queue);
185 lck_mtx_init(&nfsrv_uc_queue_tbl[i].ucq_lock, &nfsrv_uc_group, LCK_ATTR_NULL);
186 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
187 nfsrv_uc_queue_tbl[i].ucq_flags = 0;
188 }
189 }
190
191 /*
192 * Start up-call threads to service nfsrv_sock(s)
193 * Called from the first call of nfsrv_uc_addsock
194 */
195 static void
nfsrv_uc_start(void)196 nfsrv_uc_start(void)
197 {
198 int32_t i;
199 int error;
200
201 #ifdef NFS_UC_Q_DEBUG
202 if (!nfsrv_uc_use_proxy) {
203 return;
204 }
205 #endif
206 DPRINT("nfsrv_uc_start\n");
207
208 /* Wait until previous shutdown finishes */
209 lck_mtx_lock(&nfsrv_uc_shutdown_lock);
210 while (nfsrv_uc_shutdown || nfsrv_uc_thread_count > 0) {
211 msleep(&nfsrv_uc_thread_count, &nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_wait", NULL);
212 }
213
214 /* Start up-call threads */
215 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
216 error = kernel_thread_start(nfsrv_uc_thread, (void *)(uintptr_t)i, &nfsrv_uc_queue_tbl[nfsrv_uc_thread_count].ucq_thd);
217 if (!error) {
218 nfsrv_uc_thread_count++;
219 } else {
220 printf("nfsd: Could not start nfsrv_uc_thread: %d\n", error);
221 }
222 }
223 if (nfsrv_uc_thread_count == 0) {
224 printf("nfsd: Could not start nfsd proxy up-call service. Falling back\n");
225 goto out;
226 }
227
228 out:
229 #ifdef NFS_UC_Q_DEBUG
230 nfsrv_uc_queue_count = 0ULL;
231 nfsrv_uc_queue_max_seen = 0ULL;
232 #endif
233 lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
234 }
235
236 /*
237 * Stop the up-call threads.
238 * Called from nfsrv_uc_cleanup.
239 */
240 static void
nfsrv_uc_stop(void)241 nfsrv_uc_stop(void)
242 {
243 int32_t i;
244 int32_t thread_count = nfsrv_uc_thread_count;
245
246 DPRINT("Entering nfsrv_uc_stop\n");
247
248 /* Signal up-call threads to stop */
249 nfsrv_uc_shutdown = 1;
250 for (i = 0; i < thread_count; i++) {
251 lck_mtx_lock(&nfsrv_uc_queue_tbl[i].ucq_lock);
252 wakeup(&nfsrv_uc_queue_tbl[i]);
253 lck_mtx_unlock(&nfsrv_uc_queue_tbl[i].ucq_lock);
254 }
255
256 /* Wait until they are done shutting down */
257 lck_mtx_lock(&nfsrv_uc_shutdown_lock);
258 while (nfsrv_uc_thread_count > 0) {
259 msleep(&nfsrv_uc_thread_count, &nfsrv_uc_shutdown_lock, PSOCK, "nfsd_upcall_shutdown_stop", NULL);
260 }
261
262 /* Deallocate old threads */
263 for (i = 0; i < nfsrv_uc_thread_count; i++) {
264 if (nfsrv_uc_queue_tbl[i].ucq_thd != THREAD_NULL) {
265 thread_deallocate(nfsrv_uc_queue_tbl[i].ucq_thd);
266 }
267 nfsrv_uc_queue_tbl[i].ucq_thd = THREAD_NULL;
268 }
269
270 /* Enable restarting */
271 nfsrv_uc_shutdown = 0;
272 lck_mtx_unlock(&nfsrv_uc_shutdown_lock);
273 }
274
275 /*
276 * Shutdown up-calls for nfsrv_socks.
277 * Make sure nothing is queued on the up-call queues
278 * Shutdown the up-call threads
279 * Called from nfssvc_cleanup.
280 */
281 void
nfsrv_uc_cleanup(void)282 nfsrv_uc_cleanup(void)
283 {
284 int i;
285
286 DPRINT("Entering nfsrv_uc_cleanup\n");
287
288 /*
289 * Every thing should be dequeued at this point or will be as sockets are closed
290 * but to be safe, we'll make sure.
291 */
292 for (i = 0; i < NFS_UC_HASH_SZ; i++) {
293 struct nfsrv_uc_queue *queue = &nfsrv_uc_queue_tbl[i];
294
295 lck_mtx_lock(&queue->ucq_lock);
296 while (!TAILQ_EMPTY(queue->ucq_queue)) {
297 struct nfsrv_uc_arg *ep = TAILQ_FIRST(queue->ucq_queue);
298 TAILQ_REMOVE(queue->ucq_queue, ep, nua_svcq);
299 ep->nua_flags &= ~NFS_UC_QUEUED;
300 }
301 lck_mtx_unlock(&queue->ucq_lock);
302 }
303
304 nfsrv_uc_stop();
305 }
306
307 /*
308 * This is the nfs up-call routine for server sockets.
309 * We used to set nfsrv_rcv as the up-call routine, but
310 * recently that seems like we are doing to much work for
311 * the interface thread, so we just queue the arguments
312 * that we would have gotten for nfsrv_rcv and let a
313 * worker thread dequeue them and pass them on to nfsrv_rcv.
314 */
315 static void
nfsrv_uc_proxy(socket_t so,void * arg,int waitflag)316 nfsrv_uc_proxy(socket_t so, void *arg, int waitflag)
317 {
318 struct nfsrv_uc_arg *uap = (struct nfsrv_uc_arg *)arg;
319 int qi = uap->nua_qi;
320 struct nfsrv_uc_queue *myqueue = &nfsrv_uc_queue_tbl[qi];
321
322 lck_mtx_lock(&myqueue->ucq_lock);
323 DPRINT("nfsrv_uc_proxy called for %p (%p)\n", uap, uap->nua_slp);
324 DPRINT("\tUp-call queued on %d for wakeup of %p\n", qi, myqueue);
325 if (uap == NULL || uap->nua_flags & NFS_UC_QUEUED) {
326 lck_mtx_unlock(&myqueue->ucq_lock);
327 return; /* Already queued or freed */
328 }
329
330 uap->nua_so = so;
331 uap->nua_waitflag = waitflag;
332
333 TAILQ_INSERT_TAIL(myqueue->ucq_queue, uap, nua_svcq);
334
335 uap->nua_flags |= NFS_UC_QUEUED;
336 if (myqueue->ucq_flags & NFS_UC_QUEUE_SLEEPING) {
337 wakeup(myqueue);
338 }
339
340 #ifdef NFS_UC_Q_DEBUG
341 {
342 uint32_t count = OSIncrementAtomic(&nfsrv_uc_queue_count);
343
344 /* This is a bit racey but just for debug */
345 if (count > nfsrv_uc_queue_max_seen) {
346 nfsrv_uc_queue_max_seen = count;
347 }
348
349 if (nfsrv_uc_queue_limit && count > nfsrv_uc_queue_limit) {
350 panic("nfsd up-call queue limit exceeded");
351 }
352 }
353 #endif
354 lck_mtx_unlock(&myqueue->ucq_lock);
355 }
356
357
358 /*
359 * Set the up-call routine on the socket associated with the passed in
360 * nfsrv_sock.
361 * Assumes nfsd_mutex is held.
362 */
363 void
nfsrv_uc_addsock(struct nfsrv_sock * slp,int start)364 nfsrv_uc_addsock(struct nfsrv_sock *slp, int start)
365 {
366 int on = 1;
367 struct nfsrv_uc_arg *arg;
368
369 if (start && nfsrv_uc_thread_count == 0) {
370 nfsrv_uc_start();
371 }
372
373 /*
374 * We don't take a lock since once we're up nfsrv_uc_thread_count does
375 * not change until shutdown and then we should not be adding sockets to
376 * generate up-calls.
377 */
378 if (nfsrv_uc_thread_count) {
379 arg = kalloc_type(struct nfsrv_uc_arg,
380 Z_WAITOK | Z_ZERO | Z_NOFAIL);
381
382 slp->ns_ua = arg;
383 arg->nua_slp = slp;
384 arg->nua_qi = NFS_UC_HASH(slp);
385
386 sock_setupcall(slp->ns_so, nfsrv_uc_proxy, arg);
387 } else {
388 slp->ns_ua = NULL;
389 DPRINT("setting nfsrv_rcv up-call\n");
390 sock_setupcall(slp->ns_so, nfsrv_rcv, slp);
391 }
392
393 /* just playin' it safe */
394 sock_setsockopt(slp->ns_so, SOL_SOCKET, SO_UPCALLCLOSEWAIT, &on, sizeof(on));
395
396 return;
397 }
398
399 #endif /* CONFIG_NFS_SERVER */
400