1 /*
2 * Copyright (c) 2000-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <sys/errno.h>
30
31 #include <mach/mach_types.h>
32 #include <mach/kern_return.h>
33 #include <mach/memory_object_control.h>
34 #include <mach/memory_object_types.h>
35 #include <mach/port.h>
36 #include <mach/policy.h>
37 #include <mach/upl.h>
38 #include <kern/kern_types.h>
39 #include <kern/ipc_kobject.h>
40 #include <kern/host.h>
41 #include <kern/thread.h>
42 #include <ipc/ipc_port.h>
43 #include <ipc/ipc_space.h>
44 #include <device/device_port.h>
45 #include <vm/memory_object.h>
46 #include <vm/vm_pageout.h>
47 #include <vm/vm_map.h>
48 #include <vm/vm_kern.h>
49 #include <vm/vm_pageout.h>
50 #include <vm/vm_protos.h>
51 #include <mach/sdt.h>
52 #include <os/refcnt.h>
53
54
55 /* Device VM COMPONENT INTERFACES */
56
57
58 /*
59 * Device PAGER
60 */
61
62
63 /* until component support available */
64
65
66
67 /* until component support available */
68 const struct memory_object_pager_ops device_pager_ops = {
69 .memory_object_reference = device_pager_reference,
70 .memory_object_deallocate = device_pager_deallocate,
71 .memory_object_init = device_pager_init,
72 .memory_object_terminate = device_pager_terminate,
73 .memory_object_data_request = device_pager_data_request,
74 .memory_object_data_return = device_pager_data_return,
75 .memory_object_data_initialize = device_pager_data_initialize,
76 .memory_object_data_unlock = device_pager_data_unlock,
77 .memory_object_synchronize = device_pager_synchronize,
78 .memory_object_map = device_pager_map,
79 .memory_object_last_unmap = device_pager_last_unmap,
80 .memory_object_data_reclaim = NULL,
81 .memory_object_backing_object = NULL,
82 .memory_object_pager_name = "device pager"
83 };
84
85 typedef uintptr_t device_port_t;
86
87 /*
88 * The start of "struct device_pager" MUST match a "struct memory_object".
89 */
90 typedef struct device_pager {
91 /* mandatory generic header */
92 struct memory_object dev_pgr_hdr;
93
94 /* pager-specific data */
95 lck_mtx_t lock;
96 device_port_t device_handle; /* device_handle */
97 vm_size_t size;
98 #if MEMORY_OBJECT_HAS_REFCOUNT
99 #define dev_pgr_hdr_ref dev_pgr_hdr.mo_ref
100 #else
101 os_ref_atomic_t dev_pgr_hdr_ref;
102 #endif
103 int flags;
104 boolean_t is_mapped;
105 } *device_pager_t;
106
107 __header_always_inline os_ref_count_t
device_pager_get_refcount(device_pager_t device_object)108 device_pager_get_refcount(device_pager_t device_object)
109 {
110 return os_ref_get_count_raw(&device_object->dev_pgr_hdr_ref);
111 }
112
113 LCK_GRP_DECLARE(device_pager_lck_grp, "device_pager");
114
115 ZONE_DECLARE(device_pager_zone, "device node pager structures",
116 sizeof(struct device_pager), ZC_NONE);
117
118 #define device_pager_lock_init(pager) \
119 lck_mtx_init(&(pager)->lock, &device_pager_lck_grp, LCK_ATTR_NULL)
120 #define device_pager_lock_destroy(pager) \
121 lck_mtx_destroy(&(pager)->lock, &device_pager_lck_grp)
122 #define device_pager_lock(pager) lck_mtx_lock(&(pager)->lock)
123 #define device_pager_unlock(pager) lck_mtx_unlock(&(pager)->lock)
124
125 device_pager_t
126 device_pager_lookup( /* forward */
127 memory_object_t);
128
129 device_pager_t
130 device_object_create(void); /* forward */
131
132 #define DEVICE_PAGER_NULL ((device_pager_t) 0)
133
134 #define MAX_DNODE 10000
135
136
137 /*
138 *
139 */
140 memory_object_t
device_pager_setup(__unused memory_object_t device,uintptr_t device_handle,vm_size_t size,int flags)141 device_pager_setup(
142 __unused memory_object_t device,
143 uintptr_t device_handle,
144 vm_size_t size,
145 int flags)
146 {
147 device_pager_t device_object;
148 memory_object_control_t control;
149 vm_object_t object;
150
151 device_object = device_object_create();
152 if (device_object == DEVICE_PAGER_NULL) {
153 panic("device_pager_setup: device_object_create() failed");
154 }
155
156 device_object->device_handle = device_handle;
157 device_object->size = size;
158 device_object->flags = flags;
159
160 memory_object_create_named((memory_object_t) device_object,
161 size,
162 &control);
163 object = memory_object_control_to_vm_object(control);
164
165 memory_object_mark_trusted(control);
166
167 assert(object != VM_OBJECT_NULL);
168 vm_object_lock(object);
169 object->true_share = TRUE;
170 if (object->copy_strategy == MEMORY_OBJECT_COPY_SYMMETRIC) {
171 object->copy_strategy = MEMORY_OBJECT_COPY_DELAY;
172 }
173 vm_object_unlock(object);
174
175 return (memory_object_t)device_object;
176 }
177
178 /*
179 *
180 */
181 kern_return_t
device_pager_populate_object(memory_object_t device,memory_object_offset_t offset,ppnum_t page_num,vm_size_t size)182 device_pager_populate_object(
183 memory_object_t device,
184 memory_object_offset_t offset,
185 ppnum_t page_num,
186 vm_size_t size)
187 {
188 device_pager_t device_object;
189 vm_object_t vm_object;
190 kern_return_t kr;
191 upl_t upl;
192
193 device_object = device_pager_lookup(device);
194 if (device_object == DEVICE_PAGER_NULL) {
195 return KERN_FAILURE;
196 }
197
198 vm_object = (vm_object_t)memory_object_control_to_vm_object(
199 device_object->dev_pgr_hdr.mo_control);
200 if (vm_object == NULL) {
201 return KERN_FAILURE;
202 }
203
204 kr = vm_object_populate_with_private(
205 vm_object, offset, page_num, size);
206 if (kr != KERN_SUCCESS) {
207 return kr;
208 }
209
210 if (!vm_object->phys_contiguous) {
211 unsigned int null_size = 0;
212 assert((upl_size_t) size == size);
213 kr = vm_object_upl_request(vm_object,
214 (vm_object_offset_t)offset,
215 (upl_size_t) size, &upl, NULL,
216 &null_size,
217 (UPL_NO_SYNC | UPL_CLEAN_IN_PLACE),
218 VM_KERN_MEMORY_NONE);
219 if (kr != KERN_SUCCESS) {
220 panic("device_pager_populate_object: list_req failed");
221 }
222
223 upl_commit(upl, NULL, 0);
224 upl_deallocate(upl);
225 }
226
227
228 return kr;
229 }
230
231 /*
232 *
233 */
234 device_pager_t
device_pager_lookup(memory_object_t mem_obj)235 device_pager_lookup(
236 memory_object_t mem_obj)
237 {
238 device_pager_t device_object;
239
240 assert(mem_obj->mo_pager_ops == &device_pager_ops);
241 device_object = (device_pager_t)mem_obj;
242 assert(device_pager_get_refcount(device_object) > 0);
243 return device_object;
244 }
245
246 /*
247 *
248 */
249 kern_return_t
device_pager_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pg_size)250 device_pager_init(
251 memory_object_t mem_obj,
252 memory_object_control_t control,
253 __unused memory_object_cluster_size_t pg_size)
254 {
255 device_pager_t device_object;
256 kern_return_t kr;
257 memory_object_attr_info_data_t attributes;
258
259 vm_object_t vm_object;
260
261
262 if (control == MEMORY_OBJECT_CONTROL_NULL) {
263 return KERN_INVALID_ARGUMENT;
264 }
265
266 device_object = device_pager_lookup(mem_obj);
267
268 memory_object_control_reference(control);
269 device_object->dev_pgr_hdr.mo_control = control;
270
271
272 /* The following settings should be done through an expanded change */
273 /* attributes call */
274
275 vm_object = (vm_object_t)memory_object_control_to_vm_object(control);
276 vm_object_lock(vm_object);
277 vm_object->private = TRUE;
278 if (device_object->flags & DEVICE_PAGER_CONTIGUOUS) {
279 vm_object->phys_contiguous = TRUE;
280 }
281 if (device_object->flags & DEVICE_PAGER_NOPHYSCACHE) {
282 vm_object->nophyscache = TRUE;
283 }
284
285 vm_object->wimg_bits = device_object->flags & VM_WIMG_MASK;
286 vm_object_unlock(vm_object);
287
288
289 attributes.copy_strategy = MEMORY_OBJECT_COPY_DELAY;
290 /* attributes.cluster_size = (1 << (CLUSTER_SHIFT + PAGE_SHIFT));*/
291 attributes.cluster_size = (1 << (PAGE_SHIFT));
292 attributes.may_cache_object = FALSE;
293 attributes.temporary = TRUE;
294
295 kr = memory_object_change_attributes(
296 control,
297 MEMORY_OBJECT_ATTRIBUTE_INFO,
298 (memory_object_info_t) &attributes,
299 MEMORY_OBJECT_ATTR_INFO_COUNT);
300 if (kr != KERN_SUCCESS) {
301 panic("device_pager_init: memory_object_change_attributes() failed");
302 }
303
304 return KERN_SUCCESS;
305 }
306
307 /*
308 *
309 */
310 /*ARGSUSED6*/
311 kern_return_t
device_pager_data_return(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t data_cnt,__unused memory_object_offset_t * resid_offset,__unused int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,__unused int upl_flags)312 device_pager_data_return(
313 memory_object_t mem_obj,
314 memory_object_offset_t offset,
315 memory_object_cluster_size_t data_cnt,
316 __unused memory_object_offset_t *resid_offset,
317 __unused int *io_error,
318 __unused boolean_t dirty,
319 __unused boolean_t kernel_copy,
320 __unused int upl_flags)
321 {
322 device_pager_t device_object;
323
324 device_object = device_pager_lookup(mem_obj);
325 if (device_object == DEVICE_PAGER_NULL) {
326 panic("device_pager_data_return: lookup failed");
327 }
328
329 __IGNORE_WCASTALIGN(return device_data_action(device_object->device_handle,
330 (ipc_port_t) device_object,
331 VM_PROT_READ | VM_PROT_WRITE,
332 offset, data_cnt));
333 }
334
335 /*
336 *
337 */
338 kern_return_t
device_pager_data_request(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t length,__unused vm_prot_t protection_required,__unused memory_object_fault_info_t fault_info)339 device_pager_data_request(
340 memory_object_t mem_obj,
341 memory_object_offset_t offset,
342 memory_object_cluster_size_t length,
343 __unused vm_prot_t protection_required,
344 __unused memory_object_fault_info_t fault_info)
345 {
346 device_pager_t device_object;
347
348 device_object = device_pager_lookup(mem_obj);
349
350 if (device_object == DEVICE_PAGER_NULL) {
351 panic("device_pager_data_request: lookup failed");
352 }
353
354 __IGNORE_WCASTALIGN(device_data_action(device_object->device_handle,
355 (ipc_port_t) device_object,
356 VM_PROT_READ, offset, length));
357 return KERN_SUCCESS;
358 }
359
360 /*
361 *
362 */
363 void
device_pager_reference(memory_object_t mem_obj)364 device_pager_reference(
365 memory_object_t mem_obj)
366 {
367 device_pager_t device_object;
368
369 device_object = device_pager_lookup(mem_obj);
370 os_ref_retain_raw(&device_object->dev_pgr_hdr_ref, NULL);
371 DTRACE_VM2(device_pager_reference,
372 device_pager_t, device_object,
373 unsigned int, device_pager_get_refcount(device_object));
374 }
375
376 /*
377 *
378 */
379 void
device_pager_deallocate(memory_object_t mem_obj)380 device_pager_deallocate(
381 memory_object_t mem_obj)
382 {
383 device_pager_t device_object;
384 memory_object_control_t device_control;
385 os_ref_count_t ref_count;
386
387 device_object = device_pager_lookup(mem_obj);
388
389 DTRACE_VM2(device_pager_deallocate,
390 device_pager_t, device_object,
391 unsigned int, device_pager_get_refcount(device_object));
392
393 ref_count = os_ref_release_raw(&device_object->dev_pgr_hdr_ref, NULL);
394
395 if (ref_count == 1) {
396 /*
397 * The last reference is our "named" reference.
398 * Close the device and "destroy" the VM object.
399 */
400
401 DTRACE_VM2(device_pager_destroy,
402 device_pager_t, device_object,
403 unsigned int, device_pager_get_refcount(device_object));
404
405 assert(device_object->is_mapped == FALSE);
406 if (device_object->device_handle != (device_port_t) NULL) {
407 device_close(device_object->device_handle);
408 device_object->device_handle = (device_port_t) NULL;
409 }
410 device_control = device_object->dev_pgr_hdr.mo_control;
411 memory_object_destroy(device_control, 0);
412 } else if (ref_count == 0) {
413 /*
414 * No more references: free the pager.
415 */
416 DTRACE_VM2(device_pager_free,
417 device_pager_t, device_object,
418 unsigned int, device_pager_get_refcount(device_object));
419
420 device_control = device_object->dev_pgr_hdr.mo_control;
421
422 if (device_control != MEMORY_OBJECT_CONTROL_NULL) {
423 memory_object_control_deallocate(device_control);
424 device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
425 }
426 device_pager_lock_destroy(device_object);
427
428 zfree(device_pager_zone, device_object);
429 }
430 return;
431 }
432
433 kern_return_t
device_pager_data_initialize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t data_cnt)434 device_pager_data_initialize(
435 __unused memory_object_t mem_obj,
436 __unused memory_object_offset_t offset,
437 __unused memory_object_cluster_size_t data_cnt)
438 {
439 panic("device_pager_data_initialize");
440 return KERN_FAILURE;
441 }
442
443 kern_return_t
device_pager_data_unlock(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_size_t size,__unused vm_prot_t desired_access)444 device_pager_data_unlock(
445 __unused memory_object_t mem_obj,
446 __unused memory_object_offset_t offset,
447 __unused memory_object_size_t size,
448 __unused vm_prot_t desired_access)
449 {
450 return KERN_FAILURE;
451 }
452
453 kern_return_t
device_pager_terminate(__unused memory_object_t mem_obj)454 device_pager_terminate(
455 __unused memory_object_t mem_obj)
456 {
457 return KERN_SUCCESS;
458 }
459
460
461
462 /*
463 *
464 */
465 kern_return_t
device_pager_synchronize(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_size_t length,__unused vm_sync_t sync_flags)466 device_pager_synchronize(
467 __unused memory_object_t mem_obj,
468 __unused memory_object_offset_t offset,
469 __unused memory_object_size_t length,
470 __unused vm_sync_t sync_flags)
471 {
472 panic("device_pager_synchronize: memory_object_synchronize no longer supported");
473 return KERN_FAILURE;
474 }
475
476 /*
477 *
478 */
479 kern_return_t
device_pager_map(memory_object_t mem_obj,__unused vm_prot_t prot)480 device_pager_map(
481 memory_object_t mem_obj,
482 __unused vm_prot_t prot)
483 {
484 device_pager_t device_object;
485
486 device_object = device_pager_lookup(mem_obj);
487
488 device_pager_lock(device_object);
489 assert(device_pager_get_refcount(device_object) > 0);
490 if (device_object->is_mapped == FALSE) {
491 /*
492 * First mapping of this pager: take an extra reference
493 * that will remain until all the mappings of this pager
494 * are removed.
495 */
496 device_object->is_mapped = TRUE;
497 device_pager_reference(mem_obj);
498 }
499 device_pager_unlock(device_object);
500
501 return KERN_SUCCESS;
502 }
503
504 kern_return_t
device_pager_last_unmap(memory_object_t mem_obj)505 device_pager_last_unmap(
506 memory_object_t mem_obj)
507 {
508 device_pager_t device_object;
509 boolean_t drop_ref;
510
511 device_object = device_pager_lookup(mem_obj);
512
513 device_pager_lock(device_object);
514 assert(device_pager_get_refcount(device_object) > 0);
515 if (device_object->is_mapped) {
516 device_object->is_mapped = FALSE;
517 drop_ref = TRUE;
518 } else {
519 drop_ref = FALSE;
520 }
521 device_pager_unlock(device_object);
522
523 if (drop_ref) {
524 device_pager_deallocate(mem_obj);
525 }
526
527 return KERN_SUCCESS;
528 }
529
530
531
532 /*
533 *
534 */
535 device_pager_t
device_object_create(void)536 device_object_create(void)
537 {
538 device_pager_t device_object;
539
540 device_object = zalloc_flags(device_pager_zone,
541 Z_WAITOK | Z_ZERO | Z_NOFAIL);
542
543 device_object->dev_pgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
544 device_object->dev_pgr_hdr.mo_pager_ops = &device_pager_ops;
545 device_object->dev_pgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
546
547 device_pager_lock_init(device_object);
548 os_ref_init_raw(&device_object->dev_pgr_hdr_ref, NULL);
549 device_object->is_mapped = FALSE;
550
551 DTRACE_VM2(device_pager_create,
552 device_pager_t, device_object,
553 unsigned int, device_pager_get_refcount(device_object));
554
555 return device_object;
556 }
557
558 boolean_t
is_device_pager_ops(const struct memory_object_pager_ops * pager_ops)559 is_device_pager_ops(const struct memory_object_pager_ops *pager_ops)
560 {
561 if (pager_ops == &device_pager_ops) {
562 return TRUE;
563 }
564 return FALSE;
565 }
566