1 /*
2 * Copyright (c) 2024 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #include <vm/vm_upl.h>
30 #include <vm/vm_pageout_internal.h>
31 #include <vm/vm_page_internal.h>
32 #include <vm/vm_map_internal.h>
33 #if HAS_MTE
34 #include <vm/vm_mteinfo_internal.h>
35 #endif /* HAS_MTE */
36 #include <mach/upl_server.h>
37 #include <kern/host_statistics.h>
38 #include <vm/vm_purgeable_internal.h>
39 #include <vm/vm_object_internal.h>
40 #include <vm/vm_ubc.h>
41 #include <sys/kdebug.h>
42 #include <sys/kdebug_kernel.h>
43
44 extern boolean_t hibernate_cleaning_in_progress;
45
46 /* map a (whole) upl into an address space */
47 kern_return_t
vm_upl_map(vm_map_t map,upl_t upl,vm_address_t * dst_addr)48 vm_upl_map(
49 vm_map_t map,
50 upl_t upl,
51 vm_address_t *dst_addr)
52 {
53 vm_map_offset_t map_addr;
54 kern_return_t kr;
55
56 if (VM_MAP_NULL == map) {
57 return KERN_INVALID_ARGUMENT;
58 }
59
60 kr = vm_map_enter_upl(map, upl, &map_addr);
61 *dst_addr = CAST_DOWN(vm_address_t, map_addr);
62 return kr;
63 }
64
65 kern_return_t
vm_upl_unmap(vm_map_t map,upl_t upl)66 vm_upl_unmap(
67 vm_map_t map,
68 upl_t upl)
69 {
70 if (VM_MAP_NULL == map) {
71 return KERN_INVALID_ARGUMENT;
72 }
73
74 return vm_map_remove_upl(map, upl);
75 }
76
77 /* map a part of a upl into an address space with requested protection. */
78 kern_return_t
vm_upl_map_range(vm_map_t map,upl_t upl,vm_offset_t offset_to_map,vm_size_t size_to_map,vm_prot_t prot_to_map,vm_address_t * dst_addr)79 vm_upl_map_range(
80 vm_map_t map,
81 upl_t upl,
82 vm_offset_t offset_to_map,
83 vm_size_t size_to_map,
84 vm_prot_t prot_to_map,
85 vm_address_t *dst_addr)
86 {
87 vm_map_offset_t map_addr, aligned_offset_to_map, adjusted_offset;
88 kern_return_t kr;
89
90 if (VM_MAP_NULL == map) {
91 return KERN_INVALID_ARGUMENT;
92 }
93 aligned_offset_to_map = vm_map_trunc_page(offset_to_map, vm_map_page_mask(map));
94 adjusted_offset = offset_to_map - aligned_offset_to_map;
95 size_to_map = vm_map_round_page(size_to_map + adjusted_offset, vm_map_page_mask(map));
96
97 kr = vm_map_enter_upl_range(map, upl, aligned_offset_to_map, size_to_map, prot_to_map, &map_addr);
98 *dst_addr = CAST_DOWN(vm_address_t, (map_addr + adjusted_offset));
99 return kr;
100 }
101
102 /* unmap a part of a upl that was mapped in the address space. */
103 kern_return_t
vm_upl_unmap_range(vm_map_t map,upl_t upl,vm_offset_t offset_to_unmap,vm_size_t size_to_unmap)104 vm_upl_unmap_range(
105 vm_map_t map,
106 upl_t upl,
107 vm_offset_t offset_to_unmap,
108 vm_size_t size_to_unmap)
109 {
110 vm_map_offset_t aligned_offset_to_unmap, page_offset;
111
112 if (VM_MAP_NULL == map) {
113 return KERN_INVALID_ARGUMENT;
114 }
115
116 aligned_offset_to_unmap = vm_map_trunc_page(offset_to_unmap, vm_map_page_mask(map));
117 page_offset = offset_to_unmap - aligned_offset_to_unmap;
118 size_to_unmap = vm_map_round_page(size_to_unmap + page_offset, vm_map_page_mask(map));
119
120 return vm_map_remove_upl_range(map, upl, aligned_offset_to_unmap, size_to_unmap);
121 }
122
123 /* Retrieve a upl for an object underlying an address range in a map */
124
125 kern_return_t
vm_map_get_upl(vm_map_t map,vm_map_offset_t map_offset,upl_size_t * upl_size,upl_t * upl,upl_page_info_array_t page_list,unsigned int * count,upl_control_flags_t * flags,vm_tag_t tag,int force_data_sync)126 vm_map_get_upl(
127 vm_map_t map,
128 vm_map_offset_t map_offset,
129 upl_size_t *upl_size,
130 upl_t *upl,
131 upl_page_info_array_t page_list,
132 unsigned int *count,
133 upl_control_flags_t *flags,
134 vm_tag_t tag,
135 int force_data_sync)
136 {
137 upl_control_flags_t map_flags;
138 kern_return_t kr;
139
140 if (VM_MAP_NULL == map) {
141 return KERN_INVALID_ARGUMENT;
142 }
143
144 map_flags = *flags & ~UPL_NOZEROFILL;
145 if (force_data_sync) {
146 map_flags |= UPL_FORCE_DATA_SYNC;
147 }
148
149 kr = vm_map_create_upl(map,
150 map_offset,
151 upl_size,
152 upl,
153 page_list,
154 count,
155 &map_flags,
156 tag);
157
158 *flags = (map_flags & ~UPL_FORCE_DATA_SYNC);
159 return kr;
160 }
161
162 uint64_t upl_pages_wired_busy = 0;
163
164 kern_return_t
upl_abort_range(upl_t upl,upl_offset_t offset,upl_size_t size,int error,boolean_t * empty)165 upl_abort_range(
166 upl_t upl,
167 upl_offset_t offset,
168 upl_size_t size,
169 int error,
170 boolean_t *empty)
171 {
172 upl_size_t xfer_size, subupl_size;
173 vm_object_t shadow_object;
174 vm_object_t object;
175 vm_object_offset_t target_offset;
176 upl_offset_t subupl_offset = offset;
177 int occupied;
178 struct vm_page_delayed_work dw_array;
179 struct vm_page_delayed_work *dwp, *dwp_start;
180 bool dwp_finish_ctx = TRUE;
181 int dw_count;
182 int dw_limit;
183 int isVectorUPL = 0;
184 upl_t vector_upl = NULL;
185 vm_object_offset_t obj_start, obj_end, obj_offset;
186 kern_return_t kr = KERN_SUCCESS;
187
188 // DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p offset 0x%llx size 0x%llx error 0x%x\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object, (uint64_t)offset, (uint64_t)size, error);
189
190 dwp_start = dwp = NULL;
191
192 subupl_size = size;
193 *empty = FALSE;
194
195 if (upl == UPL_NULL) {
196 return KERN_INVALID_ARGUMENT;
197 }
198
199 if ((upl->flags & UPL_IO_WIRE) && !(error & UPL_ABORT_DUMP_PAGES)) {
200 return upl_commit_range(upl, offset, size, UPL_COMMIT_FREE_ABSENT, NULL, 0, empty);
201 }
202
203 dw_count = 0;
204 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
205 dwp_start = vm_page_delayed_work_get_ctx();
206 if (dwp_start == NULL) {
207 dwp_start = &dw_array;
208 dw_limit = 1;
209 dwp_finish_ctx = FALSE;
210 }
211
212 dwp = dwp_start;
213
214 if ((isVectorUPL = vector_upl_is_valid(upl))) {
215 vector_upl = upl;
216 upl_lock(vector_upl);
217 } else {
218 upl_lock(upl);
219 }
220
221 process_upl_to_abort:
222 if (isVectorUPL) {
223 size = subupl_size;
224 offset = subupl_offset;
225 if (size == 0) {
226 upl_unlock(vector_upl);
227 kr = KERN_SUCCESS;
228 goto done;
229 }
230 upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
231 if (upl == NULL) {
232 upl_unlock(vector_upl);
233 kr = KERN_FAILURE;
234 goto done;
235 }
236 subupl_size -= size;
237 subupl_offset += size;
238 }
239
240 *empty = FALSE;
241
242 #if UPL_DEBUG
243 if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
244 upl->upl_commit_records[upl->upl_commit_index].c_btref = btref_get(__builtin_frame_address(0), 0);
245 upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
246 upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
247 upl->upl_commit_records[upl->upl_commit_index].c_aborted = 1;
248
249 upl->upl_commit_index++;
250 }
251 #endif
252 if (upl->flags & UPL_DEVICE_MEMORY) {
253 xfer_size = 0;
254 } else if ((offset + size) <= upl_adjusted_size(upl, PAGE_MASK)) {
255 xfer_size = size;
256 } else {
257 if (!isVectorUPL) {
258 upl_unlock(upl);
259 } else {
260 upl_unlock(vector_upl);
261 }
262 DEBUG4K_ERROR("upl %p (u_offset 0x%llx u_size 0x%x) offset 0x%x size 0x%x\n", upl, upl->u_offset, upl->u_size, offset, size);
263 kr = KERN_FAILURE;
264 goto done;
265 }
266 object = upl->map_object;
267
268 if (upl->flags & UPL_SHADOWED) {
269 vm_object_lock(object);
270 shadow_object = object->shadow;
271 } else {
272 shadow_object = object;
273 }
274
275 target_offset = (vm_object_offset_t)offset;
276
277 if (upl->flags & UPL_KERNEL_OBJECT) {
278 vm_object_lock_shared(shadow_object);
279 } else {
280 vm_object_lock(shadow_object);
281 }
282
283 if (upl->flags & UPL_ACCESS_BLOCKED) {
284 assert(shadow_object->blocked_access);
285 shadow_object->blocked_access = FALSE;
286 vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
287 }
288
289 if ((error & UPL_ABORT_DUMP_PAGES) && (upl->flags & UPL_KERNEL_OBJECT)) {
290 panic("upl_abort_range: kernel_object being DUMPED");
291 }
292
293 obj_start = target_offset + upl->u_offset - shadow_object->paging_offset;
294 obj_end = obj_start + xfer_size;
295 obj_start = vm_object_trunc_page(obj_start);
296 obj_end = vm_object_round_page(obj_end);
297 for (obj_offset = obj_start;
298 obj_offset < obj_end;
299 obj_offset += PAGE_SIZE) {
300 vm_page_t t, m;
301 unsigned int pg_num;
302 boolean_t needed;
303
304 pg_num = (unsigned int) (target_offset / PAGE_SIZE);
305 assert(pg_num == target_offset / PAGE_SIZE);
306
307 needed = FALSE;
308
309 if (upl->flags & UPL_INTERNAL) {
310 needed = upl->page_list[pg_num].needed;
311 }
312
313 dwp->dw_mask = 0;
314 m = VM_PAGE_NULL;
315
316 if (upl->flags & UPL_LITE) {
317 if (bitmap_test(upl->lite_list, pg_num)) {
318 bitmap_clear(upl->lite_list, pg_num);
319
320 if (!(upl->flags & UPL_KERNEL_OBJECT)) {
321 m = vm_page_lookup(shadow_object, obj_offset);
322 }
323 }
324 }
325 if (upl->flags & UPL_SHADOWED) {
326 if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
327 t->vmp_free_when_done = FALSE;
328
329 VM_PAGE_FREE(t);
330
331 if (m == VM_PAGE_NULL) {
332 m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
333 }
334 }
335 }
336 if ((upl->flags & UPL_KERNEL_OBJECT)) {
337 goto abort_next_page;
338 }
339
340 if (m != VM_PAGE_NULL) {
341 assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
342
343 if (m->vmp_absent) {
344 boolean_t must_free = TRUE;
345
346 /*
347 * COPYOUT = FALSE case
348 * check for error conditions which must
349 * be passed back to the pages customer
350 */
351 if (error & UPL_ABORT_RESTART) {
352 m->vmp_restart = TRUE;
353 m->vmp_absent = FALSE;
354 m->vmp_unusual = TRUE;
355 must_free = FALSE;
356 } else if (error & UPL_ABORT_UNAVAILABLE) {
357 m->vmp_restart = FALSE;
358 m->vmp_unusual = TRUE;
359 must_free = FALSE;
360 } else if (error & UPL_ABORT_ERROR) {
361 m->vmp_restart = FALSE;
362 m->vmp_absent = FALSE;
363 m->vmp_error = TRUE;
364 m->vmp_unusual = TRUE;
365 must_free = FALSE;
366 }
367 if (m->vmp_clustered && needed == FALSE) {
368 /*
369 * This page was a part of a speculative
370 * read-ahead initiated by the kernel
371 * itself. No one is expecting this
372 * page and no one will clean up its
373 * error state if it ever becomes valid
374 * in the future.
375 * We have to free it here.
376 */
377 must_free = TRUE;
378 if (upl->flags & UPL_PAGEIN) {
379 counter_inc(&vm_statistics_pageins_aborted);
380 /*
381 * Due to split pageins, aborted head I/O results in over-counting of
382 * page-ins. Decrement the counter here so that the counter is closer
383 * to reality.
384 */
385 counter_dec(&vm_statistics_pageins);
386 }
387 }
388 m->vmp_cleaning = FALSE;
389
390 if (m->vmp_overwriting && !m->vmp_busy) {
391 /*
392 * this shouldn't happen since
393 * this is an 'absent' page, but
394 * it doesn't hurt to check for
395 * the 'alternate' method of
396 * stabilizing the page...
397 * we will mark 'busy' to be cleared
398 * in the following code which will
399 * take care of the primary stabilzation
400 * method (i.e. setting 'busy' to TRUE)
401 */
402 dwp->dw_mask |= DW_vm_page_unwire;
403 }
404 m->vmp_overwriting = FALSE;
405
406 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
407
408 if (must_free == TRUE) {
409 dwp->dw_mask |= DW_vm_page_free;
410 } else {
411 dwp->dw_mask |= DW_vm_page_activate;
412 }
413 } else {
414 /*
415 * Handle the trusted pager throttle.
416 */
417 if (m->vmp_laundry) {
418 dwp->dw_mask |= DW_vm_pageout_throttle_up;
419 }
420
421 if (upl->flags & UPL_ACCESS_BLOCKED) {
422 /*
423 * We blocked access to the pages in this UPL.
424 * Clear the "busy" bit and wake up any waiter
425 * for this page.
426 */
427 dwp->dw_mask |= DW_clear_busy;
428 }
429 if (m->vmp_overwriting) {
430 if (VM_PAGE_WIRED(m)) {
431 /*
432 * deal with the 'alternate' method
433 * of stabilizing the page...
434 * we will either free the page
435 * or mark 'busy' to be cleared
436 * in the following code which will
437 * take care of the primary stabilzation
438 * method (i.e. setting 'busy' to TRUE)
439 */
440 if (m->vmp_busy) {
441 // printf("******* FBDP %s:%d page %p object %p ofsfet 0x%llx wired and busy\n", __FUNCTION__, __LINE__, m, VM_PAGE_OBJECT(m), m->vmp_offset);
442 upl_pages_wired_busy++;
443 }
444 dwp->dw_mask |= DW_vm_page_unwire;
445 } else {
446 assert(m->vmp_busy);
447 dwp->dw_mask |= DW_clear_busy;
448 }
449 m->vmp_overwriting = FALSE;
450 }
451 m->vmp_free_when_done = FALSE;
452 m->vmp_cleaning = FALSE;
453
454 if (error & UPL_ABORT_DUMP_PAGES) {
455 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
456
457 dwp->dw_mask |= DW_vm_page_free;
458 } else {
459 if (!(dwp->dw_mask & DW_vm_page_unwire)) {
460 if (error & UPL_ABORT_REFERENCE) {
461 /*
462 * we've been told to explictly
463 * reference this page... for
464 * file I/O, this is done by
465 * implementing an LRU on the inactive q
466 */
467 dwp->dw_mask |= DW_vm_page_lru;
468 } else if (!VM_PAGE_PAGEABLE(m)) {
469 dwp->dw_mask |= DW_vm_page_deactivate_internal;
470 }
471 }
472 dwp->dw_mask |= DW_PAGE_WAKEUP;
473 }
474 }
475 }
476 abort_next_page:
477 target_offset += PAGE_SIZE_64;
478 xfer_size -= PAGE_SIZE;
479
480 if (dwp->dw_mask) {
481 if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
482 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
483
484 if (dw_count >= dw_limit) {
485 vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
486
487 dwp = dwp_start;
488 dw_count = 0;
489 }
490 } else {
491 if (dwp->dw_mask & DW_clear_busy) {
492 m->vmp_busy = FALSE;
493 }
494
495 if (dwp->dw_mask & DW_PAGE_WAKEUP) {
496 vm_page_wakeup(shadow_object, m);
497 }
498 }
499 }
500 }
501 if (dw_count) {
502 vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
503 dwp = dwp_start;
504 dw_count = 0;
505 }
506
507 if (upl->flags & UPL_DEVICE_MEMORY) {
508 occupied = 0;
509 } else if (upl->flags & UPL_LITE) {
510 uint32_t pages = (uint32_t)atop(upl_adjusted_size(upl, PAGE_MASK));
511
512 occupied = !bitmap_is_empty(upl->lite_list, pages);
513 } else {
514 occupied = !vm_page_queue_empty(&upl->map_object->memq);
515 }
516 if (occupied == 0) {
517 /*
518 * If this UPL element belongs to a Vector UPL and is
519 * empty, then this is the right function to deallocate
520 * it. So go ahead set the *empty variable. The flag
521 * UPL_COMMIT_NOTIFY_EMPTY, from the caller's point of view
522 * should be considered relevant for the Vector UPL and
523 * not the internal UPLs.
524 */
525 if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) {
526 *empty = TRUE;
527 }
528
529 if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
530 /*
531 * this is not a paging object
532 * so we need to drop the paging reference
533 * that was taken when we created the UPL
534 * against this object
535 */
536 vm_object_activity_end(shadow_object);
537 vm_object_collapse(shadow_object, 0, TRUE);
538 } else {
539 /*
540 * we dontated the paging reference to
541 * the map object... vm_pageout_object_terminate
542 * will drop this reference
543 */
544 }
545 }
546 vm_object_unlock(shadow_object);
547 if (object != shadow_object) {
548 vm_object_unlock(object);
549 }
550
551 if (!isVectorUPL) {
552 upl_unlock(upl);
553 } else {
554 /*
555 * If we completed our operations on an UPL that is
556 * part of a Vectored UPL and if empty is TRUE, then
557 * we should go ahead and deallocate this UPL element.
558 * Then we check if this was the last of the UPL elements
559 * within that Vectored UPL. If so, set empty to TRUE
560 * so that in ubc_upl_abort_range or ubc_upl_abort, we
561 * can go ahead and deallocate the Vector UPL too.
562 */
563 if (*empty == TRUE) {
564 *empty = vector_upl_set_subupl(vector_upl, upl, 0);
565 upl_deallocate(upl);
566 }
567 goto process_upl_to_abort;
568 }
569
570 kr = KERN_SUCCESS;
571
572 done:
573 if (dwp_start && dwp_finish_ctx) {
574 vm_page_delayed_work_finish_ctx(dwp_start);
575 dwp_start = dwp = NULL;
576 }
577
578 return kr;
579 }
580
581 kern_return_t
upl_abort(upl_t upl,int error)582 upl_abort(
583 upl_t upl,
584 int error)
585 {
586 boolean_t empty;
587
588 if (upl == UPL_NULL) {
589 return KERN_INVALID_ARGUMENT;
590 }
591
592 return upl_abort_range(upl, 0, upl->u_size, error, &empty);
593 }
594
595 kern_return_t
upl_commit_range(upl_t upl,upl_offset_t offset,upl_size_t size,int flags,upl_page_info_t * page_list,mach_msg_type_number_t count,boolean_t * empty)596 upl_commit_range(
597 upl_t upl,
598 upl_offset_t offset,
599 upl_size_t size,
600 int flags,
601 upl_page_info_t *page_list,
602 mach_msg_type_number_t count,
603 boolean_t *empty)
604 {
605 upl_size_t xfer_size, subupl_size;
606 vm_object_t shadow_object;
607 vm_object_t object;
608 vm_object_t m_object;
609 vm_object_offset_t target_offset;
610 upl_offset_t subupl_offset = offset;
611 int entry;
612 int occupied;
613 int clear_refmod = 0;
614 int pgpgout_count = 0;
615 struct vm_page_delayed_work dw_array;
616 struct vm_page_delayed_work *dwp, *dwp_start;
617 bool dwp_finish_ctx = TRUE;
618 int dw_count;
619 int dw_limit;
620 int isVectorUPL = 0;
621 upl_t vector_upl = NULL;
622 boolean_t should_be_throttled = FALSE;
623
624 vm_page_t nxt_page = VM_PAGE_NULL;
625 int fast_path_possible = 0;
626 int fast_path_full_commit = 0;
627 int throttle_page = 0;
628 int unwired_count = 0;
629 int local_queue_count = 0;
630 vm_page_t first_local, last_local;
631 vm_object_offset_t obj_start, obj_end, obj_offset;
632 kern_return_t kr = KERN_SUCCESS;
633
634 // DEBUG4K_UPL("upl %p (u_offset 0x%llx u_size 0x%llx) object %p offset 0x%llx size 0x%llx flags 0x%x\n", upl, (uint64_t)upl->u_offset, (uint64_t)upl->u_size, upl->map_object, (uint64_t)offset, (uint64_t)size, flags);
635
636 dwp_start = dwp = NULL;
637
638 subupl_size = size;
639 *empty = FALSE;
640
641 if (upl == UPL_NULL) {
642 return KERN_INVALID_ARGUMENT;
643 }
644
645 dw_count = 0;
646 dw_limit = DELAYED_WORK_LIMIT(DEFAULT_DELAYED_WORK_LIMIT);
647 dwp_start = vm_page_delayed_work_get_ctx();
648 if (dwp_start == NULL) {
649 dwp_start = &dw_array;
650 dw_limit = 1;
651 dwp_finish_ctx = FALSE;
652 }
653
654 dwp = dwp_start;
655
656 if (count == 0) {
657 page_list = NULL;
658 }
659
660 if ((isVectorUPL = vector_upl_is_valid(upl))) {
661 vector_upl = upl;
662 upl_lock(vector_upl);
663 } else {
664 upl_lock(upl);
665 }
666
667 process_upl_to_commit:
668
669 if (isVectorUPL) {
670 size = subupl_size;
671 offset = subupl_offset;
672 if (size == 0) {
673 upl_unlock(vector_upl);
674 kr = KERN_SUCCESS;
675 goto done;
676 }
677 upl = vector_upl_subupl_byoffset(vector_upl, &offset, &size);
678 if (upl == NULL) {
679 upl_unlock(vector_upl);
680 kr = KERN_FAILURE;
681 goto done;
682 }
683 assertf(upl->flags & UPL_INTERNAL, "%s: sub-upl %p of vector upl %p has no internal page list",
684 __func__, upl, vector_upl);
685 page_list = upl->page_list;
686 subupl_size -= size;
687 subupl_offset += size;
688 }
689
690 #if UPL_DEBUG
691 if (upl->upl_commit_index < UPL_DEBUG_COMMIT_RECORDS) {
692 upl->upl_commit_records[upl->upl_commit_index].c_btref = btref_get(__builtin_frame_address(0), 0);
693 upl->upl_commit_records[upl->upl_commit_index].c_beg = offset;
694 upl->upl_commit_records[upl->upl_commit_index].c_end = (offset + size);
695
696 upl->upl_commit_index++;
697 }
698 #endif
699 if (upl->flags & UPL_DEVICE_MEMORY) {
700 xfer_size = 0;
701 } else if ((offset + size) <= upl_adjusted_size(upl, PAGE_MASK)) {
702 xfer_size = size;
703 } else {
704 if (!isVectorUPL) {
705 upl_unlock(upl);
706 } else {
707 upl_unlock(vector_upl);
708 }
709 DEBUG4K_ERROR("upl %p (u_offset 0x%llx u_size 0x%x) offset 0x%x size 0x%x\n", upl, upl->u_offset, upl->u_size, offset, size);
710 kr = KERN_FAILURE;
711 goto done;
712 }
713 if (upl->flags & UPL_SET_DIRTY) {
714 flags |= UPL_COMMIT_SET_DIRTY;
715 }
716 if (upl->flags & UPL_CLEAR_DIRTY) {
717 flags |= UPL_COMMIT_CLEAR_DIRTY;
718 }
719
720 object = upl->map_object;
721
722 if (upl->flags & UPL_SHADOWED) {
723 vm_object_lock(object);
724 shadow_object = object->shadow;
725 } else {
726 shadow_object = object;
727 }
728 entry = offset / PAGE_SIZE;
729 target_offset = (vm_object_offset_t)offset;
730
731 if (upl->flags & UPL_KERNEL_OBJECT) {
732 vm_object_lock_shared(shadow_object);
733 } else {
734 vm_object_lock(shadow_object);
735 }
736
737 if (upl->flags & UPL_IO_WIRE &&
738 !(flags & (UPL_COMMIT_INACTIVATE | UPL_COMMIT_SPECULATE)) &&
739 !is_kernel_object(shadow_object) &&
740 vm_page_deactivate_behind &&
741 (shadow_object->resident_page_count - shadow_object->wired_page_count + atop_64(xfer_size) >
742 vm_page_active_count / vm_page_deactivate_behind_min_resident_ratio)) {
743 /*
744 * We're being asked to un-wire pages from a very-large resident vm-object
745 * Naively inserting the pages into the active queue is likely to induce
746 * thrashing with the backing store -- i.e. we will be forced to
747 * evict hot pages that are likely to be re-faulted before we can get to
748 * this UPL's pages in the LRU. Immediately deactivate the pages instead so
749 * that we can evict them before currently-active pages.
750 */
751 flags |= UPL_COMMIT_INACTIVATE;
752 KDBG(VMDBG_CODE(DBG_VM_UPL_COMMIT_FORCE_DEACTIVATE) | DBG_FUNC_NONE,
753 VM_KERNEL_ADDRHIDE(shadow_object), upl->u_offset, xfer_size);
754 }
755
756 VM_OBJECT_WIRED_PAGE_UPDATE_START(shadow_object);
757
758 if (upl->flags & UPL_ACCESS_BLOCKED) {
759 assert(shadow_object->blocked_access);
760 shadow_object->blocked_access = FALSE;
761 vm_object_wakeup(object, VM_OBJECT_EVENT_UNBLOCKED);
762 }
763
764 if (shadow_object->code_signed) {
765 /*
766 * CODE SIGNING:
767 * If the object is code-signed, do not let this UPL tell
768 * us if the pages are valid or not. Let the pages be
769 * validated by VM the normal way (when they get mapped or
770 * copied).
771 */
772 flags &= ~UPL_COMMIT_CS_VALIDATED;
773 }
774 if (!page_list) {
775 /*
776 * No page list to get the code-signing info from !?
777 */
778 flags &= ~UPL_COMMIT_CS_VALIDATED;
779 }
780 if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal) {
781 should_be_throttled = TRUE;
782 }
783
784 if ((upl->flags & UPL_IO_WIRE) &&
785 !(flags & UPL_COMMIT_FREE_ABSENT) &&
786 !isVectorUPL &&
787 shadow_object->purgable != VM_PURGABLE_VOLATILE &&
788 shadow_object->purgable != VM_PURGABLE_EMPTY) {
789 if (!vm_page_queue_empty(&shadow_object->memq)) {
790 if (shadow_object->internal && size == shadow_object->vo_size) {
791 nxt_page = (vm_page_t)vm_page_queue_first(&shadow_object->memq);
792 fast_path_full_commit = 1;
793 }
794 fast_path_possible = 1;
795
796 if (!VM_DYNAMIC_PAGING_ENABLED() && shadow_object->internal &&
797 (shadow_object->purgable == VM_PURGABLE_DENY ||
798 shadow_object->purgable == VM_PURGABLE_NONVOLATILE ||
799 shadow_object->purgable == VM_PURGABLE_VOLATILE)) {
800 throttle_page = 1;
801 }
802 }
803 }
804 first_local = VM_PAGE_NULL;
805 last_local = VM_PAGE_NULL;
806
807 obj_start = target_offset + upl->u_offset - shadow_object->paging_offset;
808 obj_end = obj_start + xfer_size;
809 obj_start = vm_object_trunc_page(obj_start);
810 obj_end = vm_object_round_page(obj_end);
811 for (obj_offset = obj_start;
812 obj_offset < obj_end;
813 obj_offset += PAGE_SIZE) {
814 vm_page_t t, m;
815
816 dwp->dw_mask = 0;
817 clear_refmod = 0;
818
819 m = VM_PAGE_NULL;
820
821 if (upl->flags & UPL_LITE) {
822 unsigned int pg_num;
823
824 if (nxt_page != VM_PAGE_NULL) {
825 m = nxt_page;
826 nxt_page = (vm_page_t)vm_page_queue_next(&nxt_page->vmp_listq);
827 target_offset = m->vmp_offset;
828 }
829 pg_num = (unsigned int) (target_offset / PAGE_SIZE);
830 assert(pg_num == target_offset / PAGE_SIZE);
831
832 if (bitmap_test(upl->lite_list, pg_num)) {
833 bitmap_clear(upl->lite_list, pg_num);
834
835 if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {
836 m = vm_page_lookup(shadow_object, obj_offset);
837 }
838 } else {
839 m = NULL;
840 }
841 }
842 if (upl->flags & UPL_SHADOWED) {
843 if ((t = vm_page_lookup(object, target_offset)) != VM_PAGE_NULL) {
844 t->vmp_free_when_done = FALSE;
845
846 VM_PAGE_FREE(t);
847
848 if (!(upl->flags & UPL_KERNEL_OBJECT) && m == VM_PAGE_NULL) {
849 m = vm_page_lookup(shadow_object, target_offset + object->vo_shadow_offset);
850 }
851 }
852 }
853 if (m == VM_PAGE_NULL) {
854 goto commit_next_page;
855 }
856
857 m_object = VM_PAGE_OBJECT(m);
858
859 if (m->vmp_q_state == VM_PAGE_USED_BY_COMPRESSOR) {
860 assert(m->vmp_busy);
861
862 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
863 #if HAS_MTE
864 if (vm_page_is_tag_storage_pnum(m, VM_PAGE_GET_PHYS_PAGE(m)) &&
865 m->vmp_ts_wanted) {
866 dwp->dw_mask |= DW_vm_page_wakeup_tag_storage;
867 }
868 #endif /* HAS_MTE */
869 goto commit_next_page;
870 }
871
872 if (flags & UPL_COMMIT_CS_VALIDATED) {
873 /*
874 * CODE SIGNING:
875 * Set the code signing bits according to
876 * what the UPL says they should be.
877 */
878 m->vmp_cs_validated |= page_list[entry].cs_validated;
879 m->vmp_cs_tainted |= page_list[entry].cs_tainted;
880 m->vmp_cs_nx |= page_list[entry].cs_nx;
881 }
882 if (flags & UPL_COMMIT_WRITTEN_BY_KERNEL) {
883 m->vmp_written_by_kernel = TRUE;
884 }
885
886 if (upl->flags & UPL_IO_WIRE) {
887 if (page_list) {
888 page_list[entry].phys_addr = 0;
889 }
890
891 if (flags & UPL_COMMIT_SET_DIRTY) {
892 SET_PAGE_DIRTY(m, FALSE);
893 } else if (flags & UPL_COMMIT_CLEAR_DIRTY) {
894 m->vmp_dirty = FALSE;
895
896 if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
897 m->vmp_cs_validated &&
898 m->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
899 /*
900 * CODE SIGNING:
901 * This page is no longer dirty
902 * but could have been modified,
903 * so it will need to be
904 * re-validated.
905 */
906 m->vmp_cs_validated = VMP_CS_ALL_FALSE;
907
908 VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
909
910 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
911 }
912 clear_refmod |= VM_MEM_MODIFIED;
913 }
914 if (upl->flags & UPL_ACCESS_BLOCKED) {
915 /*
916 * We blocked access to the pages in this UPL.
917 * Clear the "busy" bit and wake up any waiter
918 * for this page.
919 */
920 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
921 }
922 if (fast_path_possible) {
923 assert(m_object->purgable != VM_PURGABLE_EMPTY);
924 assert(m_object->purgable != VM_PURGABLE_VOLATILE);
925 if (m->vmp_absent) {
926 assert(m->vmp_q_state == VM_PAGE_NOT_ON_Q);
927 assert(m->vmp_wire_count == 0);
928 assert(m->vmp_busy);
929
930 m->vmp_absent = FALSE;
931 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
932 } else {
933 if (m->vmp_wire_count == 0) {
934 panic("wire_count == 0, m = %p, obj = %p", m, shadow_object);
935 }
936 assert(m->vmp_q_state == VM_PAGE_IS_WIRED);
937
938 /*
939 * XXX FBDP need to update some other
940 * counters here (purgeable_wired_count)
941 * (ledgers), ...
942 */
943 assert(m->vmp_wire_count > 0);
944 m->vmp_wire_count--;
945
946 if (m->vmp_wire_count == 0) {
947 m->vmp_q_state = VM_PAGE_NOT_ON_Q;
948 m->vmp_iopl_wired = false;
949 unwired_count++;
950
951 #if HAS_MTE
952 mteinfo_decrement_wire_count(m, false);
953 #endif /* HAS_MTE */
954 }
955 }
956 if (m->vmp_wire_count == 0) {
957 assert(m->vmp_pageq.next == 0 && m->vmp_pageq.prev == 0);
958
959 if (last_local == VM_PAGE_NULL) {
960 assert(first_local == VM_PAGE_NULL);
961
962 last_local = m;
963 first_local = m;
964 } else {
965 assert(first_local != VM_PAGE_NULL);
966
967 m->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
968 first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(m);
969 first_local = m;
970 }
971 local_queue_count++;
972
973 if (throttle_page) {
974 m->vmp_q_state = VM_PAGE_ON_THROTTLED_Q;
975 } else {
976 if (flags & UPL_COMMIT_INACTIVATE) {
977 if (shadow_object->internal) {
978 m->vmp_q_state = VM_PAGE_ON_INACTIVE_INTERNAL_Q;
979 } else {
980 m->vmp_q_state = VM_PAGE_ON_INACTIVE_EXTERNAL_Q;
981 }
982 } else {
983 m->vmp_q_state = VM_PAGE_ON_ACTIVE_Q;
984 }
985 }
986 }
987 } else {
988 if (flags & UPL_COMMIT_INACTIVATE) {
989 dwp->dw_mask |= DW_vm_page_deactivate_internal;
990 clear_refmod |= VM_MEM_REFERENCED;
991 }
992 if (m->vmp_absent) {
993 if (flags & UPL_COMMIT_FREE_ABSENT) {
994 dwp->dw_mask |= DW_vm_page_free;
995 } else {
996 m->vmp_absent = FALSE;
997 dwp->dw_mask |= (DW_clear_busy | DW_PAGE_WAKEUP);
998
999 if (!(dwp->dw_mask & DW_vm_page_deactivate_internal)) {
1000 dwp->dw_mask |= DW_vm_page_activate;
1001 }
1002 }
1003 } else {
1004 dwp->dw_mask |= DW_vm_page_unwire;
1005 }
1006 }
1007 goto commit_next_page;
1008 }
1009 assert(m->vmp_q_state != VM_PAGE_USED_BY_COMPRESSOR);
1010
1011 if (page_list) {
1012 page_list[entry].phys_addr = 0;
1013 }
1014
1015 /*
1016 * make sure to clear the hardware
1017 * modify or reference bits before
1018 * releasing the BUSY bit on this page
1019 * otherwise we risk losing a legitimate
1020 * change of state
1021 */
1022 if (flags & UPL_COMMIT_CLEAR_DIRTY) {
1023 m->vmp_dirty = FALSE;
1024
1025 clear_refmod |= VM_MEM_MODIFIED;
1026 }
1027 if (m->vmp_laundry) {
1028 dwp->dw_mask |= DW_vm_pageout_throttle_up;
1029 }
1030
1031 if (VM_PAGE_WIRED(m)) {
1032 m->vmp_free_when_done = FALSE;
1033 }
1034
1035 if (!(flags & UPL_COMMIT_CS_VALIDATED) &&
1036 m->vmp_cs_validated &&
1037 m->vmp_cs_tainted != VMP_CS_ALL_TRUE) {
1038 /*
1039 * CODE SIGNING:
1040 * This page is no longer dirty
1041 * but could have been modified,
1042 * so it will need to be
1043 * re-validated.
1044 */
1045 m->vmp_cs_validated = VMP_CS_ALL_FALSE;
1046
1047 VM_PAGEOUT_DEBUG(vm_cs_validated_resets, 1);
1048
1049 pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m));
1050 }
1051 if (m->vmp_overwriting) {
1052 /*
1053 * the (COPY_OUT_FROM == FALSE) request_page_list case
1054 */
1055 if (VM_PAGE_WIRED(m)) {
1056 /*
1057 * alternate (COPY_OUT_FROM == FALSE) page_list case
1058 * Occurs when the original page was wired
1059 * at the time of the list request
1060 */
1061 if (m->vmp_busy) {
1062 // printf("******* FBDP %s:%d page %p object %p ofsfet 0x%llx wired and busy\n", __FUNCTION__, __LINE__, m, VM_PAGE_OBJECT(m), m->vmp_offset);
1063 upl_pages_wired_busy++;
1064 }
1065 assert(!m->vmp_absent);
1066 dwp->dw_mask |= DW_vm_page_unwire; /* reactivates */
1067 } else {
1068 assert(m->vmp_busy);
1069 #if CONFIG_PHANTOM_CACHE
1070 if (m->vmp_absent && !m_object->internal) {
1071 dwp->dw_mask |= DW_vm_phantom_cache_update;
1072 }
1073 #endif
1074 m->vmp_absent = FALSE;
1075
1076 dwp->dw_mask |= DW_clear_busy;
1077 }
1078 m->vmp_overwriting = FALSE;
1079 }
1080 m->vmp_cleaning = FALSE;
1081
1082 if (m->vmp_free_when_done) {
1083 /*
1084 * With the clean queue enabled, UPL_PAGEOUT should
1085 * no longer set the pageout bit. Its pages now go
1086 * to the clean queue.
1087 *
1088 * We don't use the cleaned Q anymore and so this
1089 * assert isn't correct. The code for the clean Q
1090 * still exists and might be used in the future. If we
1091 * go back to the cleaned Q, we will re-enable this
1092 * assert.
1093 *
1094 * assert(!(upl->flags & UPL_PAGEOUT));
1095 */
1096 assert(!m_object->internal);
1097
1098 m->vmp_free_when_done = FALSE;
1099
1100 if ((flags & UPL_COMMIT_SET_DIRTY) ||
1101 (m->vmp_pmapped && (pmap_disconnect(VM_PAGE_GET_PHYS_PAGE(m)) & VM_MEM_MODIFIED))) {
1102 /*
1103 * page was re-dirtied after we started
1104 * the pageout... reactivate it since
1105 * we don't know whether the on-disk
1106 * copy matches what is now in memory
1107 */
1108 SET_PAGE_DIRTY(m, FALSE);
1109
1110 dwp->dw_mask |= DW_vm_page_activate | DW_PAGE_WAKEUP;
1111
1112 if (upl->flags & UPL_PAGEOUT) {
1113 counter_inc(&vm_statistics_reactivations);
1114 DTRACE_VM2(pgrec, int, 1, (uint64_t *), NULL);
1115 }
1116 } else if (m->vmp_busy && !(upl->flags & UPL_HAS_BUSY)) {
1117 /*
1118 * Someone else might still be handling this
1119 * page (vm_fault() for example), so let's not
1120 * free it or "un-busy" it!
1121 * Put that page in the "speculative" queue
1122 * for now (since we would otherwise have freed
1123 * it) and let whoever is keeping the page
1124 * "busy" move it if needed when they're done
1125 * with it.
1126 */
1127 dwp->dw_mask |= DW_vm_page_speculate;
1128 } else {
1129 /*
1130 * page has been successfully cleaned
1131 * go ahead and free it for other use
1132 */
1133 if (m_object->internal) {
1134 DTRACE_VM2(anonpgout, int, 1, (uint64_t *), NULL);
1135 } else {
1136 DTRACE_VM2(fspgout, int, 1, (uint64_t *), NULL);
1137 }
1138 m->vmp_dirty = FALSE;
1139 if (!(upl->flags & UPL_HAS_BUSY)) {
1140 assert(!m->vmp_busy);
1141 }
1142 m->vmp_busy = TRUE;
1143
1144 dwp->dw_mask |= DW_vm_page_free;
1145 }
1146 goto commit_next_page;
1147 }
1148 /*
1149 * It is a part of the semantic of COPYOUT_FROM
1150 * UPLs that a commit implies cache sync
1151 * between the vm page and the backing store
1152 * this can be used to strip the precious bit
1153 * as well as clean
1154 */
1155 if ((upl->flags & UPL_PAGE_SYNC_DONE) || (flags & UPL_COMMIT_CLEAR_PRECIOUS)) {
1156 m->vmp_precious = FALSE;
1157 }
1158
1159 if (flags & UPL_COMMIT_SET_DIRTY) {
1160 SET_PAGE_DIRTY(m, FALSE);
1161 } else {
1162 m->vmp_dirty = FALSE;
1163 }
1164
1165 /* with the clean queue on, move *all* cleaned pages to the clean queue */
1166 if (hibernate_cleaning_in_progress == FALSE && !m->vmp_dirty && (upl->flags & UPL_PAGEOUT)) {
1167 pgpgout_count++;
1168
1169 counter_inc(&vm_statistics_pageouts);
1170 DTRACE_VM2(pgout, int, 1, (uint64_t *), NULL);
1171
1172 dwp->dw_mask |= DW_enqueue_cleaned;
1173 } else if (should_be_throttled == TRUE && (m->vmp_q_state == VM_PAGE_NOT_ON_Q)) {
1174 /*
1175 * page coming back in from being 'frozen'...
1176 * it was dirty before it was frozen, so keep it so
1177 * the vm_page_activate will notice that it really belongs
1178 * on the throttle queue and put it there
1179 */
1180 SET_PAGE_DIRTY(m, FALSE);
1181 dwp->dw_mask |= DW_vm_page_activate;
1182 } else {
1183 if ((flags & UPL_COMMIT_INACTIVATE) && !m->vmp_clustered && (m->vmp_q_state != VM_PAGE_ON_SPECULATIVE_Q)) {
1184 dwp->dw_mask |= DW_vm_page_deactivate_internal;
1185 clear_refmod |= VM_MEM_REFERENCED;
1186 } else if (!VM_PAGE_PAGEABLE(m)) {
1187 if (m->vmp_clustered || (flags & UPL_COMMIT_SPECULATE)) {
1188 dwp->dw_mask |= DW_vm_page_speculate;
1189 } else if (m->vmp_reference) {
1190 dwp->dw_mask |= DW_vm_page_activate;
1191 } else {
1192 dwp->dw_mask |= DW_vm_page_deactivate_internal;
1193 clear_refmod |= VM_MEM_REFERENCED;
1194 }
1195 }
1196 }
1197 if (upl->flags & UPL_ACCESS_BLOCKED) {
1198 /*
1199 * We blocked access to the pages in this URL.
1200 * Clear the "busy" bit on this page before we
1201 * wake up any waiter.
1202 */
1203 dwp->dw_mask |= DW_clear_busy;
1204 }
1205 /*
1206 * Wakeup any thread waiting for the page to be un-cleaning.
1207 */
1208 dwp->dw_mask |= DW_PAGE_WAKEUP;
1209
1210 commit_next_page:
1211 if (clear_refmod) {
1212 pmap_clear_refmod(VM_PAGE_GET_PHYS_PAGE(m), clear_refmod);
1213 }
1214
1215 target_offset += PAGE_SIZE_64;
1216 xfer_size -= PAGE_SIZE;
1217 entry++;
1218
1219 if (dwp->dw_mask) {
1220 if (dwp->dw_mask & ~(DW_clear_busy | DW_PAGE_WAKEUP)) {
1221 VM_PAGE_ADD_DELAYED_WORK(dwp, m, dw_count);
1222
1223 if (dw_count >= dw_limit) {
1224 vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
1225
1226 dwp = dwp_start;
1227 dw_count = 0;
1228 }
1229 } else {
1230 if (dwp->dw_mask & DW_clear_busy) {
1231 m->vmp_busy = FALSE;
1232 }
1233
1234 if (dwp->dw_mask & DW_PAGE_WAKEUP) {
1235 vm_page_wakeup(m_object, m);
1236 }
1237 }
1238 }
1239 }
1240 if (dw_count) {
1241 vm_page_do_delayed_work(shadow_object, VM_KERN_MEMORY_NONE, dwp_start, dw_count);
1242 dwp = dwp_start;
1243 dw_count = 0;
1244 }
1245
1246 if (fast_path_possible) {
1247 assert(shadow_object->purgable != VM_PURGABLE_VOLATILE);
1248 assert(shadow_object->purgable != VM_PURGABLE_EMPTY);
1249
1250 if (local_queue_count || unwired_count) {
1251 if (local_queue_count) {
1252 vm_page_t first_target;
1253 vm_page_queue_head_t *target_queue;
1254
1255 if (throttle_page) {
1256 target_queue = &vm_page_queue_throttled;
1257 } else {
1258 if (flags & UPL_COMMIT_INACTIVATE) {
1259 if (shadow_object->internal) {
1260 target_queue = &vm_page_queue_anonymous;
1261 } else {
1262 target_queue = &vm_page_queue_inactive;
1263 }
1264 } else {
1265 target_queue = &vm_page_queue_active;
1266 }
1267 }
1268 /*
1269 * Transfer the entire local queue to a regular LRU page queues.
1270 */
1271 vm_page_lockspin_queues();
1272
1273 first_target = (vm_page_t) vm_page_queue_first(target_queue);
1274
1275 if (vm_page_queue_empty(target_queue)) {
1276 target_queue->prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
1277 } else {
1278 first_target->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(last_local);
1279 }
1280
1281 target_queue->next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_local);
1282 first_local->vmp_pageq.prev = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(target_queue);
1283 last_local->vmp_pageq.next = VM_PAGE_CONVERT_TO_QUEUE_ENTRY(first_target);
1284
1285 /*
1286 * Adjust the global page counts.
1287 */
1288 if (throttle_page) {
1289 vm_page_throttled_count += local_queue_count;
1290 } else {
1291 if (flags & UPL_COMMIT_INACTIVATE) {
1292 if (shadow_object->internal) {
1293 vm_page_anonymous_count += local_queue_count;
1294 }
1295 vm_page_inactive_count += local_queue_count;
1296
1297 token_new_pagecount += local_queue_count;
1298 } else {
1299 vm_page_active_count += local_queue_count;
1300 }
1301
1302 if (shadow_object->internal) {
1303 vm_page_pageable_internal_count += local_queue_count;
1304 } else {
1305 vm_page_pageable_external_count += local_queue_count;
1306 }
1307 }
1308 } else {
1309 vm_page_lockspin_queues();
1310 }
1311 if (unwired_count) {
1312 vm_page_wire_count -= unwired_count;
1313 VM_CHECK_MEMORYSTATUS;
1314 }
1315 vm_page_unlock_queues();
1316
1317 VM_OBJECT_WIRED_PAGE_COUNT(shadow_object, -unwired_count);
1318 }
1319 }
1320
1321 if (upl->flags & UPL_DEVICE_MEMORY) {
1322 occupied = 0;
1323 } else if (upl->flags & UPL_LITE) {
1324 uint32_t pages = (uint32_t)atop(upl_adjusted_size(upl, PAGE_MASK));
1325
1326 occupied = !fast_path_full_commit &&
1327 !bitmap_is_empty(upl->lite_list, pages);
1328 } else {
1329 occupied = !vm_page_queue_empty(&upl->map_object->memq);
1330 }
1331 if (occupied == 0) {
1332 /*
1333 * If this UPL element belongs to a Vector UPL and is
1334 * empty, then this is the right function to deallocate
1335 * it. So go ahead set the *empty variable. The flag
1336 * UPL_COMMIT_NOTIFY_EMPTY, from the caller's point of view
1337 * should be considered relevant for the Vector UPL and not
1338 * the internal UPLs.
1339 */
1340 if ((upl->flags & UPL_COMMIT_NOTIFY_EMPTY) || isVectorUPL) {
1341 *empty = TRUE;
1342 }
1343
1344 if (object == shadow_object && !(upl->flags & UPL_KERNEL_OBJECT)) {
1345 /*
1346 * this is not a paging object
1347 * so we need to drop the paging reference
1348 * that was taken when we created the UPL
1349 * against this object
1350 */
1351 vm_object_activity_end(shadow_object);
1352 vm_object_collapse(shadow_object, 0, TRUE);
1353 } else {
1354 /*
1355 * we dontated the paging reference to
1356 * the map object... vm_pageout_object_terminate
1357 * will drop this reference
1358 */
1359 }
1360 }
1361 VM_OBJECT_WIRED_PAGE_UPDATE_END(shadow_object, shadow_object->wire_tag);
1362 vm_object_unlock(shadow_object);
1363 if (object != shadow_object) {
1364 vm_object_unlock(object);
1365 }
1366
1367 if (!isVectorUPL) {
1368 upl_unlock(upl);
1369 } else {
1370 /*
1371 * If we completed our operations on an UPL that is
1372 * part of a Vectored UPL and if empty is TRUE, then
1373 * we should go ahead and deallocate this UPL element.
1374 * Then we check if this was the last of the UPL elements
1375 * within that Vectored UPL. If so, set empty to TRUE
1376 * so that in ubc_upl_commit_range or ubc_upl_commit, we
1377 * can go ahead and deallocate the Vector UPL too.
1378 */
1379 if (*empty == TRUE) {
1380 *empty = vector_upl_set_subupl(vector_upl, upl, 0);
1381 upl_deallocate(upl);
1382 }
1383 goto process_upl_to_commit;
1384 }
1385 if (pgpgout_count) {
1386 DTRACE_VM2(pgpgout, int, pgpgout_count, (uint64_t *), NULL);
1387 }
1388
1389 kr = KERN_SUCCESS;
1390 done:
1391 if (dwp_start && dwp_finish_ctx) {
1392 vm_page_delayed_work_finish_ctx(dwp_start);
1393 dwp_start = dwp = NULL;
1394 }
1395
1396 return kr;
1397 }
1398
1399 /* an option on commit should be wire */
1400 kern_return_t
upl_commit(upl_t upl,upl_page_info_t * page_list,mach_msg_type_number_t count)1401 upl_commit(
1402 upl_t upl,
1403 upl_page_info_t *page_list,
1404 mach_msg_type_number_t count)
1405 {
1406 boolean_t empty;
1407
1408 if (upl == UPL_NULL) {
1409 return KERN_INVALID_ARGUMENT;
1410 }
1411
1412 return upl_commit_range(upl, 0, upl->u_size, 0,
1413 page_list, count, &empty);
1414 }
1415