1 /*
2 * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 /*
58 * Compressor Pager.
59 * Memory Object Management.
60 */
61
62 #include <kern/host_statistics.h>
63 #include <kern/kalloc.h>
64 #include <kern/ipc_kobject.h>
65
66 #include <machine/atomic.h>
67
68 #include <mach/memory_object_control.h>
69 #include <mach/memory_object_types.h>
70 #include <mach/upl.h>
71
72 #include <vm/memory_object.h>
73 #include <vm/vm_compressor_pager.h>
74 #include <vm/vm_external.h>
75 #include <vm/vm_pageout.h>
76 #include <vm/vm_protos.h>
77
78 #include <sys/kdebug_triage.h>
79
80 /* memory_object interfaces */
81 void compressor_memory_object_reference(memory_object_t mem_obj);
82 void compressor_memory_object_deallocate(memory_object_t mem_obj);
83 kern_return_t compressor_memory_object_init(
84 memory_object_t mem_obj,
85 memory_object_control_t control,
86 memory_object_cluster_size_t pager_page_size);
87 kern_return_t compressor_memory_object_terminate(memory_object_t mem_obj);
88 kern_return_t compressor_memory_object_data_request(
89 memory_object_t mem_obj,
90 memory_object_offset_t offset,
91 memory_object_cluster_size_t length,
92 __unused vm_prot_t protection_required,
93 memory_object_fault_info_t fault_info);
94 kern_return_t compressor_memory_object_data_return(
95 memory_object_t mem_obj,
96 memory_object_offset_t offset,
97 memory_object_cluster_size_t size,
98 __unused memory_object_offset_t *resid_offset,
99 __unused int *io_error,
100 __unused boolean_t dirty,
101 __unused boolean_t kernel_copy,
102 __unused int upl_flags);
103 kern_return_t compressor_memory_object_data_initialize(
104 memory_object_t mem_obj,
105 memory_object_offset_t offset,
106 memory_object_cluster_size_t size);
107 kern_return_t compressor_memory_object_map(
108 __unused memory_object_t mem_obj,
109 __unused vm_prot_t prot);
110 kern_return_t compressor_memory_object_last_unmap(memory_object_t mem_obj);
111
112 const struct memory_object_pager_ops compressor_pager_ops = {
113 .memory_object_reference = compressor_memory_object_reference,
114 .memory_object_deallocate = compressor_memory_object_deallocate,
115 .memory_object_init = compressor_memory_object_init,
116 .memory_object_terminate = compressor_memory_object_terminate,
117 .memory_object_data_request = compressor_memory_object_data_request,
118 .memory_object_data_return = compressor_memory_object_data_return,
119 .memory_object_data_initialize = compressor_memory_object_data_initialize,
120 .memory_object_map = compressor_memory_object_map,
121 .memory_object_last_unmap = compressor_memory_object_last_unmap,
122 .memory_object_backing_object = NULL,
123 .memory_object_pager_name = "compressor pager"
124 };
125
126 /* internal data structures */
127
128 struct {
129 uint64_t data_returns;
130 uint64_t data_requests;
131 uint64_t put;
132 uint64_t get;
133 uint64_t state_clr;
134 uint64_t state_get;
135 uint64_t transfer;
136 } compressor_pager_stats;
137
138 typedef int compressor_slot_t;
139
140 typedef struct compressor_pager {
141 /* mandatory generic header */
142 struct memory_object cpgr_hdr;
143
144 /* pager-specific data */
145 lck_mtx_t cpgr_lock;
146 #if MEMORY_OBJECT_HAS_REFCOUNT
147 #define cpgr_references cpgr_hdr.mo_ref
148 #else
149 os_ref_atomic_t cpgr_references;
150 #endif
151 unsigned int cpgr_num_slots;
152 unsigned int cpgr_num_slots_occupied;
153 union {
154 compressor_slot_t cpgr_eslots[2]; /* embedded slots */
155 compressor_slot_t *cpgr_dslots; /* direct slots */
156 compressor_slot_t **cpgr_islots; /* indirect slots */
157 } cpgr_slots;
158 } *compressor_pager_t;
159
160 #define compressor_pager_lookup(_mem_obj_, _cpgr_) \
161 MACRO_BEGIN \
162 if (_mem_obj_ == NULL || \
163 _mem_obj_->mo_pager_ops != &compressor_pager_ops) { \
164 _cpgr_ = NULL; \
165 } else { \
166 _cpgr_ = (compressor_pager_t) _mem_obj_; \
167 } \
168 MACRO_END
169
170 /* embedded slot pointers in compressor_pager get packed, so VA restricted */
171 static ZONE_DEFINE_TYPE(compressor_pager_zone, "compressor_pager",
172 struct compressor_pager, ZC_NOENCRYPT | ZC_VM | ZC_NOTBITAG);
173
174 LCK_GRP_DECLARE(compressor_pager_lck_grp, "compressor_pager");
175
176 #define compressor_pager_lock(_cpgr_) \
177 lck_mtx_lock(&(_cpgr_)->cpgr_lock)
178 #define compressor_pager_unlock(_cpgr_) \
179 lck_mtx_unlock(&(_cpgr_)->cpgr_lock)
180 #define compressor_pager_lock_init(_cpgr_) \
181 lck_mtx_init(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp, LCK_ATTR_NULL)
182 #define compressor_pager_lock_destroy(_cpgr_) \
183 lck_mtx_destroy(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp)
184
185 #define COMPRESSOR_SLOTS_CHUNK_SIZE (512)
186 #define COMPRESSOR_SLOTS_PER_CHUNK (COMPRESSOR_SLOTS_CHUNK_SIZE / sizeof (compressor_slot_t))
187
188 /* forward declarations */
189 unsigned int compressor_pager_slots_chunk_free(compressor_slot_t *chunk,
190 int num_slots,
191 int flags,
192 int *failures);
193 void compressor_pager_slot_lookup(
194 compressor_pager_t pager,
195 boolean_t do_alloc,
196 memory_object_offset_t offset,
197 compressor_slot_t **slot_pp);
198
199 #if defined(__LP64__)
200
201 /* restricted VA zones for slots */
202
203 #define NUM_SLOTS_ZONES 3
204
205 static const size_t compressor_slots_zones_sizes[NUM_SLOTS_ZONES] = {
206 16,
207 64,
208 COMPRESSOR_SLOTS_CHUNK_SIZE
209 };
210
211 static const char * compressor_slots_zones_names[NUM_SLOTS_ZONES] = {
212 "compressor_slots.16",
213 "compressor_slots.64",
214 "compressor_slots.512"
215 };
216
217 static zone_t
218 compressor_slots_zones[NUM_SLOTS_ZONES];
219
220 #endif /* defined(__LP64__) */
221
222 static void
223 zfree_slot_array(compressor_slot_t *slots, size_t size);
224 static compressor_slot_t *
225 zalloc_slot_array(size_t size, zalloc_flags_t);
226
227 static inline unsigned int
compressor_pager_num_chunks(compressor_pager_t pager)228 compressor_pager_num_chunks(
229 compressor_pager_t pager)
230 {
231 unsigned int num_chunks;
232
233 num_chunks = pager->cpgr_num_slots / COMPRESSOR_SLOTS_PER_CHUNK;
234 if (num_chunks * COMPRESSOR_SLOTS_PER_CHUNK < pager->cpgr_num_slots) {
235 num_chunks++;
236 }
237 return num_chunks;
238 }
239
240 kern_return_t
compressor_memory_object_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pager_page_size)241 compressor_memory_object_init(
242 memory_object_t mem_obj,
243 memory_object_control_t control,
244 __unused memory_object_cluster_size_t pager_page_size)
245 {
246 compressor_pager_t pager;
247
248 assert(pager_page_size == PAGE_SIZE);
249
250 memory_object_control_reference(control);
251
252 compressor_pager_lookup(mem_obj, pager);
253 compressor_pager_lock(pager);
254
255 if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
256 panic("compressor_memory_object_init: bad request");
257 }
258 pager->cpgr_hdr.mo_control = control;
259
260 compressor_pager_unlock(pager);
261
262 return KERN_SUCCESS;
263 }
264
265 kern_return_t
compressor_memory_object_map(__unused memory_object_t mem_obj,__unused vm_prot_t prot)266 compressor_memory_object_map(
267 __unused memory_object_t mem_obj,
268 __unused vm_prot_t prot)
269 {
270 panic("compressor_memory_object_map");
271 return KERN_FAILURE;
272 }
273
274 kern_return_t
compressor_memory_object_last_unmap(__unused memory_object_t mem_obj)275 compressor_memory_object_last_unmap(
276 __unused memory_object_t mem_obj)
277 {
278 panic("compressor_memory_object_last_unmap");
279 return KERN_FAILURE;
280 }
281
282 kern_return_t
compressor_memory_object_terminate(memory_object_t mem_obj)283 compressor_memory_object_terminate(
284 memory_object_t mem_obj)
285 {
286 memory_object_control_t control;
287 compressor_pager_t pager;
288
289 /*
290 * control port is a receive right, not a send right.
291 */
292
293 compressor_pager_lookup(mem_obj, pager);
294 compressor_pager_lock(pager);
295
296 /*
297 * After memory_object_terminate both memory_object_init
298 * and a no-senders notification are possible, so we need
299 * to clean up our reference to the memory_object_control
300 * to prepare for a new init.
301 */
302
303 control = pager->cpgr_hdr.mo_control;
304 pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
305
306 compressor_pager_unlock(pager);
307
308 /*
309 * Now we deallocate our reference on the control.
310 */
311 memory_object_control_deallocate(control);
312 return KERN_SUCCESS;
313 }
314
315 void
compressor_memory_object_reference(memory_object_t mem_obj)316 compressor_memory_object_reference(
317 memory_object_t mem_obj)
318 {
319 compressor_pager_t pager;
320
321 compressor_pager_lookup(mem_obj, pager);
322 if (pager == NULL) {
323 return;
324 }
325
326 compressor_pager_lock(pager);
327 os_ref_retain_locked_raw(&pager->cpgr_references, NULL);
328 compressor_pager_unlock(pager);
329 }
330
331 void
compressor_memory_object_deallocate(memory_object_t mem_obj)332 compressor_memory_object_deallocate(
333 memory_object_t mem_obj)
334 {
335 compressor_pager_t pager;
336 unsigned int num_slots_freed;
337
338 /*
339 * Because we don't give out multiple first references
340 * for a memory object, there can't be a race
341 * between getting a deallocate call and creating
342 * a new reference for the object.
343 */
344
345 compressor_pager_lookup(mem_obj, pager);
346 if (pager == NULL) {
347 return;
348 }
349
350 compressor_pager_lock(pager);
351 if (os_ref_release_locked_raw(&pager->cpgr_references, NULL) > 0) {
352 compressor_pager_unlock(pager);
353 return;
354 }
355
356 /*
357 * We shouldn't get a deallocation call
358 * when the kernel has the object cached.
359 */
360 if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
361 panic("compressor_memory_object_deallocate(): bad request");
362 }
363
364 /*
365 * Unlock the pager (though there should be no one
366 * waiting for it).
367 */
368 compressor_pager_unlock(pager);
369
370 /* free the compressor slots */
371 unsigned int num_chunks;
372 unsigned int i;
373 compressor_slot_t *chunk;
374
375 num_chunks = compressor_pager_num_chunks(pager);
376 if (num_chunks > 1) {
377 /* we have an array of chunks */
378 for (i = 0; i < num_chunks; i++) {
379 chunk = pager->cpgr_slots.cpgr_islots[i];
380 if (chunk != NULL) {
381 num_slots_freed =
382 compressor_pager_slots_chunk_free(
383 chunk,
384 COMPRESSOR_SLOTS_PER_CHUNK,
385 0,
386 NULL);
387 pager->cpgr_slots.cpgr_islots[i] = NULL;
388 zfree_slot_array(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
389 }
390 }
391 kfree_type(compressor_slot_t *, num_chunks,
392 pager->cpgr_slots.cpgr_islots);
393 pager->cpgr_slots.cpgr_islots = NULL;
394 } else if (pager->cpgr_num_slots > 2) {
395 chunk = pager->cpgr_slots.cpgr_dslots;
396 num_slots_freed =
397 compressor_pager_slots_chunk_free(
398 chunk,
399 pager->cpgr_num_slots,
400 0,
401 NULL);
402 pager->cpgr_slots.cpgr_dslots = NULL;
403 zfree_slot_array(chunk,
404 (pager->cpgr_num_slots *
405 sizeof(pager->cpgr_slots.cpgr_dslots[0])));
406 } else {
407 chunk = &pager->cpgr_slots.cpgr_eslots[0];
408 num_slots_freed =
409 compressor_pager_slots_chunk_free(
410 chunk,
411 pager->cpgr_num_slots,
412 0,
413 NULL);
414 }
415
416 compressor_pager_lock_destroy(pager);
417 zfree(compressor_pager_zone, pager);
418 }
419
420 kern_return_t
compressor_memory_object_data_request(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t length,__unused vm_prot_t protection_required,__unused memory_object_fault_info_t fault_info)421 compressor_memory_object_data_request(
422 memory_object_t mem_obj,
423 memory_object_offset_t offset,
424 memory_object_cluster_size_t length,
425 __unused vm_prot_t protection_required,
426 __unused memory_object_fault_info_t fault_info)
427 {
428 compressor_pager_t pager;
429 kern_return_t kr;
430 compressor_slot_t *slot_p;
431
432 compressor_pager_stats.data_requests++;
433
434 /*
435 * Request must be on a page boundary and a multiple of pages.
436 */
437 if ((offset & PAGE_MASK) != 0 || (length & PAGE_MASK) != 0) {
438 panic("compressor_memory_object_data_request(): bad alignment");
439 }
440
441 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
442 panic("%s: offset 0x%llx overflow",
443 __FUNCTION__, (uint64_t) offset);
444 return KERN_FAILURE;
445 }
446
447 compressor_pager_lookup(mem_obj, pager);
448
449 if (length == 0) {
450 /* we're only querying the pager for this page */
451 } else {
452 panic("compressor: data_request");
453 }
454
455 /* find the compressor slot for that page */
456 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
457
458 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
459 /* out of range */
460 kr = KERN_FAILURE;
461 } else if (slot_p == NULL || *slot_p == 0) {
462 /* compressor does not have this page */
463 kr = KERN_FAILURE;
464 } else {
465 /* compressor does have this page */
466 kr = KERN_SUCCESS;
467 }
468 return kr;
469 }
470
471 /*
472 * memory_object_data_initialize: check whether we already have each page, and
473 * write it if we do not. The implementation is far from optimized, and
474 * also assumes that the default_pager is single-threaded.
475 */
476 /* It is questionable whether or not a pager should decide what is relevant */
477 /* and what is not in data sent from the kernel. Data initialize has been */
478 /* changed to copy back all data sent to it in preparation for its eventual */
479 /* merge with data return. It is the kernel that should decide what pages */
480 /* to write back. As of the writing of this note, this is indeed the case */
481 /* the kernel writes back one page at a time through this interface */
482
483 kern_return_t
compressor_memory_object_data_initialize(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t size)484 compressor_memory_object_data_initialize(
485 memory_object_t mem_obj,
486 memory_object_offset_t offset,
487 memory_object_cluster_size_t size)
488 {
489 compressor_pager_t pager;
490 memory_object_offset_t cur_offset;
491
492 compressor_pager_lookup(mem_obj, pager);
493 compressor_pager_lock(pager);
494
495 for (cur_offset = offset;
496 cur_offset < offset + size;
497 cur_offset += PAGE_SIZE) {
498 panic("do a data_return() if slot for this page is empty");
499 }
500
501 compressor_pager_unlock(pager);
502
503 return KERN_SUCCESS;
504 }
505
506
507 /*ARGSUSED*/
508 kern_return_t
compressor_memory_object_data_return(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t size,__unused memory_object_offset_t * resid_offset,__unused int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,__unused int upl_flags)509 compressor_memory_object_data_return(
510 __unused memory_object_t mem_obj,
511 __unused memory_object_offset_t offset,
512 __unused memory_object_cluster_size_t size,
513 __unused memory_object_offset_t *resid_offset,
514 __unused int *io_error,
515 __unused boolean_t dirty,
516 __unused boolean_t kernel_copy,
517 __unused int upl_flags)
518 {
519 panic("compressor: data_return");
520 return KERN_FAILURE;
521 }
522
523 /*
524 * Routine: default_pager_memory_object_create
525 * Purpose:
526 * Handle requests for memory objects from the
527 * kernel.
528 * Notes:
529 * Because we only give out the default memory
530 * manager port to the kernel, we don't have to
531 * be so paranoid about the contents.
532 */
533 kern_return_t
compressor_memory_object_create(memory_object_size_t new_size,memory_object_t * new_mem_obj)534 compressor_memory_object_create(
535 memory_object_size_t new_size,
536 memory_object_t *new_mem_obj)
537 {
538 compressor_pager_t pager;
539 unsigned int num_chunks;
540
541 if ((uint32_t)(new_size / PAGE_SIZE) != (new_size / PAGE_SIZE)) {
542 /* 32-bit overflow for number of pages */
543 panic("%s: size 0x%llx overflow",
544 __FUNCTION__, (uint64_t) new_size);
545 return KERN_INVALID_ARGUMENT;
546 }
547
548 pager = zalloc_flags(compressor_pager_zone, Z_WAITOK | Z_NOFAIL);
549
550 compressor_pager_lock_init(pager);
551 os_ref_init_raw(&pager->cpgr_references, NULL);
552 pager->cpgr_num_slots = (uint32_t)(new_size / PAGE_SIZE);
553 pager->cpgr_num_slots_occupied = 0;
554
555 num_chunks = compressor_pager_num_chunks(pager);
556 if (num_chunks > 1) {
557 pager->cpgr_slots.cpgr_islots = kalloc_type(compressor_slot_t *,
558 num_chunks, Z_WAITOK | Z_ZERO);
559 } else if (pager->cpgr_num_slots > 2) {
560 pager->cpgr_slots.cpgr_dslots = zalloc_slot_array(pager->cpgr_num_slots *
561 sizeof(pager->cpgr_slots.cpgr_dslots[0]), Z_WAITOK | Z_ZERO);
562 } else {
563 pager->cpgr_slots.cpgr_eslots[0] = 0;
564 pager->cpgr_slots.cpgr_eslots[1] = 0;
565 }
566
567 /*
568 * Set up associations between this memory object
569 * and this compressor_pager structure
570 */
571 pager->cpgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
572 pager->cpgr_hdr.mo_pager_ops = &compressor_pager_ops;
573 pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
574
575 *new_mem_obj = (memory_object_t) pager;
576 return KERN_SUCCESS;
577 }
578
579
580 unsigned int
compressor_pager_slots_chunk_free(compressor_slot_t * chunk,int num_slots,int flags,int * failures)581 compressor_pager_slots_chunk_free(
582 compressor_slot_t *chunk,
583 int num_slots,
584 int flags,
585 int *failures)
586 {
587 int i;
588 int retval;
589 unsigned int num_slots_freed;
590
591 if (failures) {
592 *failures = 0;
593 }
594 num_slots_freed = 0;
595 for (i = 0; i < num_slots; i++) {
596 if (chunk[i] != 0) {
597 retval = vm_compressor_free(&chunk[i], flags);
598
599 if (retval == 0) {
600 num_slots_freed++;
601 } else {
602 if (retval == -2) {
603 assert(flags & C_DONT_BLOCK);
604 }
605
606 if (failures) {
607 *failures += 1;
608 }
609 }
610 }
611 }
612 return num_slots_freed;
613 }
614
615 void
compressor_pager_slot_lookup(compressor_pager_t pager,boolean_t do_alloc,memory_object_offset_t offset,compressor_slot_t ** slot_pp)616 compressor_pager_slot_lookup(
617 compressor_pager_t pager,
618 boolean_t do_alloc,
619 memory_object_offset_t offset,
620 compressor_slot_t **slot_pp)
621 {
622 unsigned int num_chunks;
623 uint32_t page_num;
624 unsigned int chunk_idx;
625 int slot_idx;
626 compressor_slot_t *chunk;
627 compressor_slot_t *t_chunk;
628
629 page_num = (uint32_t)(offset / PAGE_SIZE);
630 if (page_num != (offset / PAGE_SIZE)) {
631 /* overflow */
632 panic("%s: offset 0x%llx overflow",
633 __FUNCTION__, (uint64_t) offset);
634 *slot_pp = NULL;
635 return;
636 }
637 if (page_num >= pager->cpgr_num_slots) {
638 /* out of range */
639 *slot_pp = NULL;
640 return;
641 }
642 num_chunks = compressor_pager_num_chunks(pager);
643 if (num_chunks > 1) {
644 /* we have an array of chunks */
645 chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK;
646 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
647
648 if (chunk == NULL && do_alloc) {
649 t_chunk = zalloc_slot_array(COMPRESSOR_SLOTS_CHUNK_SIZE,
650 Z_WAITOK | Z_ZERO);
651
652 compressor_pager_lock(pager);
653
654 if ((chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]) == NULL) {
655 /*
656 * On some platforms, the memory stores from
657 * the bzero(t_chunk) above might not have been
658 * made visible and another thread might see
659 * the contents of this new chunk before it's
660 * been fully zero-filled.
661 * This memory barrier should take care of this
662 * according to the platform requirements.
663 */
664 os_atomic_thread_fence(release);
665
666 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
667 t_chunk = NULL;
668 }
669 compressor_pager_unlock(pager);
670
671 if (t_chunk) {
672 zfree_slot_array(t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
673 }
674 }
675 if (chunk == NULL) {
676 *slot_pp = NULL;
677 } else {
678 slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK;
679 *slot_pp = &chunk[slot_idx];
680 }
681 } else if (pager->cpgr_num_slots > 2) {
682 slot_idx = page_num;
683 *slot_pp = &pager->cpgr_slots.cpgr_dslots[slot_idx];
684 } else {
685 slot_idx = page_num;
686 *slot_pp = &pager->cpgr_slots.cpgr_eslots[slot_idx];
687 }
688 }
689
690 #if defined(__LP64__)
691 __startup_func
692 static void
vm_compressor_slots_init(void)693 vm_compressor_slots_init(void)
694 {
695 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
696 compressor_slots_zones[idx] = zone_create(
697 compressor_slots_zones_names[idx],
698 compressor_slots_zones_sizes[idx],
699 ZC_PGZ_USE_GUARDS | ZC_VM | ZC_NOTBITAG);
700 }
701 }
702 STARTUP(ZALLOC, STARTUP_RANK_MIDDLE, vm_compressor_slots_init);
703 #endif /* defined(__LP64__) */
704
705 static compressor_slot_t *
zalloc_slot_array(size_t size,zalloc_flags_t flags)706 zalloc_slot_array(size_t size, zalloc_flags_t flags)
707 {
708 #if defined(__LP64__)
709 compressor_slot_t *slots = NULL;
710
711 assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
712 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
713 if (size > compressor_slots_zones_sizes[idx]) {
714 continue;
715 }
716 slots = zalloc_flags(compressor_slots_zones[idx], flags);
717 break;
718 }
719 return slots;
720 #else /* defined(__LP64__) */
721 return kalloc_data(size, flags);
722 #endif /* !defined(__LP64__) */
723 }
724
725 static void
zfree_slot_array(compressor_slot_t * slots,size_t size)726 zfree_slot_array(compressor_slot_t *slots, size_t size)
727 {
728 #if defined(__LP64__)
729 assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
730 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
731 if (size > compressor_slots_zones_sizes[idx]) {
732 continue;
733 }
734 zfree(compressor_slots_zones[idx], slots);
735 break;
736 }
737 #else /* defined(__LP64__) */
738 kfree_data(slots, size);
739 #endif /* !defined(__LP64__) */
740 }
741
742 kern_return_t
vm_compressor_pager_put(memory_object_t mem_obj,memory_object_offset_t offset,ppnum_t ppnum,void ** current_chead,char * scratch_buf,int * compressed_count_delta_p)743 vm_compressor_pager_put(
744 memory_object_t mem_obj,
745 memory_object_offset_t offset,
746 ppnum_t ppnum,
747 void **current_chead,
748 char *scratch_buf,
749 int *compressed_count_delta_p)
750 {
751 compressor_pager_t pager;
752 compressor_slot_t *slot_p;
753
754 compressor_pager_stats.put++;
755
756 *compressed_count_delta_p = 0;
757
758 /* This routine is called by the pageout thread. The pageout thread */
759 /* cannot be blocked by read activities unless the read activities */
760 /* Therefore the grant of vs lock must be done on a try versus a */
761 /* blocking basis. The code below relies on the fact that the */
762 /* interface is synchronous. Should this interface be again async */
763 /* for some type of pager in the future the pages will have to be */
764 /* returned through a separate, asynchronous path. */
765
766 compressor_pager_lookup(mem_obj, pager);
767
768 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
769 /* overflow */
770 panic("%s: offset 0x%llx overflow",
771 __FUNCTION__, (uint64_t) offset);
772 return KERN_RESOURCE_SHORTAGE;
773 }
774
775 compressor_pager_slot_lookup(pager, TRUE, offset, &slot_p);
776
777 if (slot_p == NULL) {
778 /* out of range ? */
779 panic("vm_compressor_pager_put: out of range");
780 }
781 if (*slot_p != 0) {
782 /*
783 * Already compressed: forget about the old one.
784 *
785 * This can happen after a vm_object_do_collapse() when
786 * the "backing_object" had some pages paged out and the
787 * "object" had an equivalent page resident.
788 */
789 vm_compressor_free(slot_p, 0);
790 *compressed_count_delta_p -= 1;
791 }
792
793 /*
794 * If the compressor operation succeeds, we presumably don't need to
795 * undo any previous WIMG update, as all live mappings should be
796 * disconnected.
797 */
798
799 if (vm_compressor_put(ppnum, slot_p, current_chead, scratch_buf)) {
800 return KERN_RESOURCE_SHORTAGE;
801 }
802 *compressed_count_delta_p += 1;
803
804 return KERN_SUCCESS;
805 }
806
807
808 kern_return_t
vm_compressor_pager_get(memory_object_t mem_obj,memory_object_offset_t offset,ppnum_t ppnum,int * my_fault_type,int flags,int * compressed_count_delta_p)809 vm_compressor_pager_get(
810 memory_object_t mem_obj,
811 memory_object_offset_t offset,
812 ppnum_t ppnum,
813 int *my_fault_type,
814 int flags,
815 int *compressed_count_delta_p)
816 {
817 compressor_pager_t pager;
818 kern_return_t kr;
819 compressor_slot_t *slot_p;
820
821 compressor_pager_stats.get++;
822
823 *compressed_count_delta_p = 0;
824
825 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
826 panic("%s: offset 0x%llx overflow",
827 __FUNCTION__, (uint64_t) offset);
828 return KERN_MEMORY_ERROR;
829 }
830
831 compressor_pager_lookup(mem_obj, pager);
832
833 /* find the compressor slot for that page */
834 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
835
836 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
837 /* out of range */
838 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_OUT_OF_RANGE), 0 /* arg */);
839 kr = KERN_MEMORY_FAILURE;
840 } else if (slot_p == NULL || *slot_p == 0) {
841 /* compressor does not have this page */
842 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_NO_PAGE), 0 /* arg */);
843 kr = KERN_MEMORY_ERROR;
844 } else {
845 /* compressor does have this page */
846 kr = KERN_SUCCESS;
847 }
848 *my_fault_type = DBG_COMPRESSOR_FAULT;
849
850 if (kr == KERN_SUCCESS) {
851 int retval;
852
853 /* get the page from the compressor */
854 retval = vm_compressor_get(ppnum, slot_p, flags);
855 if (retval == -1) {
856 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_DECOMPRESS_FAILED), 0 /* arg */);
857 kr = KERN_MEMORY_FAILURE;
858 } else if (retval == 1) {
859 *my_fault_type = DBG_COMPRESSOR_SWAPIN_FAULT;
860 } else if (retval == -2) {
861 assert((flags & C_DONT_BLOCK));
862 /*
863 * Not a fatal failure because we just retry with a blocking get later. So we skip ktriage to avoid noise.
864 */
865 kr = KERN_FAILURE;
866 }
867 }
868
869 if (kr == KERN_SUCCESS) {
870 assert(slot_p != NULL);
871 if (*slot_p != 0) {
872 /*
873 * We got the page for a copy-on-write fault
874 * and we kept the original in place. Slot
875 * is still occupied.
876 */
877 } else {
878 *compressed_count_delta_p -= 1;
879 }
880 }
881
882 return kr;
883 }
884
885 unsigned int
vm_compressor_pager_state_clr(memory_object_t mem_obj,memory_object_offset_t offset)886 vm_compressor_pager_state_clr(
887 memory_object_t mem_obj,
888 memory_object_offset_t offset)
889 {
890 compressor_pager_t pager;
891 compressor_slot_t *slot_p;
892 unsigned int num_slots_freed;
893
894 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
895
896 compressor_pager_stats.state_clr++;
897
898 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
899 /* overflow */
900 panic("%s: offset 0x%llx overflow",
901 __FUNCTION__, (uint64_t) offset);
902 return 0;
903 }
904
905 compressor_pager_lookup(mem_obj, pager);
906
907 /* find the compressor slot for that page */
908 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
909
910 num_slots_freed = 0;
911 if (slot_p && *slot_p != 0) {
912 vm_compressor_free(slot_p, 0);
913 num_slots_freed++;
914 assert(*slot_p == 0);
915 }
916
917 return num_slots_freed;
918 }
919
920 vm_external_state_t
vm_compressor_pager_state_get(memory_object_t mem_obj,memory_object_offset_t offset)921 vm_compressor_pager_state_get(
922 memory_object_t mem_obj,
923 memory_object_offset_t offset)
924 {
925 compressor_pager_t pager;
926 compressor_slot_t *slot_p;
927
928 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
929
930 compressor_pager_stats.state_get++;
931
932 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
933 /* overflow */
934 panic("%s: offset 0x%llx overflow",
935 __FUNCTION__, (uint64_t) offset);
936 return VM_EXTERNAL_STATE_ABSENT;
937 }
938
939 compressor_pager_lookup(mem_obj, pager);
940
941 /* find the compressor slot for that page */
942 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
943
944 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
945 /* out of range */
946 return VM_EXTERNAL_STATE_ABSENT;
947 } else if (slot_p == NULL || *slot_p == 0) {
948 /* compressor does not have this page */
949 return VM_EXTERNAL_STATE_ABSENT;
950 } else {
951 /* compressor does have this page */
952 return VM_EXTERNAL_STATE_EXISTS;
953 }
954 }
955
956 unsigned int
vm_compressor_pager_reap_pages(memory_object_t mem_obj,int flags)957 vm_compressor_pager_reap_pages(
958 memory_object_t mem_obj,
959 int flags)
960 {
961 compressor_pager_t pager;
962 unsigned int num_chunks;
963 int failures;
964 unsigned int i;
965 compressor_slot_t *chunk;
966 unsigned int num_slots_freed;
967
968 compressor_pager_lookup(mem_obj, pager);
969 if (pager == NULL) {
970 return 0;
971 }
972
973 compressor_pager_lock(pager);
974
975 /* reap the compressor slots */
976 num_slots_freed = 0;
977
978 num_chunks = compressor_pager_num_chunks(pager);
979 if (num_chunks > 1) {
980 /* we have an array of chunks */
981 for (i = 0; i < num_chunks; i++) {
982 chunk = pager->cpgr_slots.cpgr_islots[i];
983 if (chunk != NULL) {
984 num_slots_freed +=
985 compressor_pager_slots_chunk_free(
986 chunk,
987 COMPRESSOR_SLOTS_PER_CHUNK,
988 flags,
989 &failures);
990 if (failures == 0) {
991 pager->cpgr_slots.cpgr_islots[i] = NULL;
992 zfree_slot_array(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
993 }
994 }
995 }
996 } else if (pager->cpgr_num_slots > 2) {
997 chunk = pager->cpgr_slots.cpgr_dslots;
998 num_slots_freed +=
999 compressor_pager_slots_chunk_free(
1000 chunk,
1001 pager->cpgr_num_slots,
1002 flags,
1003 NULL);
1004 } else {
1005 chunk = &pager->cpgr_slots.cpgr_eslots[0];
1006 num_slots_freed +=
1007 compressor_pager_slots_chunk_free(
1008 chunk,
1009 pager->cpgr_num_slots,
1010 flags,
1011 NULL);
1012 }
1013
1014 compressor_pager_unlock(pager);
1015
1016 return num_slots_freed;
1017 }
1018
1019 void
vm_compressor_pager_transfer(memory_object_t dst_mem_obj,memory_object_offset_t dst_offset,memory_object_t src_mem_obj,memory_object_offset_t src_offset)1020 vm_compressor_pager_transfer(
1021 memory_object_t dst_mem_obj,
1022 memory_object_offset_t dst_offset,
1023 memory_object_t src_mem_obj,
1024 memory_object_offset_t src_offset)
1025 {
1026 compressor_pager_t src_pager, dst_pager;
1027 compressor_slot_t *src_slot_p, *dst_slot_p;
1028
1029 compressor_pager_stats.transfer++;
1030
1031 /* find the compressor slot for the destination */
1032 compressor_pager_lookup(dst_mem_obj, dst_pager);
1033 assert(dst_offset / PAGE_SIZE < dst_pager->cpgr_num_slots);
1034 compressor_pager_slot_lookup(dst_pager, TRUE, dst_offset, &dst_slot_p);
1035 assert(dst_slot_p != NULL);
1036 assert(*dst_slot_p == 0);
1037
1038 /* find the compressor slot for the source */
1039 compressor_pager_lookup(src_mem_obj, src_pager);
1040 assert(src_offset / PAGE_SIZE < src_pager->cpgr_num_slots);
1041 compressor_pager_slot_lookup(src_pager, FALSE, src_offset, &src_slot_p);
1042 assert(src_slot_p != NULL);
1043 assert(*src_slot_p != 0);
1044
1045 /* transfer the slot from source to destination */
1046 vm_compressor_transfer(dst_slot_p, src_slot_p);
1047 OSAddAtomic(-1, &src_pager->cpgr_num_slots_occupied);
1048 OSAddAtomic(+1, &dst_pager->cpgr_num_slots_occupied);
1049 }
1050
1051 memory_object_offset_t
vm_compressor_pager_next_compressed(memory_object_t mem_obj,memory_object_offset_t offset)1052 vm_compressor_pager_next_compressed(
1053 memory_object_t mem_obj,
1054 memory_object_offset_t offset)
1055 {
1056 compressor_pager_t pager;
1057 unsigned int num_chunks;
1058 uint32_t page_num;
1059 unsigned int chunk_idx;
1060 uint32_t slot_idx;
1061 compressor_slot_t *chunk;
1062
1063 compressor_pager_lookup(mem_obj, pager);
1064
1065 page_num = (uint32_t)(offset / PAGE_SIZE);
1066 if (page_num != (offset / PAGE_SIZE)) {
1067 /* overflow */
1068 return (memory_object_offset_t) -1;
1069 }
1070 if (page_num >= pager->cpgr_num_slots) {
1071 /* out of range */
1072 return (memory_object_offset_t) -1;
1073 }
1074
1075 num_chunks = compressor_pager_num_chunks(pager);
1076 if (num_chunks == 1) {
1077 if (pager->cpgr_num_slots > 2) {
1078 chunk = pager->cpgr_slots.cpgr_dslots;
1079 } else {
1080 chunk = &pager->cpgr_slots.cpgr_eslots[0];
1081 }
1082 for (slot_idx = page_num;
1083 slot_idx < pager->cpgr_num_slots;
1084 slot_idx++) {
1085 if (chunk[slot_idx] != 0) {
1086 /* found a non-NULL slot in this chunk */
1087 return (memory_object_offset_t) slot_idx *
1088 PAGE_SIZE;
1089 }
1090 }
1091 return (memory_object_offset_t) -1;
1092 }
1093
1094 /* we have an array of chunks; find the next non-NULL chunk */
1095 chunk = NULL;
1096 for (chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK,
1097 slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK;
1098 chunk_idx < num_chunks;
1099 chunk_idx++,
1100 slot_idx = 0) {
1101 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
1102 if (chunk == NULL) {
1103 /* no chunk here: try the next one */
1104 continue;
1105 }
1106 /* search for an occupied slot in this chunk */
1107 for (;
1108 slot_idx < COMPRESSOR_SLOTS_PER_CHUNK;
1109 slot_idx++) {
1110 if (chunk[slot_idx] != 0) {
1111 /* found an occupied slot in this chunk */
1112 uint32_t next_slot;
1113
1114 next_slot = ((chunk_idx *
1115 COMPRESSOR_SLOTS_PER_CHUNK) +
1116 slot_idx);
1117 if (next_slot >= pager->cpgr_num_slots) {
1118 /* went beyond end of object */
1119 return (memory_object_offset_t) -1;
1120 }
1121 return (memory_object_offset_t) next_slot *
1122 PAGE_SIZE;
1123 }
1124 }
1125 }
1126 return (memory_object_offset_t) -1;
1127 }
1128
1129 unsigned int
vm_compressor_pager_get_count(memory_object_t mem_obj)1130 vm_compressor_pager_get_count(
1131 memory_object_t mem_obj)
1132 {
1133 compressor_pager_t pager;
1134
1135 compressor_pager_lookup(mem_obj, pager);
1136 if (pager == NULL) {
1137 return 0;
1138 }
1139
1140 /*
1141 * The caller should have the VM object locked and one
1142 * needs that lock to do a page-in or page-out, so no
1143 * need to lock the pager here.
1144 */
1145 assert(pager->cpgr_num_slots_occupied >= 0);
1146
1147 return pager->cpgr_num_slots_occupied;
1148 }
1149
1150 void
vm_compressor_pager_count(memory_object_t mem_obj,int compressed_count_delta,boolean_t shared_lock,vm_object_t object __unused)1151 vm_compressor_pager_count(
1152 memory_object_t mem_obj,
1153 int compressed_count_delta,
1154 boolean_t shared_lock,
1155 vm_object_t object __unused)
1156 {
1157 compressor_pager_t pager;
1158
1159 if (compressed_count_delta == 0) {
1160 return;
1161 }
1162
1163 compressor_pager_lookup(mem_obj, pager);
1164 if (pager == NULL) {
1165 return;
1166 }
1167
1168 if (compressed_count_delta < 0) {
1169 assert(pager->cpgr_num_slots_occupied >=
1170 (unsigned int) -compressed_count_delta);
1171 }
1172
1173 /*
1174 * The caller should have the VM object locked,
1175 * shared or exclusive.
1176 */
1177 if (shared_lock) {
1178 vm_object_lock_assert_shared(object);
1179 OSAddAtomic(compressed_count_delta,
1180 &pager->cpgr_num_slots_occupied);
1181 } else {
1182 vm_object_lock_assert_exclusive(object);
1183 pager->cpgr_num_slots_occupied += compressed_count_delta;
1184 }
1185 }
1186
1187 #if CONFIG_FREEZE
1188 kern_return_t
vm_compressor_pager_relocate(memory_object_t mem_obj,memory_object_offset_t offset,void ** current_chead)1189 vm_compressor_pager_relocate(
1190 memory_object_t mem_obj,
1191 memory_object_offset_t offset,
1192 void **current_chead)
1193 {
1194 /*
1195 * Has the page at this offset been compressed?
1196 */
1197
1198 compressor_slot_t *slot_p;
1199 compressor_pager_t dst_pager;
1200
1201 assert(mem_obj);
1202
1203 compressor_pager_lookup(mem_obj, dst_pager);
1204 if (dst_pager == NULL) {
1205 return KERN_FAILURE;
1206 }
1207
1208 compressor_pager_slot_lookup(dst_pager, FALSE, offset, &slot_p);
1209 return vm_compressor_relocate(current_chead, slot_p);
1210 }
1211 #endif /* CONFIG_FREEZE */
1212
1213 #if DEVELOPMENT || DEBUG
1214
1215 kern_return_t
vm_compressor_pager_inject_error(memory_object_t mem_obj,memory_object_offset_t offset)1216 vm_compressor_pager_inject_error(memory_object_t mem_obj,
1217 memory_object_offset_t offset)
1218 {
1219 kern_return_t result = KERN_FAILURE;
1220 compressor_slot_t *slot_p;
1221 compressor_pager_t pager;
1222
1223 assert(mem_obj);
1224
1225 compressor_pager_lookup(mem_obj, pager);
1226 if (pager != NULL) {
1227 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
1228 if (slot_p != NULL && *slot_p != 0) {
1229 vm_compressor_inject_error(slot_p);
1230 result = KERN_SUCCESS;
1231 }
1232 }
1233
1234 return result;
1235 }
1236
1237 #endif
1238