1 /*
2 * Copyright (c) 2019-2020 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28 /*
29 * @OSF_COPYRIGHT@
30 */
31 /*
32 * Mach Operating System
33 * Copyright (c) 1991,1990,1989 Carnegie Mellon University
34 * All Rights Reserved.
35 *
36 * Permission to use, copy, modify and distribute this software and its
37 * documentation is hereby granted, provided that both the copyright
38 * notice and this permission notice appear in all copies of the
39 * software, derivative works or modified versions, and any portions
40 * thereof, and that both notices appear in supporting documentation.
41 *
42 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
43 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR
44 * ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
45 *
46 * Carnegie Mellon requests users of this software to return to
47 *
48 * Software Distribution Coordinator or [email protected]
49 * School of Computer Science
50 * Carnegie Mellon University
51 * Pittsburgh PA 15213-3890
52 *
53 * any improvements or extensions that they make and grant Carnegie Mellon
54 * the rights to redistribute these changes.
55 */
56
57 /*
58 * Compressor Pager.
59 * Memory Object Management.
60 */
61
62 #include <kern/host_statistics.h>
63 #include <kern/kalloc.h>
64 #include <kern/ipc_kobject.h>
65
66 #include <machine/atomic.h>
67
68 #include <mach/memory_object_control.h>
69 #include <mach/memory_object_types.h>
70 #include <mach/upl.h>
71
72 #include <vm/memory_object.h>
73 #include <vm/vm_compressor_pager.h>
74 #include <vm/vm_external.h>
75 #include <vm/vm_pageout.h>
76 #include <vm/vm_protos.h>
77
78 #include <sys/kdebug_triage.h>
79
80 /* memory_object interfaces */
81 void compressor_memory_object_reference(memory_object_t mem_obj);
82 void compressor_memory_object_deallocate(memory_object_t mem_obj);
83 kern_return_t compressor_memory_object_init(
84 memory_object_t mem_obj,
85 memory_object_control_t control,
86 memory_object_cluster_size_t pager_page_size);
87 kern_return_t compressor_memory_object_terminate(memory_object_t mem_obj);
88 kern_return_t compressor_memory_object_data_request(
89 memory_object_t mem_obj,
90 memory_object_offset_t offset,
91 memory_object_cluster_size_t length,
92 __unused vm_prot_t protection_required,
93 memory_object_fault_info_t fault_info);
94 kern_return_t compressor_memory_object_data_return(
95 memory_object_t mem_obj,
96 memory_object_offset_t offset,
97 memory_object_cluster_size_t size,
98 __unused memory_object_offset_t *resid_offset,
99 __unused int *io_error,
100 __unused boolean_t dirty,
101 __unused boolean_t kernel_copy,
102 __unused int upl_flags);
103 kern_return_t compressor_memory_object_data_initialize(
104 memory_object_t mem_obj,
105 memory_object_offset_t offset,
106 memory_object_cluster_size_t size);
107 kern_return_t compressor_memory_object_map(
108 __unused memory_object_t mem_obj,
109 __unused vm_prot_t prot);
110 kern_return_t compressor_memory_object_last_unmap(memory_object_t mem_obj);
111
112 const struct memory_object_pager_ops compressor_pager_ops = {
113 .memory_object_reference = compressor_memory_object_reference,
114 .memory_object_deallocate = compressor_memory_object_deallocate,
115 .memory_object_init = compressor_memory_object_init,
116 .memory_object_terminate = compressor_memory_object_terminate,
117 .memory_object_data_request = compressor_memory_object_data_request,
118 .memory_object_data_return = compressor_memory_object_data_return,
119 .memory_object_data_initialize = compressor_memory_object_data_initialize,
120 .memory_object_map = compressor_memory_object_map,
121 .memory_object_last_unmap = compressor_memory_object_last_unmap,
122 .memory_object_backing_object = NULL,
123 .memory_object_pager_name = "compressor pager"
124 };
125
126 /* internal data structures */
127
128 struct {
129 uint64_t data_returns;
130 uint64_t data_requests;
131 uint64_t put;
132 uint64_t get;
133 uint64_t state_clr;
134 uint64_t state_get;
135 uint64_t transfer;
136 } compressor_pager_stats;
137
138 typedef int compressor_slot_t;
139
140 typedef struct compressor_pager {
141 /* mandatory generic header */
142 struct memory_object cpgr_hdr;
143
144 /* pager-specific data */
145 lck_mtx_t cpgr_lock;
146 #if MEMORY_OBJECT_HAS_REFCOUNT
147 #define cpgr_references cpgr_hdr.mo_ref
148 #else
149 os_ref_atomic_t cpgr_references;
150 #endif
151 unsigned int cpgr_num_slots;
152 unsigned int cpgr_num_slots_occupied;
153 union {
154 compressor_slot_t cpgr_eslots[2]; /* embedded slots */
155 compressor_slot_t *cpgr_dslots; /* direct slots */
156 compressor_slot_t **cpgr_islots; /* indirect slots */
157 } cpgr_slots;
158 } *compressor_pager_t;
159
160 #define compressor_pager_lookup(_mem_obj_, _cpgr_) \
161 MACRO_BEGIN \
162 if (_mem_obj_ == NULL || \
163 _mem_obj_->mo_pager_ops != &compressor_pager_ops) { \
164 _cpgr_ = NULL; \
165 } else { \
166 _cpgr_ = (compressor_pager_t) _mem_obj_; \
167 } \
168 MACRO_END
169
170 /* embedded slot pointers in compressor_pager get packed, so VA restricted */
171 static ZONE_DEFINE_TYPE(compressor_pager_zone, "compressor_pager",
172 struct compressor_pager, ZC_NOENCRYPT | ZC_VM_LP64 | ZC_NOTBITAG);
173
174 LCK_GRP_DECLARE(compressor_pager_lck_grp, "compressor_pager");
175
176 #define compressor_pager_lock(_cpgr_) \
177 lck_mtx_lock(&(_cpgr_)->cpgr_lock)
178 #define compressor_pager_unlock(_cpgr_) \
179 lck_mtx_unlock(&(_cpgr_)->cpgr_lock)
180 #define compressor_pager_lock_init(_cpgr_) \
181 lck_mtx_init(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp, LCK_ATTR_NULL)
182 #define compressor_pager_lock_destroy(_cpgr_) \
183 lck_mtx_destroy(&(_cpgr_)->cpgr_lock, &compressor_pager_lck_grp)
184
185 #define COMPRESSOR_SLOTS_CHUNK_SIZE (512)
186 #define COMPRESSOR_SLOTS_PER_CHUNK (COMPRESSOR_SLOTS_CHUNK_SIZE / sizeof (compressor_slot_t))
187
188 /* forward declarations */
189 unsigned int compressor_pager_slots_chunk_free(compressor_slot_t *chunk,
190 int num_slots,
191 int flags,
192 int *failures);
193 void compressor_pager_slot_lookup(
194 compressor_pager_t pager,
195 boolean_t do_alloc,
196 memory_object_offset_t offset,
197 compressor_slot_t **slot_pp);
198
199 #if defined(__LP64__)
200
201 /* restricted VA zones for slots */
202
203 #define NUM_SLOTS_ZONES 3
204
205 static const size_t compressor_slots_zones_sizes[NUM_SLOTS_ZONES] = {
206 16,
207 64,
208 COMPRESSOR_SLOTS_CHUNK_SIZE
209 };
210
211 static const char * compressor_slots_zones_names[NUM_SLOTS_ZONES] = {
212 "compressor_slots.16",
213 "compressor_slots.64",
214 "compressor_slots.512"
215 };
216
217 static zone_t
218 compressor_slots_zones[NUM_SLOTS_ZONES];
219
220 #endif /* defined(__LP64__) */
221
222 static void
223 zfree_slot_array(compressor_slot_t *slots, size_t size);
224 static compressor_slot_t *
225 zalloc_slot_array(size_t size, zalloc_flags_t);
226
227 static inline unsigned int
compressor_pager_num_chunks(compressor_pager_t pager)228 compressor_pager_num_chunks(
229 compressor_pager_t pager)
230 {
231 unsigned int num_chunks;
232
233 num_chunks = pager->cpgr_num_slots / COMPRESSOR_SLOTS_PER_CHUNK;
234 if (num_chunks * COMPRESSOR_SLOTS_PER_CHUNK < pager->cpgr_num_slots) {
235 num_chunks++;
236 }
237 return num_chunks;
238 }
239
240 kern_return_t
compressor_memory_object_init(memory_object_t mem_obj,memory_object_control_t control,__unused memory_object_cluster_size_t pager_page_size)241 compressor_memory_object_init(
242 memory_object_t mem_obj,
243 memory_object_control_t control,
244 __unused memory_object_cluster_size_t pager_page_size)
245 {
246 compressor_pager_t pager;
247
248 assert(pager_page_size == PAGE_SIZE);
249
250 memory_object_control_reference(control);
251
252 compressor_pager_lookup(mem_obj, pager);
253 compressor_pager_lock(pager);
254
255 if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
256 panic("compressor_memory_object_init: bad request");
257 }
258 pager->cpgr_hdr.mo_control = control;
259
260 compressor_pager_unlock(pager);
261
262 return KERN_SUCCESS;
263 }
264
265 kern_return_t
compressor_memory_object_map(__unused memory_object_t mem_obj,__unused vm_prot_t prot)266 compressor_memory_object_map(
267 __unused memory_object_t mem_obj,
268 __unused vm_prot_t prot)
269 {
270 panic("compressor_memory_object_map");
271 return KERN_FAILURE;
272 }
273
274 kern_return_t
compressor_memory_object_last_unmap(__unused memory_object_t mem_obj)275 compressor_memory_object_last_unmap(
276 __unused memory_object_t mem_obj)
277 {
278 panic("compressor_memory_object_last_unmap");
279 return KERN_FAILURE;
280 }
281
282 kern_return_t
compressor_memory_object_terminate(memory_object_t mem_obj)283 compressor_memory_object_terminate(
284 memory_object_t mem_obj)
285 {
286 memory_object_control_t control;
287 compressor_pager_t pager;
288
289 /*
290 * control port is a receive right, not a send right.
291 */
292
293 compressor_pager_lookup(mem_obj, pager);
294 compressor_pager_lock(pager);
295
296 /*
297 * After memory_object_terminate both memory_object_init
298 * and a no-senders notification are possible, so we need
299 * to clean up our reference to the memory_object_control
300 * to prepare for a new init.
301 */
302
303 control = pager->cpgr_hdr.mo_control;
304 pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
305
306 compressor_pager_unlock(pager);
307
308 /*
309 * Now we deallocate our reference on the control.
310 */
311 memory_object_control_deallocate(control);
312 return KERN_SUCCESS;
313 }
314
315 void
compressor_memory_object_reference(memory_object_t mem_obj)316 compressor_memory_object_reference(
317 memory_object_t mem_obj)
318 {
319 compressor_pager_t pager;
320
321 compressor_pager_lookup(mem_obj, pager);
322 if (pager == NULL) {
323 return;
324 }
325
326 compressor_pager_lock(pager);
327 os_ref_retain_locked_raw(&pager->cpgr_references, NULL);
328 compressor_pager_unlock(pager);
329 }
330
331 void
compressor_memory_object_deallocate(memory_object_t mem_obj)332 compressor_memory_object_deallocate(
333 memory_object_t mem_obj)
334 {
335 compressor_pager_t pager;
336 unsigned int num_slots_freed;
337
338 /*
339 * Because we don't give out multiple first references
340 * for a memory object, there can't be a race
341 * between getting a deallocate call and creating
342 * a new reference for the object.
343 */
344
345 compressor_pager_lookup(mem_obj, pager);
346 if (pager == NULL) {
347 return;
348 }
349
350 compressor_pager_lock(pager);
351 if (os_ref_release_locked_raw(&pager->cpgr_references, NULL) > 0) {
352 compressor_pager_unlock(pager);
353 return;
354 }
355
356 /*
357 * We shouldn't get a deallocation call
358 * when the kernel has the object cached.
359 */
360 if (pager->cpgr_hdr.mo_control != MEMORY_OBJECT_CONTROL_NULL) {
361 panic("compressor_memory_object_deallocate(): bad request");
362 }
363
364 /*
365 * Unlock the pager (though there should be no one
366 * waiting for it).
367 */
368 compressor_pager_unlock(pager);
369
370 /* free the compressor slots */
371 unsigned int num_chunks;
372 unsigned int i;
373 compressor_slot_t *chunk;
374
375 num_chunks = compressor_pager_num_chunks(pager);
376 if (num_chunks > 1) {
377 /* we have an array of chunks */
378 for (i = 0; i < num_chunks; i++) {
379 chunk = pager->cpgr_slots.cpgr_islots[i];
380 if (chunk != NULL) {
381 num_slots_freed =
382 compressor_pager_slots_chunk_free(
383 chunk,
384 COMPRESSOR_SLOTS_PER_CHUNK,
385 0,
386 NULL);
387 pager->cpgr_slots.cpgr_islots[i] = NULL;
388 zfree_slot_array(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
389 }
390 }
391 kfree_type(compressor_slot_t *, num_chunks,
392 pager->cpgr_slots.cpgr_islots);
393 pager->cpgr_slots.cpgr_islots = NULL;
394 } else if (pager->cpgr_num_slots > 2) {
395 chunk = pager->cpgr_slots.cpgr_dslots;
396 num_slots_freed =
397 compressor_pager_slots_chunk_free(
398 chunk,
399 pager->cpgr_num_slots,
400 0,
401 NULL);
402 pager->cpgr_slots.cpgr_dslots = NULL;
403 zfree_slot_array(chunk,
404 (pager->cpgr_num_slots *
405 sizeof(pager->cpgr_slots.cpgr_dslots[0])));
406 } else {
407 chunk = &pager->cpgr_slots.cpgr_eslots[0];
408 num_slots_freed =
409 compressor_pager_slots_chunk_free(
410 chunk,
411 pager->cpgr_num_slots,
412 0,
413 NULL);
414 }
415
416 compressor_pager_lock_destroy(pager);
417 zfree(compressor_pager_zone, pager);
418 }
419
420 kern_return_t
compressor_memory_object_data_request(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t length,__unused vm_prot_t protection_required,__unused memory_object_fault_info_t fault_info)421 compressor_memory_object_data_request(
422 memory_object_t mem_obj,
423 memory_object_offset_t offset,
424 memory_object_cluster_size_t length,
425 __unused vm_prot_t protection_required,
426 __unused memory_object_fault_info_t fault_info)
427 {
428 compressor_pager_t pager;
429 kern_return_t kr;
430 compressor_slot_t *slot_p;
431
432 compressor_pager_stats.data_requests++;
433
434 /*
435 * Request must be on a page boundary and a multiple of pages.
436 */
437 if ((offset & PAGE_MASK) != 0 || (length & PAGE_MASK) != 0) {
438 panic("compressor_memory_object_data_request(): bad alignment");
439 }
440
441 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
442 panic("%s: offset 0x%llx overflow",
443 __FUNCTION__, (uint64_t) offset);
444 return KERN_FAILURE;
445 }
446
447 compressor_pager_lookup(mem_obj, pager);
448
449 if (length == 0) {
450 /* we're only querying the pager for this page */
451 } else {
452 panic("compressor: data_request");
453 }
454
455 /* find the compressor slot for that page */
456 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
457
458 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
459 /* out of range */
460 kr = KERN_FAILURE;
461 } else if (slot_p == NULL || *slot_p == 0) {
462 /* compressor does not have this page */
463 kr = KERN_FAILURE;
464 } else {
465 /* compressor does have this page */
466 kr = KERN_SUCCESS;
467 }
468 return kr;
469 }
470
471 /*
472 * memory_object_data_initialize: check whether we already have each page, and
473 * write it if we do not. The implementation is far from optimized, and
474 * also assumes that the default_pager is single-threaded.
475 */
476 /* It is questionable whether or not a pager should decide what is relevant */
477 /* and what is not in data sent from the kernel. Data initialize has been */
478 /* changed to copy back all data sent to it in preparation for its eventual */
479 /* merge with data return. It is the kernel that should decide what pages */
480 /* to write back. As of the writing of this note, this is indeed the case */
481 /* the kernel writes back one page at a time through this interface */
482
483 kern_return_t
compressor_memory_object_data_initialize(memory_object_t mem_obj,memory_object_offset_t offset,memory_object_cluster_size_t size)484 compressor_memory_object_data_initialize(
485 memory_object_t mem_obj,
486 memory_object_offset_t offset,
487 memory_object_cluster_size_t size)
488 {
489 compressor_pager_t pager;
490 memory_object_offset_t cur_offset;
491
492 compressor_pager_lookup(mem_obj, pager);
493 compressor_pager_lock(pager);
494
495 for (cur_offset = offset;
496 cur_offset < offset + size;
497 cur_offset += PAGE_SIZE) {
498 panic("do a data_return() if slot for this page is empty");
499 }
500
501 compressor_pager_unlock(pager);
502
503 return KERN_SUCCESS;
504 }
505
506
507 /*ARGSUSED*/
508 kern_return_t
compressor_memory_object_data_return(__unused memory_object_t mem_obj,__unused memory_object_offset_t offset,__unused memory_object_cluster_size_t size,__unused memory_object_offset_t * resid_offset,__unused int * io_error,__unused boolean_t dirty,__unused boolean_t kernel_copy,__unused int upl_flags)509 compressor_memory_object_data_return(
510 __unused memory_object_t mem_obj,
511 __unused memory_object_offset_t offset,
512 __unused memory_object_cluster_size_t size,
513 __unused memory_object_offset_t *resid_offset,
514 __unused int *io_error,
515 __unused boolean_t dirty,
516 __unused boolean_t kernel_copy,
517 __unused int upl_flags)
518 {
519 panic("compressor: data_return");
520 return KERN_FAILURE;
521 }
522
523 /*
524 * Routine: default_pager_memory_object_create
525 * Purpose:
526 * Handle requests for memory objects from the
527 * kernel.
528 * Notes:
529 * Because we only give out the default memory
530 * manager port to the kernel, we don't have to
531 * be so paranoid about the contents.
532 */
533 kern_return_t
compressor_memory_object_create(memory_object_size_t new_size,memory_object_t * new_mem_obj)534 compressor_memory_object_create(
535 memory_object_size_t new_size,
536 memory_object_t *new_mem_obj)
537 {
538 compressor_pager_t pager;
539 unsigned int num_chunks;
540
541 if ((uint32_t)(new_size / PAGE_SIZE) != (new_size / PAGE_SIZE)) {
542 /* 32-bit overflow for number of pages */
543 panic("%s: size 0x%llx overflow",
544 __FUNCTION__, (uint64_t) new_size);
545 return KERN_INVALID_ARGUMENT;
546 }
547
548 pager = zalloc_flags(compressor_pager_zone, Z_WAITOK | Z_NOFAIL);
549
550 compressor_pager_lock_init(pager);
551 os_ref_init_raw(&pager->cpgr_references, NULL);
552 pager->cpgr_num_slots = (uint32_t)(new_size / PAGE_SIZE);
553 pager->cpgr_num_slots_occupied = 0;
554
555 num_chunks = compressor_pager_num_chunks(pager);
556 if (num_chunks > 1) {
557 pager->cpgr_slots.cpgr_islots = kalloc_type(compressor_slot_t *,
558 num_chunks, Z_WAITOK | Z_ZERO);
559 } else if (pager->cpgr_num_slots > 2) {
560 pager->cpgr_slots.cpgr_dslots = zalloc_slot_array(pager->cpgr_num_slots *
561 sizeof(pager->cpgr_slots.cpgr_dslots[0]), Z_WAITOK | Z_ZERO);
562 } else {
563 pager->cpgr_slots.cpgr_eslots[0] = 0;
564 pager->cpgr_slots.cpgr_eslots[1] = 0;
565 }
566
567 /*
568 * Set up associations between this memory object
569 * and this compressor_pager structure
570 */
571 pager->cpgr_hdr.mo_ikot = IKOT_MEMORY_OBJECT;
572 pager->cpgr_hdr.mo_pager_ops = &compressor_pager_ops;
573 pager->cpgr_hdr.mo_control = MEMORY_OBJECT_CONTROL_NULL;
574
575 *new_mem_obj = (memory_object_t) pager;
576 return KERN_SUCCESS;
577 }
578
579
580 unsigned int
compressor_pager_slots_chunk_free(compressor_slot_t * chunk,int num_slots,int flags,int * failures)581 compressor_pager_slots_chunk_free(
582 compressor_slot_t *chunk,
583 int num_slots,
584 int flags,
585 int *failures)
586 {
587 int i;
588 int retval;
589 unsigned int num_slots_freed;
590
591 if (failures) {
592 *failures = 0;
593 }
594 num_slots_freed = 0;
595 for (i = 0; i < num_slots; i++) {
596 if (chunk[i] != 0) {
597 retval = vm_compressor_free(&chunk[i], flags);
598
599 if (retval == 0) {
600 num_slots_freed++;
601 } else {
602 if (retval == -2) {
603 assert(flags & C_DONT_BLOCK);
604 }
605
606 if (failures) {
607 *failures += 1;
608 }
609 }
610 }
611 }
612 return num_slots_freed;
613 }
614
615 void
compressor_pager_slot_lookup(compressor_pager_t pager,boolean_t do_alloc,memory_object_offset_t offset,compressor_slot_t ** slot_pp)616 compressor_pager_slot_lookup(
617 compressor_pager_t pager,
618 boolean_t do_alloc,
619 memory_object_offset_t offset,
620 compressor_slot_t **slot_pp)
621 {
622 unsigned int num_chunks;
623 uint32_t page_num;
624 unsigned int chunk_idx;
625 int slot_idx;
626 compressor_slot_t *chunk;
627 compressor_slot_t *t_chunk;
628
629 page_num = (uint32_t)(offset / PAGE_SIZE);
630 if (page_num != (offset / PAGE_SIZE)) {
631 /* overflow */
632 panic("%s: offset 0x%llx overflow",
633 __FUNCTION__, (uint64_t) offset);
634 *slot_pp = NULL;
635 return;
636 }
637 if (page_num >= pager->cpgr_num_slots) {
638 /* out of range */
639 *slot_pp = NULL;
640 return;
641 }
642 num_chunks = compressor_pager_num_chunks(pager);
643 if (num_chunks > 1) {
644 /* we have an array of chunks */
645 chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK;
646 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
647
648 if (chunk == NULL && do_alloc) {
649 t_chunk = zalloc_slot_array(COMPRESSOR_SLOTS_CHUNK_SIZE,
650 Z_WAITOK | Z_ZERO);
651
652 compressor_pager_lock(pager);
653
654 if ((chunk = pager->cpgr_slots.cpgr_islots[chunk_idx]) == NULL) {
655 /*
656 * On some platforms, the memory stores from
657 * the bzero(t_chunk) above might not have been
658 * made visible and another thread might see
659 * the contents of this new chunk before it's
660 * been fully zero-filled.
661 * This memory barrier should take care of this
662 * according to the platform requirements.
663 */
664 os_atomic_thread_fence(release);
665
666 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx] = t_chunk;
667 t_chunk = NULL;
668 }
669 compressor_pager_unlock(pager);
670
671 if (t_chunk) {
672 zfree_slot_array(t_chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
673 }
674 }
675 if (chunk == NULL) {
676 *slot_pp = NULL;
677 } else {
678 slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK;
679 *slot_pp = &chunk[slot_idx];
680 }
681 } else if (pager->cpgr_num_slots > 2) {
682 slot_idx = page_num;
683 *slot_pp = &pager->cpgr_slots.cpgr_dslots[slot_idx];
684 } else {
685 slot_idx = page_num;
686 *slot_pp = &pager->cpgr_slots.cpgr_eslots[slot_idx];
687 }
688 }
689
690 #if defined(__LP64__)
691 __startup_func
692 static void
vm_compressor_slots_init(void)693 vm_compressor_slots_init(void)
694 {
695 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
696 compressor_slots_zones[idx] = zone_create(
697 compressor_slots_zones_names[idx],
698 compressor_slots_zones_sizes[idx],
699 ZC_PGZ_USE_GUARDS | ZC_VM_LP64 | ZC_NOTBITAG);
700 }
701 }
702 STARTUP(ZALLOC, STARTUP_RANK_MIDDLE, vm_compressor_slots_init);
703 #endif /* defined(__LP64__) */
704
705 static compressor_slot_t *
zalloc_slot_array(size_t size,zalloc_flags_t flags)706 zalloc_slot_array(size_t size, zalloc_flags_t flags)
707 {
708 #if defined(__LP64__)
709 compressor_slot_t *slots = NULL;
710
711 assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
712 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
713 if (size > compressor_slots_zones_sizes[idx]) {
714 continue;
715 }
716 slots = zalloc_flags(compressor_slots_zones[idx], flags);
717 break;
718 }
719 return slots;
720 #else /* defined(__LP64__) */
721 return kalloc_data(size, flags);
722 #endif /* !defined(__LP64__) */
723 }
724
725 static void
zfree_slot_array(compressor_slot_t * slots,size_t size)726 zfree_slot_array(compressor_slot_t *slots, size_t size)
727 {
728 #if defined(__LP64__)
729 assert(size <= COMPRESSOR_SLOTS_CHUNK_SIZE);
730 for (unsigned int idx = 0; idx < NUM_SLOTS_ZONES; idx++) {
731 if (size > compressor_slots_zones_sizes[idx]) {
732 continue;
733 }
734 zfree(compressor_slots_zones[idx], slots);
735 break;
736 }
737 #else /* defined(__LP64__) */
738 kfree_data(slots, size);
739 #endif /* !defined(__LP64__) */
740 }
741
742 kern_return_t
vm_compressor_pager_put(memory_object_t mem_obj,memory_object_offset_t offset,ppnum_t ppnum,void ** current_chead,char * scratch_buf,int * compressed_count_delta_p)743 vm_compressor_pager_put(
744 memory_object_t mem_obj,
745 memory_object_offset_t offset,
746 ppnum_t ppnum,
747 void **current_chead,
748 char *scratch_buf,
749 int *compressed_count_delta_p)
750 {
751 compressor_pager_t pager;
752 compressor_slot_t *slot_p;
753
754 compressor_pager_stats.put++;
755
756 *compressed_count_delta_p = 0;
757
758 /* This routine is called by the pageout thread. The pageout thread */
759 /* cannot be blocked by read activities unless the read activities */
760 /* Therefore the grant of vs lock must be done on a try versus a */
761 /* blocking basis. The code below relies on the fact that the */
762 /* interface is synchronous. Should this interface be again async */
763 /* for some type of pager in the future the pages will have to be */
764 /* returned through a separate, asynchronous path. */
765
766 compressor_pager_lookup(mem_obj, pager);
767
768 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
769 /* overflow */
770 panic("%s: offset 0x%llx overflow",
771 __FUNCTION__, (uint64_t) offset);
772 return KERN_RESOURCE_SHORTAGE;
773 }
774
775 compressor_pager_slot_lookup(pager, TRUE, offset, &slot_p);
776
777 if (slot_p == NULL) {
778 /* out of range ? */
779 panic("vm_compressor_pager_put: out of range");
780 }
781 if (*slot_p != 0) {
782 /*
783 * Already compressed: forget about the old one.
784 *
785 * This can happen after a vm_object_do_collapse() when
786 * the "backing_object" had some pages paged out and the
787 * "object" had an equivalent page resident.
788 */
789 vm_compressor_free(slot_p, 0);
790 *compressed_count_delta_p -= 1;
791 }
792
793 /*
794 * If the compressor operation succeeds, we presumably don't need to
795 * undo any previous WIMG update, as all live mappings should be
796 * disconnected.
797 */
798
799 if (vm_compressor_put(ppnum, slot_p, current_chead, scratch_buf)) {
800 return KERN_RESOURCE_SHORTAGE;
801 }
802 *compressed_count_delta_p += 1;
803
804 return KERN_SUCCESS;
805 }
806
807
808 kern_return_t
vm_compressor_pager_get(memory_object_t mem_obj,memory_object_offset_t offset,ppnum_t ppnum,int * my_fault_type,int flags,int * compressed_count_delta_p)809 vm_compressor_pager_get(
810 memory_object_t mem_obj,
811 memory_object_offset_t offset,
812 ppnum_t ppnum,
813 int *my_fault_type,
814 int flags,
815 int *compressed_count_delta_p)
816 {
817 compressor_pager_t pager;
818 kern_return_t kr;
819 compressor_slot_t *slot_p;
820
821 compressor_pager_stats.get++;
822
823 *compressed_count_delta_p = 0;
824
825 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
826 panic("%s: offset 0x%llx overflow",
827 __FUNCTION__, (uint64_t) offset);
828 return KERN_MEMORY_ERROR;
829 }
830
831 compressor_pager_lookup(mem_obj, pager);
832
833 /* find the compressor slot for that page */
834 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
835
836 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
837 /* out of range */
838 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_OUT_OF_RANGE), 0 /* arg */);
839 kr = KERN_MEMORY_FAILURE;
840 } else if (slot_p == NULL || *slot_p == 0) {
841 /* compressor does not have this page */
842 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_GET_NO_PAGE), 0 /* arg */);
843 kr = KERN_MEMORY_ERROR;
844 } else {
845 /* compressor does have this page */
846 kr = KERN_SUCCESS;
847 }
848 *my_fault_type = DBG_COMPRESSOR_FAULT;
849
850 if (kr == KERN_SUCCESS) {
851 int retval;
852
853 /* get the page from the compressor */
854 retval = vm_compressor_get(ppnum, slot_p, flags);
855 if (retval == -1) {
856 ktriage_record(thread_tid(current_thread()), KDBG_TRIAGE_EVENTID(KDBG_TRIAGE_SUBSYS_VM, KDBG_TRIAGE_RESERVED, KDBG_TRIAGE_VM_COMPRESSOR_DECOMPRESS_FAILED), 0 /* arg */);
857 kr = KERN_MEMORY_FAILURE;
858 } else if (retval == 1) {
859 *my_fault_type = DBG_COMPRESSOR_SWAPIN_FAULT;
860 } else if (retval == -2) {
861 assert((flags & C_DONT_BLOCK));
862 /*
863 * Not a fatal failure because we just retry with a blocking get later. So we skip ktriage to avoid noise.
864 */
865 kr = KERN_FAILURE;
866 }
867 }
868
869 if (kr == KERN_SUCCESS) {
870 assert(slot_p != NULL);
871 if (*slot_p != 0) {
872 /*
873 * We got the page for a copy-on-write fault
874 * and we kept the original in place. Slot
875 * is still occupied.
876 */
877 } else {
878 *compressed_count_delta_p -= 1;
879 }
880 }
881
882 return kr;
883 }
884
885 unsigned int
vm_compressor_pager_state_clr(memory_object_t mem_obj,memory_object_offset_t offset)886 vm_compressor_pager_state_clr(
887 memory_object_t mem_obj,
888 memory_object_offset_t offset)
889 {
890 compressor_pager_t pager;
891 compressor_slot_t *slot_p;
892 unsigned int num_slots_freed;
893
894 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
895
896 compressor_pager_stats.state_clr++;
897
898 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
899 /* overflow */
900 panic("%s: offset 0x%llx overflow",
901 __FUNCTION__, (uint64_t) offset);
902 return 0;
903 }
904
905 compressor_pager_lookup(mem_obj, pager);
906
907 /* find the compressor slot for that page */
908 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
909
910 num_slots_freed = 0;
911 if (slot_p && *slot_p != 0) {
912 vm_compressor_free(slot_p, 0);
913 num_slots_freed++;
914 assert(*slot_p == 0);
915 }
916
917 return num_slots_freed;
918 }
919
920 vm_external_state_t
vm_compressor_pager_state_get(memory_object_t mem_obj,memory_object_offset_t offset)921 vm_compressor_pager_state_get(
922 memory_object_t mem_obj,
923 memory_object_offset_t offset)
924 {
925 compressor_pager_t pager;
926 compressor_slot_t *slot_p;
927
928 assert(VM_CONFIG_COMPRESSOR_IS_PRESENT);
929
930 compressor_pager_stats.state_get++;
931
932 if ((uint32_t)(offset / PAGE_SIZE) != (offset / PAGE_SIZE)) {
933 /* overflow */
934 panic("%s: offset 0x%llx overflow",
935 __FUNCTION__, (uint64_t) offset);
936 return VM_EXTERNAL_STATE_ABSENT;
937 }
938
939 compressor_pager_lookup(mem_obj, pager);
940
941 /* find the compressor slot for that page */
942 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
943
944 if (offset / PAGE_SIZE >= pager->cpgr_num_slots) {
945 /* out of range */
946 return VM_EXTERNAL_STATE_ABSENT;
947 } else if (slot_p == NULL || *slot_p == 0) {
948 /* compressor does not have this page */
949 return VM_EXTERNAL_STATE_ABSENT;
950 } else {
951 /* compressor does have this page */
952 return VM_EXTERNAL_STATE_EXISTS;
953 }
954 }
955
956 unsigned int
vm_compressor_pager_reap_pages(memory_object_t mem_obj,int flags)957 vm_compressor_pager_reap_pages(
958 memory_object_t mem_obj,
959 int flags)
960 {
961 compressor_pager_t pager;
962 unsigned int num_chunks;
963 int failures;
964 unsigned int i;
965 compressor_slot_t *chunk;
966 unsigned int num_slots_freed;
967
968 compressor_pager_lookup(mem_obj, pager);
969 if (pager == NULL) {
970 return 0;
971 }
972
973 compressor_pager_lock(pager);
974
975 /* reap the compressor slots */
976 num_slots_freed = 0;
977
978 num_chunks = compressor_pager_num_chunks(pager);
979 if (num_chunks > 1) {
980 /* we have an array of chunks */
981 for (i = 0; i < num_chunks; i++) {
982 chunk = pager->cpgr_slots.cpgr_islots[i];
983 if (chunk != NULL) {
984 num_slots_freed +=
985 compressor_pager_slots_chunk_free(
986 chunk,
987 COMPRESSOR_SLOTS_PER_CHUNK,
988 flags,
989 &failures);
990 if (failures == 0) {
991 pager->cpgr_slots.cpgr_islots[i] = NULL;
992 zfree_slot_array(chunk, COMPRESSOR_SLOTS_CHUNK_SIZE);
993 }
994 }
995 }
996 } else if (pager->cpgr_num_slots > 2) {
997 chunk = pager->cpgr_slots.cpgr_dslots;
998 num_slots_freed +=
999 compressor_pager_slots_chunk_free(
1000 chunk,
1001 pager->cpgr_num_slots,
1002 flags,
1003 NULL);
1004 } else {
1005 chunk = &pager->cpgr_slots.cpgr_eslots[0];
1006 num_slots_freed +=
1007 compressor_pager_slots_chunk_free(
1008 chunk,
1009 pager->cpgr_num_slots,
1010 flags,
1011 NULL);
1012 }
1013
1014 compressor_pager_unlock(pager);
1015
1016 return num_slots_freed;
1017 }
1018
1019 void
vm_compressor_pager_transfer(memory_object_t dst_mem_obj,memory_object_offset_t dst_offset,memory_object_t src_mem_obj,memory_object_offset_t src_offset)1020 vm_compressor_pager_transfer(
1021 memory_object_t dst_mem_obj,
1022 memory_object_offset_t dst_offset,
1023 memory_object_t src_mem_obj,
1024 memory_object_offset_t src_offset)
1025 {
1026 compressor_pager_t src_pager, dst_pager;
1027 compressor_slot_t *src_slot_p, *dst_slot_p;
1028
1029 compressor_pager_stats.transfer++;
1030
1031 /* find the compressor slot for the destination */
1032 assert((uint32_t) dst_offset == dst_offset);
1033 compressor_pager_lookup(dst_mem_obj, dst_pager);
1034 assert(dst_offset / PAGE_SIZE < dst_pager->cpgr_num_slots);
1035 compressor_pager_slot_lookup(dst_pager, TRUE, (uint32_t) dst_offset,
1036 &dst_slot_p);
1037 assert(dst_slot_p != NULL);
1038 assert(*dst_slot_p == 0);
1039
1040 /* find the compressor slot for the source */
1041 assert((uint32_t) src_offset == src_offset);
1042 compressor_pager_lookup(src_mem_obj, src_pager);
1043 assert(src_offset / PAGE_SIZE < src_pager->cpgr_num_slots);
1044 compressor_pager_slot_lookup(src_pager, FALSE, (uint32_t) src_offset,
1045 &src_slot_p);
1046 assert(src_slot_p != NULL);
1047 assert(*src_slot_p != 0);
1048
1049 /* transfer the slot from source to destination */
1050 vm_compressor_transfer(dst_slot_p, src_slot_p);
1051 OSAddAtomic(-1, &src_pager->cpgr_num_slots_occupied);
1052 OSAddAtomic(+1, &dst_pager->cpgr_num_slots_occupied);
1053 }
1054
1055 memory_object_offset_t
vm_compressor_pager_next_compressed(memory_object_t mem_obj,memory_object_offset_t offset)1056 vm_compressor_pager_next_compressed(
1057 memory_object_t mem_obj,
1058 memory_object_offset_t offset)
1059 {
1060 compressor_pager_t pager;
1061 unsigned int num_chunks;
1062 uint32_t page_num;
1063 unsigned int chunk_idx;
1064 uint32_t slot_idx;
1065 compressor_slot_t *chunk;
1066
1067 compressor_pager_lookup(mem_obj, pager);
1068
1069 page_num = (uint32_t)(offset / PAGE_SIZE);
1070 if (page_num != (offset / PAGE_SIZE)) {
1071 /* overflow */
1072 return (memory_object_offset_t) -1;
1073 }
1074 if (page_num >= pager->cpgr_num_slots) {
1075 /* out of range */
1076 return (memory_object_offset_t) -1;
1077 }
1078
1079 num_chunks = compressor_pager_num_chunks(pager);
1080 if (num_chunks == 1) {
1081 if (pager->cpgr_num_slots > 2) {
1082 chunk = pager->cpgr_slots.cpgr_dslots;
1083 } else {
1084 chunk = &pager->cpgr_slots.cpgr_eslots[0];
1085 }
1086 for (slot_idx = page_num;
1087 slot_idx < pager->cpgr_num_slots;
1088 slot_idx++) {
1089 if (chunk[slot_idx] != 0) {
1090 /* found a non-NULL slot in this chunk */
1091 return (memory_object_offset_t) (slot_idx *
1092 PAGE_SIZE);
1093 }
1094 }
1095 return (memory_object_offset_t) -1;
1096 }
1097
1098 /* we have an array of chunks; find the next non-NULL chunk */
1099 chunk = NULL;
1100 for (chunk_idx = page_num / COMPRESSOR_SLOTS_PER_CHUNK,
1101 slot_idx = page_num % COMPRESSOR_SLOTS_PER_CHUNK;
1102 chunk_idx < num_chunks;
1103 chunk_idx++,
1104 slot_idx = 0) {
1105 chunk = pager->cpgr_slots.cpgr_islots[chunk_idx];
1106 if (chunk == NULL) {
1107 /* no chunk here: try the next one */
1108 continue;
1109 }
1110 /* search for an occupied slot in this chunk */
1111 for (;
1112 slot_idx < COMPRESSOR_SLOTS_PER_CHUNK;
1113 slot_idx++) {
1114 if (chunk[slot_idx] != 0) {
1115 /* found an occupied slot in this chunk */
1116 uint32_t next_slot;
1117
1118 next_slot = ((chunk_idx *
1119 COMPRESSOR_SLOTS_PER_CHUNK) +
1120 slot_idx);
1121 if (next_slot >= pager->cpgr_num_slots) {
1122 /* went beyond end of object */
1123 return (memory_object_offset_t) -1;
1124 }
1125 return (memory_object_offset_t) (next_slot *
1126 PAGE_SIZE);
1127 }
1128 }
1129 }
1130 return (memory_object_offset_t) -1;
1131 }
1132
1133 unsigned int
vm_compressor_pager_get_count(memory_object_t mem_obj)1134 vm_compressor_pager_get_count(
1135 memory_object_t mem_obj)
1136 {
1137 compressor_pager_t pager;
1138
1139 compressor_pager_lookup(mem_obj, pager);
1140 if (pager == NULL) {
1141 return 0;
1142 }
1143
1144 /*
1145 * The caller should have the VM object locked and one
1146 * needs that lock to do a page-in or page-out, so no
1147 * need to lock the pager here.
1148 */
1149 assert(pager->cpgr_num_slots_occupied >= 0);
1150
1151 return pager->cpgr_num_slots_occupied;
1152 }
1153
1154 void
vm_compressor_pager_count(memory_object_t mem_obj,int compressed_count_delta,boolean_t shared_lock,vm_object_t object __unused)1155 vm_compressor_pager_count(
1156 memory_object_t mem_obj,
1157 int compressed_count_delta,
1158 boolean_t shared_lock,
1159 vm_object_t object __unused)
1160 {
1161 compressor_pager_t pager;
1162
1163 if (compressed_count_delta == 0) {
1164 return;
1165 }
1166
1167 compressor_pager_lookup(mem_obj, pager);
1168 if (pager == NULL) {
1169 return;
1170 }
1171
1172 if (compressed_count_delta < 0) {
1173 assert(pager->cpgr_num_slots_occupied >=
1174 (unsigned int) -compressed_count_delta);
1175 }
1176
1177 /*
1178 * The caller should have the VM object locked,
1179 * shared or exclusive.
1180 */
1181 if (shared_lock) {
1182 vm_object_lock_assert_shared(object);
1183 OSAddAtomic(compressed_count_delta,
1184 &pager->cpgr_num_slots_occupied);
1185 } else {
1186 vm_object_lock_assert_exclusive(object);
1187 pager->cpgr_num_slots_occupied += compressed_count_delta;
1188 }
1189 }
1190
1191 #if CONFIG_FREEZE
1192 kern_return_t
vm_compressor_pager_relocate(memory_object_t mem_obj,memory_object_offset_t offset,void ** current_chead)1193 vm_compressor_pager_relocate(
1194 memory_object_t mem_obj,
1195 memory_object_offset_t offset,
1196 void **current_chead)
1197 {
1198 /*
1199 * Has the page at this offset been compressed?
1200 */
1201
1202 compressor_slot_t *slot_p;
1203 compressor_pager_t dst_pager;
1204
1205 assert(mem_obj);
1206
1207 compressor_pager_lookup(mem_obj, dst_pager);
1208 if (dst_pager == NULL) {
1209 return KERN_FAILURE;
1210 }
1211
1212 compressor_pager_slot_lookup(dst_pager, FALSE, offset, &slot_p);
1213 return vm_compressor_relocate(current_chead, slot_p);
1214 }
1215 #endif /* CONFIG_FREEZE */
1216
1217 #if DEVELOPMENT || DEBUG
1218
1219 kern_return_t
vm_compressor_pager_inject_error(memory_object_t mem_obj,memory_object_offset_t offset)1220 vm_compressor_pager_inject_error(memory_object_t mem_obj,
1221 memory_object_offset_t offset)
1222 {
1223 kern_return_t result = KERN_FAILURE;
1224 compressor_slot_t *slot_p;
1225 compressor_pager_t pager;
1226
1227 assert(mem_obj);
1228
1229 compressor_pager_lookup(mem_obj, pager);
1230 if (pager != NULL) {
1231 compressor_pager_slot_lookup(pager, FALSE, offset, &slot_p);
1232 if (slot_p != NULL && *slot_p != 0) {
1233 vm_compressor_inject_error(slot_p);
1234 result = KERN_SUCCESS;
1235 }
1236 }
1237
1238 return result;
1239 }
1240
1241 #endif
1242