190 #define UNPACK_C_SIZE(cs)       ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
191 #define PACK_C_SIZE(cs, size)   (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
689 	cdst->c_size = csrc->c_size;  in cslot_copy()
1237 	uint32_t        c_size;  in c_seg_validate()  local
1251 		if (cs->c_size) {  in c_seg_validate()
1252 			panic("c_seg_validate:  c_firstemptyslot has non-zero size (%d)", cs->c_size);  in c_seg_validate()
1260 		c_size = UNPACK_C_SIZE(cs);  in c_seg_validate()
1262 		c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_validate()
1268 …if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[c…  in c_seg_validate()
1270 …0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compre…  in c_seg_validate()
1275 		if (c_size) {  in c_seg_validate()
1277 			if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {  in c_seg_validate()
1278 …)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, (uint64_t)cs->c_offset, c_size, csvpop, cs->c_pop…  in c_seg_validate()
2042 	uint32_t        c_size;  in c_seg_trim_tail()  local
2057 		c_size = UNPACK_C_SIZE(cs);  in c_seg_trim_tail()
2059 		if (c_size) {  in c_seg_trim_tail()
2061 				c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_trim_tail()
2092 	uint32_t        c_size;  in c_seg_minor_compaction_and_unlock()  local
2131 		c_size = UNPACK_C_SIZE(c_src);  in c_seg_minor_compaction_and_unlock()
2133 		if (c_size == 0) {  in c_seg_minor_compaction_and_unlock()
2137 		c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_minor_compaction_and_unlock()
2279 	uint32_t        c_size;  in c_seg_major_compact()  local
2309 		c_size = UNPACK_C_SIZE(c_src);  in c_seg_major_compact()
2311 		if (c_size == 0) {  in c_seg_major_compact()
2316 …SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {  in c_seg_major_compact()
2344 …>c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);  in c_seg_major_compact()
2346 		c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_major_compact()
2349 		c_seg_major_compact_stats[c_seg_major_compact_stats_now].moved_bytes += c_size;  in c_seg_major_compact()
4619 c_compressed_record_data(char *src, int c_size)  in c_compressed_record_data()  argument
4621 	if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf) {  in c_compressed_record_data()
4625 	*(int *)((void *)c_compressed_record_cptr) = c_size;  in c_compressed_record_data()
4629 	memcpy(c_compressed_record_cptr, src, c_size);  in c_compressed_record_data()
4630 	c_compressed_record_cptr += c_size;  in c_compressed_record_data()
4638 	int             c_size = -1;  in c_compress_page()  local
4682 			c_size = metacompressor((const uint8_t *) src,  in c_compress_page()
4689 			if (c_size > max_csize_adj) {  in c_compress_page()
4690 				c_size = -1;  in c_compress_page()
4694 			c_size = -1;  in c_compress_page()
4704 …c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffe…  in c_compress_page()
4707 …c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buff…  in c_compress_page()
4712 …c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.…  in c_compress_page()
4716 	assertf(((c_size <= max_csize_adj) && (c_size >= -1)),  in c_compress_page()
4717 …"c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed…  in c_compress_page()
4719 	if (c_size == -1) {  in c_compress_page()
4732 		c_size = PAGE_SIZE;  in c_compress_page()
4735 			memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);  in c_compress_page()
4739 	} else if (c_size == 0) {  in c_compress_page()
4758 		c_size = 4;  in c_compress_page()
4760 		memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);  in c_compress_page()
4766 	c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);  in c_compress_page()
4769 	cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);  in c_compress_page()
4772 	cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);  in c_compress_page()
4774 	c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_compress_page()
4776 	PACK_C_SIZE(cs, c_size);  in c_compress_page()
4809 	if (c_size) {  in c_compress_page()
4810 		OSAddAtomic64(c_size, &c_segment_compressed_bytes);  in c_compress_page()
4828 …KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_c…  in c_compress_page()
4888 	uint32_t        c_size;  in c_decompress_page()  local
5009 	c_size = UNPACK_C_SIZE(cs);  in c_decompress_page()
5011 	if (__improbable(c_size == 0)) {  in c_decompress_page()
5016 	c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_decompress_page()
5063 		if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {  in c_decompress_page()
5064 …), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop…  in c_decompress_page()
5070 …_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {  in c_decompress_page()
5071 …panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compr…  in c_decompress_page()
5079 		} else if (c_size == 4) {  in c_decompress_page()
5116 				    (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf, &inline_popcount)) {  in c_decompress_page()
5127 					    (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);  in c_decompress_page()
5130 					    (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);  in c_decompress_page()
5135 				    (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);  in c_decompress_page()
5144 …, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, c…  in c_decompress_page()
5146 …h original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);  in c_decompress_page()
5654 	uint32_t                c_size;  in vm_compressor_relocate()  local
5754 	c_size = UNPACK_C_SIZE(c_src);  in vm_compressor_relocate()
5756 	assert(c_size);  in vm_compressor_relocate()
5758 	if (c_size > (uint32_t)(c_seg_bufsize - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {  in vm_compressor_relocate()
5793 …>c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);  in vm_compressor_relocate()
5797 	c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in vm_compressor_relocate()