185 #define UNPACK_C_SIZE(cs)       ((cs->c_size == (PAGE_SIZE-1)) ? PAGE_SIZE : cs->c_size)
186 #define PACK_C_SIZE(cs, size)   (cs->c_size = ((size == PAGE_SIZE) ? PAGE_SIZE - 1 : size))
595 	cdst->c_size = csrc->c_size;  in cslot_copy()
1067 	uint32_t        c_size;  in c_seg_validate()  local
1081 		if (cs->c_size) {  in c_seg_validate()
1082 			panic("c_seg_validate:  c_firstemptyslot has non-zero size (%d)", cs->c_size);  in c_seg_validate()
1090 		c_size = UNPACK_C_SIZE(cs);  in c_seg_validate()
1092 		c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_validate()
1098 …if (c_size && cs->c_hash_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[c…  in c_seg_validate()
1100 …0x%llx %d %p %d %d 0x%x 0x%x", c_seg, csvphys, cs->c_offset, cs, c_indx, c_size, cs->c_hash_compre…  in c_seg_validate()
1105 		if (c_size) {  in c_seg_validate()
1107 			if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {  in c_seg_validate()
1108 …)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, (uint64_t)cs->c_offset, c_size, csvpop, cs->c_pop…  in c_seg_validate()
1772 	uint32_t        c_size;  in c_seg_trim_tail()  local
1787 		c_size = UNPACK_C_SIZE(cs);  in c_seg_trim_tail()
1789 		if (c_size) {  in c_seg_trim_tail()
1791 				c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_trim_tail()
1822 	uint32_t        c_size;  in c_seg_minor_compaction_and_unlock()  local
1861 		c_size = UNPACK_C_SIZE(c_src);  in c_seg_minor_compaction_and_unlock()
1863 		if (c_size == 0) {  in c_seg_minor_compaction_and_unlock()
1867 		c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_minor_compaction_and_unlock()
2009 	uint32_t        c_size;  in c_seg_major_compact()  local
2038 		c_size = UNPACK_C_SIZE(c_src);  in c_seg_major_compact()
2040 		if (c_size == 0) {  in c_seg_major_compact()
2045 …SEG_OFFSET_TO_BYTES(c_seg_dst->c_populated_offset - c_seg_dst->c_nextoffset) < (unsigned) c_size) {  in c_seg_major_compact()
2073 …>c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);  in c_seg_major_compact()
2075 		c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_seg_major_compact()
2078 		c_seg_major_compact_stats[c_seg_major_compact_stats_now].moved_bytes += c_size;  in c_seg_major_compact()
4119 c_compressed_record_data(char *src, int c_size)  in c_compressed_record_data()  argument
4121 	if ((c_compressed_record_cptr + c_size + 4) >= c_compressed_record_ebuf) {  in c_compressed_record_data()
4125 	*(int *)((void *)c_compressed_record_cptr) = c_size;  in c_compressed_record_data()
4129 	memcpy(c_compressed_record_cptr, src, c_size);  in c_compressed_record_data()
4130 	c_compressed_record_cptr += c_size;  in c_compressed_record_data()
4138 	int             c_size = -1;  in c_compress_page()  local
4181 			c_size = metacompressor((const uint8_t *) src,  in c_compress_page()
4192 			if (c_size > max_csize_adj) {  in c_compress_page()
4193 				c_size = -1;  in c_compress_page()
4197 			c_size = -1;  in c_compress_page()
4209 …c_size = WKdm_compress_4k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buffe…  in c_compress_page()
4212 …c_size = WKdm_compress_16k((WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.c_buff…  in c_compress_page()
4217 …c_size = WKdm_compress_new((const WK_word *)(uintptr_t)src, (WK_word *)(uintptr_t)&c_seg->c_store.…  in c_compress_page()
4221 	assertf(((c_size <= max_csize_adj) && (c_size >= -1)),  in c_compress_page()
4222 …"c_size invalid (%d, %d), cur compressions: %d", c_size, max_csize_adj, c_segment_pages_compressed…  in c_compress_page()
4224 	if (c_size == -1) {  in c_compress_page()
4237 		c_size = PAGE_SIZE;  in c_compress_page()
4240 			memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);  in c_compress_page()
4244 	} else if (c_size == 0) {  in c_compress_page()
4262 		c_size = 4;  in c_compress_page()
4264 		memcpy(&c_seg->c_store.c_buffer[cs->c_offset], src, c_size);  in c_compress_page()
4270 	c_compressed_record_data((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);  in c_compress_page()
4273 	cs->c_hash_compressed_data = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size);  in c_compress_page()
4276 	cs->c_pop_cdata = vmc_pop((uintptr_t) &c_seg->c_store.c_buffer[cs->c_offset], c_size);  in c_compress_page()
4278 	c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_compress_page()
4280 	PACK_C_SIZE(cs, c_size);  in c_compress_page()
4304 	if (c_size) {  in c_compress_page()
4305 		OSAddAtomic64(c_size, &c_segment_compressed_bytes);  in c_compress_page()
4316 …KERNEL_DEBUG(0xe0400000 | DBG_FUNC_END, *current_chead, c_size, c_segment_input_bytes, c_segment_c…  in c_compress_page()
4376 	uint32_t        c_size;  in c_decompress_page()  local
4497 	c_size = UNPACK_C_SIZE(cs);  in c_decompress_page()
4499 	if (__improbable(c_size == 0)) {  in c_decompress_page()
4504 	c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in c_decompress_page()
4551 		if (cs->c_pop_cdata != (csvpop = vmc_pop(csvaddr, c_size))) {  in c_decompress_page()
4552 …), (void *)csvaddr, (void *) kvtophys(csvaddr), c_seg, cs, cs->c_offset, c_size, csvpop, cs->c_pop…  in c_decompress_page()
4558 …_compressed_data != (csvhash = vmc_hash((char *)&c_seg->c_store.c_buffer[cs->c_offset], c_size))) {  in c_decompress_page()
4559 …panic("Compressed data doesn't match original %p %p %u %u %u", c_seg, cs, c_size, cs->c_hash_compr…  in c_decompress_page()
4567 		} else if (c_size == 4) {  in c_decompress_page()
4604 				    (uint8_t *)dst, c_size, c_codec, (void *)scratch_buf, &inline_popcount)) {  in c_decompress_page()
4619 						    (unsigned long long)kvtophys((uintptr_t)&c_seg->c_store.c_buffer[cs->c_offset]), c_size,  in c_decompress_page()
4634 					    (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);  in c_decompress_page()
4637 					    (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);  in c_decompress_page()
4642 				    (WK_word *)(uintptr_t)dst, (WK_word *)(uintptr_t)scratch_buf, c_size);  in c_decompress_page()
4651 …, codec: %d, header: 0x%x 0x%x 0x%x", cs, cs->c_hash_data, cs->c_offset, c_size, c_rounded_size, c…  in c_decompress_page()
4653 …h original cs: %p, hash: %d, offset: 0x%x, c_size: %d", cs, cs->c_hash_data, cs->c_offset, c_size);  in c_decompress_page()
5140 	uint32_t                c_size;  in vm_compressor_relocate()  local
5240 	c_size = UNPACK_C_SIZE(c_src);  in vm_compressor_relocate()
5242 	assert(c_size);  in vm_compressor_relocate()
5244 	if (c_size > (uint32_t)(c_seg_bufsize - C_SEG_OFFSET_TO_BYTES((int32_t)c_seg_dst->c_nextoffset))) {  in vm_compressor_relocate()
5279 …>c_store.c_buffer[c_seg_dst->c_nextoffset], &c_seg_src->c_store.c_buffer[c_src->c_offset], c_size);  in vm_compressor_relocate()
5283 	c_rounded_size = (c_size + C_SEG_OFFSET_ALIGNMENT_MASK) & ~C_SEG_OFFSET_ALIGNMENT_MASK;  in vm_compressor_relocate()