1 /*
2 * Copyright (c) 2016-2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
5 *
6 * This file contains Original Code and/or Modifications of Original Code
7 * as defined in and that are subject to the Apple Public Source License
8 * Version 2.0 (the 'License'). You may not use this file except in
9 * compliance with the License. The rights granted to you under the License
10 * may not be used to create, or enable the creation or redistribution of,
11 * unlawful or unlicensed copies of an Apple operating system, or to
12 * circumvent, violate, or enable the circumvention or violation of, any
13 * terms of an Apple operating system software license agreement.
14 *
15 * Please obtain a copy of the License at
16 * http://www.opensource.apple.com/apsl/ and read it before using this file.
17 *
18 * The Original Code and all software distributed under the License are
19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
23 * Please see the License for the specific language governing rights and
24 * limitations under the License.
25 *
26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
27 */
28
29 #pragma once
30 #include <stdint.h>
31 #include <stddef.h>
32 #include <kern/assert.h>
33 #include <machine/limits.h>
34 #include "lz4_assembly_select.h"
35 #include "lz4_constants.h"
36
37 #if CONFIG_IO_COMPRESSION_STATS
38 #include <string.h>
39 #define lz4_memcpy memcpy
40 #else
41 #define lz4_memcpy __builtin_memcpy
42 #endif
43
44 #pragma mark - Building blocks
45
46 // Represents a position in the input stream
47 typedef struct { uint32_t offset; uint32_t word; } lz4_hash_entry_t;
48 static const size_t lz4_hash_table_size = LZ4_COMPRESS_HASH_ENTRIES * sizeof(lz4_hash_entry_t);
49
50 // Worker function for lz4 encode. Underlies both the buffer and stream encode operations.
51 // Performs lz4 encoding of up to 2gb of data, updates dst_ptr and src_ptr to point to the
52 // first byte of output and input that couldn't be completely processed, respectively.
53 //
54 // If skip_final_literals is 0, the entire src buffer is encoded, by emitting a final sequence of literals
55 // at the end of the compressed payload.
56 //
57 // If skip_final_literals is not 0, this final literal sequence is not emitted, and the src buffer is
58 // partially encoded (the length of this literal sequence varies).
59 extern void lz4_encode_2gb(uint8_t **dst_ptr, size_t dst_size,
60 const uint8_t **src_ptr, const uint8_t *src_begin, size_t src_size,
61 lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES], int skip_final_literals);
62
63 extern int lz4_decode(uint8_t **dst_ptr, uint8_t *dst_begin, uint8_t *dst_end,
64 const uint8_t **src_ptr, const uint8_t *src_end);
65
66 #if LZ4_ENABLE_ASSEMBLY_DECODE
67 extern int lz4_decode_asm(uint8_t **dst_ptr, uint8_t *dst_begin, uint8_t *dst_end,
68 const uint8_t **src_ptr, const uint8_t *src_end);
69 #endif
70
71 #pragma mark - Buffer interfaces
72
73 static const size_t lz4_encode_scratch_size = lz4_hash_table_size;
74 static const size_t lz4_decode_scratch_size = 0;
75
76 #pragma mark - Buffer interfaces (LZ4 RAW)
77
78 size_t lz4raw_encode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size,
79 const uint8_t * __restrict src_buffer, size_t src_size,
80 lz4_hash_entry_t hash_table[LZ4_COMPRESS_HASH_ENTRIES]);
81
82 size_t lz4raw_decode_buffer(uint8_t * __restrict dst_buffer, size_t dst_size,
83 const uint8_t * __restrict src_buffer, size_t src_size,
84 void * __restrict work __attribute__((unused)));
85
86 typedef __attribute__((__ext_vector_type__(8))) uint8_t vector_uchar8;
87 typedef __attribute__((__ext_vector_type__(16))) uint8_t vector_uchar16;
88 typedef __attribute__((__ext_vector_type__(32))) uint8_t vector_uchar32;
89 typedef __attribute__((__ext_vector_type__(64))) uint8_t vector_uchar64;
90 typedef __attribute__((__ext_vector_type__(16), __aligned__(1))) uint8_t packed_uchar16;
91 typedef __attribute__((__ext_vector_type__(32), __aligned__(1))) uint8_t packed_uchar32;
92 typedef __attribute__((__ext_vector_type__(64), __aligned__(1))) uint8_t packed_uchar64;
93
94 typedef __attribute__((__ext_vector_type__(4))) uint16_t vector_ushort4;
95 typedef __attribute__((__ext_vector_type__(4), __aligned__(2))) uint16_t packed_ushort4;
96
97 typedef __attribute__((__ext_vector_type__(2))) int32_t vector_int2;
98 typedef __attribute__((__ext_vector_type__(4))) int32_t vector_int4;
99 typedef __attribute__((__ext_vector_type__(8))) int32_t vector_int8;
100
101 typedef __attribute__((__ext_vector_type__(4))) uint32_t vector_uint4;
102
103 #define UTIL_FUNCTION static inline __attribute__((__always_inline__)) __attribute__((__overloadable__))
104
105 // Load N bytes from unaligned location PTR
106 UTIL_FUNCTION uint16_t
load2(const void * ptr)107 load2(const void * ptr)
108 {
109 uint16_t data; lz4_memcpy(&data, ptr, sizeof data); return data;
110 }
111 UTIL_FUNCTION uint32_t
load4(const void * ptr)112 load4(const void * ptr)
113 {
114 uint32_t data; lz4_memcpy(&data, ptr, sizeof data); return data;
115 }
116 UTIL_FUNCTION uint64_t
load8(const void * ptr)117 load8(const void * ptr)
118 {
119 uint64_t data; lz4_memcpy(&data, ptr, sizeof data); return data;
120 }
121 UTIL_FUNCTION vector_uchar16
load16(const void * ptr)122 load16(const void * ptr)
123 {
124 return (const vector_uchar16)*(const packed_uchar16 *)ptr;
125 }
126 UTIL_FUNCTION vector_uchar32
load32(const void * ptr)127 load32(const void * ptr)
128 {
129 return (const vector_uchar32)*(const packed_uchar32 *)ptr;
130 }
131 UTIL_FUNCTION vector_uchar64
load64(const void * ptr)132 load64(const void * ptr)
133 {
134 return (const vector_uchar64)*(const packed_uchar64 *)ptr;
135 }
136
137 // Store N bytes to unaligned location PTR
138 UTIL_FUNCTION void
store2(void * ptr,uint16_t data)139 store2(void * ptr, uint16_t data)
140 {
141 lz4_memcpy(ptr, &data, sizeof data);
142 }
143 UTIL_FUNCTION void
store4(void * ptr,uint32_t data)144 store4(void * ptr, uint32_t data)
145 {
146 lz4_memcpy(ptr, &data, sizeof data);
147 }
148 UTIL_FUNCTION void
store8(void * ptr,uint64_t data)149 store8(void * ptr, uint64_t data)
150 {
151 lz4_memcpy(ptr, &data, sizeof data);
152 }
153 UTIL_FUNCTION void
store16(void * ptr,vector_uchar16 data)154 store16(void * ptr, vector_uchar16 data)
155 {
156 *(packed_uchar16 *)ptr = (packed_uchar16)data;
157 }
158 UTIL_FUNCTION void
store32(void * ptr,vector_uchar32 data)159 store32(void * ptr, vector_uchar32 data)
160 {
161 *(packed_uchar32 *)ptr = (packed_uchar32)data;
162 }
163 UTIL_FUNCTION void
store64(void * ptr,vector_uchar64 data)164 store64(void * ptr, vector_uchar64 data)
165 {
166 *(packed_uchar64 *)ptr = (packed_uchar64)data;
167 }
168
169 // Load+Store N bytes from unaligned locations SRC to DST. No overlap allowed.
170 UTIL_FUNCTION void
copy8(void * dst,const void * src)171 copy8(void * dst, const void * src)
172 {
173 store8(dst, load8(src));
174 }
175 UTIL_FUNCTION void
copy16(void * dst,const void * src)176 copy16(void * dst, const void * src)
177 {
178 *(packed_uchar16 *)dst = *(const packed_uchar16 *)src;
179 }
180 UTIL_FUNCTION void
copy32(void * dst,const void * src)181 copy32(void * dst, const void * src)
182 {
183 *(packed_uchar32 *)dst = *(const packed_uchar32 *)src;
184 }
185
186 #undef lz4_memcpy
187