1 /*
2 * Copyright (c) 2016 Apple Inc. All rights reserved.
3 *
4 * @APPLE_APACHE_LICENSE_HEADER_START@
5 *
6 * Licensed under the Apache License, Version 2.0 (the "License");
7 * you may not use this file except in compliance with the License.
8 * You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 *
18 * @APPLE_APACHE_LICENSE_HEADER_END@
19 */
20
21 #ifndef __FIREHOSE_CHUNK_PRIVATE__
22 #define __FIREHOSE_CHUNK_PRIVATE__
23
24 #include <sys/param.h>
25 #include "firehose_types_private.h"
26 #include "tracepoint_private.h"
27
28 __BEGIN_DECLS
29
30 #define FIREHOSE_CHUNK_SIZE 4096ul
31
32 #define FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC (1ULL << 0)
33 #define FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC (1ULL << 16)
34 #define FIREHOSE_CHUNK_POS_REFCNT_INC (1ULL << 32)
35 #define FIREHOSE_CHUNK_POS_FULL_BIT (1ULL << 56)
36 #define FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(pos, stream) \
37 ((((pos).fcp_pos >> 48) & 0x1ff) == (uint16_t)stream)
38
39 typedef union {
40 os_atomic(uint64_t) fcp_atomic_pos;
41 uint64_t fcp_pos;
42 struct {
43 uint16_t fcp_next_entry_offs;
44 uint16_t fcp_private_offs;
45 uint8_t fcp_refcnt;
46 uint8_t fcp_qos;
47 uint8_t fcp_stream;
48 uint8_t fcp_flag_full : 1;
49 uint8_t fcp_flag_io : 1;
50 uint8_t fcp_quarantined : 1;
51 uint8_t _fcp_flag_unused : 5;
52 };
53 } firehose_chunk_pos_u;
54
55 typedef struct firehose_chunk_s {
56 union {
57 uint8_t fc_start[FIREHOSE_CHUNK_SIZE];
58 struct {
59 firehose_chunk_pos_u fc_pos;
60 uint64_t fc_timestamp;
61 uint8_t fc_data[FIREHOSE_CHUNK_SIZE - 8 - 8];
62 };
63 };
64 } *firehose_chunk_t;
65
66 typedef struct firehose_chunk_range_s {
67 uint16_t fcr_offset; // offset from the start of the chunk
68 uint16_t fcr_length;
69 } *firehose_chunk_range_t;
70
71 #if __has_include(<os/atomic_private.h>)
72 #if defined(KERNEL) || defined(OS_FIREHOSE_SPI)
73
74 OS_ALWAYS_INLINE
75 static inline bool
firehose_chunk_pos_fits(firehose_chunk_pos_u * pos,uint16_t size)76 firehose_chunk_pos_fits(firehose_chunk_pos_u *pos, uint16_t size)
77 {
78 return pos->fcp_next_entry_offs + size <= pos->fcp_private_offs;
79 }
80
81 #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE (-1)
82 #define FIREHOSE_CHUNK_TRY_RESERVE_FAIL ( 0)
83
84 #if OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
85 OS_ALWAYS_INLINE
86 static inline long
firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc,uint64_t stamp,firehose_stream_t stream,uint8_t qos,uint16_t pubsize,uint16_t privsize,uint8_t ** privptr)87 firehose_chunk_tracepoint_try_reserve(firehose_chunk_t fc, uint64_t stamp,
88 firehose_stream_t stream, uint8_t qos, uint16_t pubsize,
89 uint16_t privsize, uint8_t **privptr)
90 {
91 const uint16_t ft_size = offsetof(struct firehose_tracepoint_s, ft_data);
92 firehose_chunk_pos_u orig, pos;
93 bool reservation_failed, stamp_delta_fits;
94
95 stamp_delta_fits = ((stamp - fc->fc_timestamp) >> 48) == 0;
96
97 // no acquire barrier because the returned space is written to only
98 os_atomic_rmw_loop(&fc->fc_pos.fcp_atomic_pos,
99 orig.fcp_pos, pos.fcp_pos, relaxed, {
100 if (orig.fcp_pos == 0) {
101 // we acquired a really really old reference, and we probably
102 // just faulted in a new page
103 os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
104 }
105 if (!FIREHOSE_CHUNK_POS_USABLE_FOR_STREAM(orig, stream)) {
106 // nothing to do if the chunk is full, or the stream doesn't match,
107 // in which case the thread probably:
108 // - loaded the chunk ref
109 // - been suspended a long while
110 // - read the chunk to find a very old thing
111 os_atomic_rmw_loop_give_up(return FIREHOSE_CHUNK_TRY_RESERVE_FAIL);
112 }
113 pos.fcp_pos = orig.fcp_pos;
114 if (!firehose_chunk_pos_fits(&orig,
115 ft_size + pubsize + privsize) || !stamp_delta_fits) {
116 pos.fcp_flag_full = true;
117 reservation_failed = true;
118 } else {
119 if (qos > pos.fcp_qos) {
120 pos.fcp_qos = qos;
121 }
122 // using these *_INC macros is so that the compiler generates better
123 // assembly: using the struct individual fields forces the compiler
124 // to handle carry propagations, and we know it won't happen
125 pos.fcp_pos += roundup(ft_size + pubsize, 8) *
126 FIREHOSE_CHUNK_POS_ENTRY_OFFS_INC;
127 pos.fcp_pos -= privsize * FIREHOSE_CHUNK_POS_PRIVATE_OFFS_INC;
128 pos.fcp_pos += FIREHOSE_CHUNK_POS_REFCNT_INC;
129 const uint16_t minimum_payload_size = 16;
130 if (!firehose_chunk_pos_fits(&pos,
131 roundup(ft_size + minimum_payload_size, 8))) {
132 // if we can't even have minimum_payload_size bytes of payload
133 // for the next tracepoint, just flush right away
134 pos.fcp_flag_full = true;
135 }
136 reservation_failed = false;
137 }
138 });
139
140 if (reservation_failed) {
141 if (pos.fcp_refcnt) {
142 // nothing to do, there is a thread writing that will pick up
143 // the "FULL" flag on flush and push as a consequence
144 return FIREHOSE_CHUNK_TRY_RESERVE_FAIL;
145 }
146 // caller must enqueue chunk
147 return FIREHOSE_CHUNK_TRY_RESERVE_FAIL_ENQUEUE;
148 }
149 if (privptr) {
150 *privptr = fc->fc_start + pos.fcp_private_offs;
151 }
152 return orig.fcp_next_entry_offs;
153 }
154
155 OS_ALWAYS_INLINE
156 static inline firehose_tracepoint_t
firehose_chunk_tracepoint_begin(firehose_chunk_t fc,uint64_t stamp,uint16_t pubsize,uint64_t thread_id,long offset)157 firehose_chunk_tracepoint_begin(firehose_chunk_t fc, uint64_t stamp,
158 uint16_t pubsize, uint64_t thread_id, long offset)
159 {
160 firehose_tracepoint_t ft = (firehose_tracepoint_t)
161 __builtin_assume_aligned(fc->fc_start + offset, 8);
162 stamp -= fc->fc_timestamp;
163 stamp |= (uint64_t)pubsize << 48;
164 // The compiler barrier is needed for userland process death handling, see
165 // (tracepoint-begin) in libdispatch's firehose_buffer_stream_chunk_install.
166 os_atomic_std(atomic_store_explicit)(&ft->ft_atomic_stamp_and_length, stamp,
167 os_atomic_std(memory_order_relaxed));
168 __asm__ __volatile__ ("" ::: "memory");
169 ft->ft_thread = thread_id;
170 return ft;
171 }
172
173 OS_ALWAYS_INLINE
174 static inline bool
firehose_chunk_tracepoint_end(firehose_chunk_t fc,firehose_tracepoint_t ft,firehose_tracepoint_id_u ftid)175 firehose_chunk_tracepoint_end(firehose_chunk_t fc,
176 firehose_tracepoint_t ft, firehose_tracepoint_id_u ftid)
177 {
178 firehose_chunk_pos_u pos;
179
180 os_atomic_std(atomic_store_explicit)(&ft->ft_id.ftid_atomic_value,
181 ftid.ftid_value, os_atomic_std(memory_order_release));
182 pos.fcp_pos = os_atomic_std(atomic_fetch_sub_explicit)(&fc->fc_pos.fcp_atomic_pos,
183 FIREHOSE_CHUNK_POS_REFCNT_INC, os_atomic_std(memory_order_relaxed));
184 return pos.fcp_refcnt == 1 && pos.fcp_flag_full;
185 }
186 #endif // OS_ATOMIC_HAS_STARVATION_FREE_RMW || !OS_ATOMIC_CONFIG_STARVATION_FREE_ONLY
187
188 #endif // defined(KERNEL) || defined(OS_FIREHOSE_SPI)
189 #endif // __has_include(<os/atomic_private.h>)
190
191 __END_DECLS
192
193 #endif // __FIREHOSE_CHUNK_PRIVATE__
194