1 /* 2 * Copyright (c) 2000-2014 Apple Inc. All rights reserved. 3 * 4 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@ 5 * 6 * This file contains Original Code and/or Modifications of Original Code 7 * as defined in and that are subject to the Apple Public Source License 8 * Version 2.0 (the 'License'). You may not use this file except in 9 * compliance with the License. The rights granted to you under the License 10 * may not be used to create, or enable the creation or redistribution of, 11 * unlawful or unlicensed copies of an Apple operating system, or to 12 * circumvent, violate, or enable the circumvention or violation of, any 13 * terms of an Apple operating system software license agreement. 14 * 15 * Please obtain a copy of the License at 16 * http://www.opensource.apple.com/apsl/ and read it before using this file. 17 * 18 * The Original Code and all software distributed under the License are 19 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER 20 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 21 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. 23 * Please see the License for the specific language governing rights and 24 * limitations under the License. 25 * 26 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@ 27 */ 28 /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */ 29 /* 30 * Copyright (c) 1982, 1986, 1989, 1993 31 * The Regents of the University of California. All rights reserved. 32 * (c) UNIX System Laboratories, Inc. 33 * All or some portions of this file are derived from material licensed 34 * to the University of California by American Telephone and Telegraph 35 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 36 * the permission of UNIX System Laboratories, Inc. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the University of 49 * California, Berkeley and its contributors. 50 * 4. Neither the name of the University nor the names of its contributors 51 * may be used to endorse or promote products derived from this software 52 * without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 56 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 57 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 58 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 59 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 60 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 61 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 62 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 63 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 64 * SUCH DAMAGE. 65 * 66 * @(#)buf.h 8.9 (Berkeley) 3/30/95 67 */ 68 69 #ifndef _SYS_BUF_INTERNAL_H_ 70 #define _SYS_BUF_INTERNAL_H_ 71 72 #include <sys/appleapiopts.h> 73 74 #ifdef KERNEL 75 #include <sys/queue.h> 76 #include <sys/errno.h> 77 #include <sys/vm.h> 78 #include <sys/cdefs.h> 79 #include <sys/buf.h> 80 #include <sys/lock.h> 81 82 #if CONFIG_PROTECT 83 #include <sys/cprotect.h> 84 #endif 85 86 #define NOLIST ((struct buf *)0x87654321) 87 88 /* 89 * Attributes of an I/O to be used by lower layers 90 */ 91 struct bufattr { 92 #if CONFIG_PROTECT 93 struct cpx *ba_cpx; 94 uint64_t ba_cp_file_off; 95 #endif 96 uint64_t ba_flags; /* flags. Some are only in-use on embedded devices */ 97 union { 98 void *verify_ctx; 99 void *verify_ptr; /* only for metadata, B_CLUSTER not set */ 100 } ba_un; 101 vnode_verify_kind_t ba_verify_type; 102 }; 103 104 /* 105 * The buffer header describes an I/O operation in the kernel. 106 */ 107 struct buf { 108 LIST_ENTRY(buf) b_hash; /* Hash chain. */ 109 LIST_ENTRY(buf) b_vnbufs; /* Buffer's associated vnode. */ 110 TAILQ_ENTRY(buf) b_freelist; /* Free list position if not active. */ 111 int b_timestamp; /* timestamp for queuing operation */ 112 struct timeval b_timestamp_tv; /* microuptime for disk conditioner */ 113 int b_whichq; /* the free list the buffer belongs to */ 114 volatile uint32_t b_flags; /* B_* flags. */ 115 volatile uint32_t b_lflags; /* BL_BUSY | BL_WANTED flags... protected by buf_mtx */ 116 int b_error; /* errno value. */ 117 int b_bufsize; /* Allocated buffer size. */ 118 int b_bcount; /* Valid bytes in buffer. */ 119 int b_resid; /* Remaining I/O. */ 120 dev_t b_dev; /* Device associated with buffer. */ 121 uintptr_t b_datap; /* Memory, superblocks, indirect etc.*/ 122 daddr64_t b_lblkno; /* Logical block number. */ 123 daddr64_t b_blkno; /* Underlying physical block number. */ 124 void (*b_iodone)(buf_t, void *); /* Function to call upon completion. */ 125 vnode_t b_vp; /* File vnode for data, device vnode for metadata. */ 126 kauth_cred_t b_rcred; /* Read credentials reference. */ 127 kauth_cred_t b_wcred; /* Write credentials reference. */ 128 void * b_upl; /* Pointer to UPL */ 129 buf_t b_real_bp; /* used to track bp generated through cluster_bp */ 130 TAILQ_ENTRY(buf) b_act; /* Device driver queue when active */ 131 void * b_drvdata; /* Device driver private use */ 132 void * b_fsprivate; /* filesystem private use */ 133 void * b_transaction; /* journal private use */ 134 int b_dirtyoff; /* Offset in buffer of dirty region. */ 135 int b_dirtyend; /* Offset of end of dirty region. */ 136 int b_validoff; /* Offset in buffer of valid region. */ 137 int b_validend; /* Offset of end of valid region. */ 138 139 /* store extra information related to redundancy of data, such as 140 * which redundancy copy to use, etc 141 */ 142 uint32_t b_redundancy_flags; 143 144 proc_t b_proc; /* Associated proc; NULL if kernel. */ 145 #ifdef BUF_MAKE_PRIVATE 146 buf_t b_data_store; 147 #endif 148 struct bufattr b_attr; 149 #ifdef JOE_DEBUG 150 void * b_owner; 151 int b_tag; 152 void * b_lastbrelse; 153 void * b_stackbrelse[6]; 154 void * b_stackgetblk[6]; 155 #endif 156 off_t b_lblksize; /* Block size used to set b_lbkno if not B_CLUSTER*/ 157 vnode_t b_vnop_vp; /* identifies vp on which VNOP has been called */ 158 }; 159 160 extern vm_offset_t buf_kernel_addrperm; 161 162 /* cluster_io definitions for use with io bufs */ 163 #define b_uploffset b_bufsize 164 #define b_orig b_freelist.tqe_prev 165 #define b_shadow b_freelist.tqe_next 166 #define b_shadow_ref b_validoff 167 #ifdef BUF_MAKE_PRIVATE 168 #define b_data_ref b_validend 169 #endif 170 #define b_trans_head b_freelist.tqe_prev 171 #define b_trans_next b_freelist.tqe_next 172 #define b_iostate b_rcred 173 #define b_cliodone b_wcred 174 #define b_clfoffset b_lblksize 175 176 /* 177 * These flags are kept in b_lflags... 178 * buf_mtx must be held before examining/updating 179 */ 180 #define BL_BUSY 0x00000001 /* I/O in progress. */ 181 #define BL_WANTED 0x00000002 /* Process wants this buffer. */ 182 #define BL_IOBUF 0x00000004 /* buffer allocated via 'buf_alloc' */ 183 #define BL_WANTDEALLOC 0x00000010 /* buffer should be put on empty list when clean */ 184 #define BL_SHADOW 0x00000020 185 #define BL_EXTERNAL 0x00000040 186 #define BL_WAITSHADOW 0x00000080 187 #define BL_IOBUF_ALLOC 0x00000100 188 #define BL_WANTED_REF 0x00000200 189 #define BL_IOBUF_VDEV 0x00000400 /* iobuf was for a diskimage */ 190 191 /* 192 * Parameters for buffer cache garbage collection 193 */ 194 #define BUF_STALE_THRESHHOLD 30 /* Collect if untouched in the last 30 seconds */ 195 #define BUF_MAX_GC_BATCH_SIZE 64 /* Under a single grab of the lock */ 196 197 /* 198 * mask used by buf_flags... these are the readable external flags 199 */ 200 #define BUF_X_RDFLAGS (B_PHYS | B_RAW | B_LOCKED | B_ASYNC | B_READ | B_WRITE | B_PAGEIO |\ 201 B_META | B_CLUSTER | B_DELWRI | B_FUA | B_PASSIVE | B_IOSTREAMING |\ 202 B_ENCRYPTED_IO | B_STATICCONTENT) 203 /* 204 * mask used by buf_clearflags/buf_setflags... these are the writable external flags 205 */ 206 #define BUF_X_WRFLAGS (B_PHYS | B_RAW | B_LOCKED | B_ASYNC | B_READ | B_WRITE | B_PAGEIO |\ 207 B_NOCACHE | B_FUA | B_PASSIVE | B_IOSTREAMING) 208 209 #if 0 210 /* b_flags defined in buf.h */ 211 #define B_WRITE 0x00000000 /* Write buffer (pseudo flag). */ 212 #define B_READ 0x00000001 /* Read buffer. */ 213 #define B_ASYNC 0x00000002 /* Start I/O, do not wait. */ 214 #define B_NOCACHE 0x00000004 /* Do not cache block after use. */ 215 #define B_DELWRI 0x00000008 /* Delay I/O until buffer reused. */ 216 #define B_LOCKED 0x00000010 /* Locked in core (not reusable). */ 217 #define B_PHYS 0x00000020 /* I/O to user memory. */ 218 #define B_CLUSTER 0x00000040 /* UPL based I/O generated by cluster layer */ 219 #define B_PAGEIO 0x00000080 /* Page in/out */ 220 #define B_META 0x00000100 /* buffer contains meta-data. */ 221 #define B_RAW 0x00000200 /* Set by physio for raw transfers. */ 222 #define B_FUA 0x00000400 /* Write-through disk cache(if supported) */ 223 #define B_PASSIVE 0x00000800 /* PASSIVE I/Os are ignored by THROTTLE I/O */ 224 #define B_IOSTREAMING 0x00001000 /* sequential access pattern detected */ 225 #define B_ENCRYPTED_IO 0x00004000 /* Encrypted I/O */ 226 #define B_STATICCONTENT 0x00008000 /* Buffer is likely to remain unaltered */ 227 #endif 228 229 /* 230 * These flags are kept in b_flags... access is lockless 231 * External flags are defined in buf.h and cannot overlap 232 * the internal flags 233 * 234 * these flags are internal... there definition may change 235 */ 236 #define B_CACHE 0x00010000 /* getblk found us in the cache. */ 237 #define B_DONE 0x00020000 /* I/O completed. */ 238 #define B_INVAL 0x00040000 /* Does not contain valid info. */ 239 #define B_ERROR 0x00080000 /* I/O error occurred. */ 240 #define B_EINTR 0x00100000 /* I/O was interrupted */ 241 #define B_AGE 0x00200000 /* Move to age queue when I/O done. */ 242 #define B_FILTER 0x00400000 /* call b_iodone from biodone as an in-line filter */ 243 #define B_CALL 0x00800000 /* Call b_iodone from biodone, assumes b_iodone consumes bp */ 244 #define B_EOT 0x01000000 /* last buffer in a transaction list created by cluster_io */ 245 #define B_WASDIRTY 0x02000000 /* page was found dirty in the VM cache */ 246 #define B_HDRALLOC 0x04000000 /* zone allocated buffer header */ 247 #define B_ZALLOC 0x08000000 /* b_datap is zalloc()ed */ 248 /* 249 * private flags used by by the cluster layer 250 */ 251 #define B_COMMIT_UPL 0x40000000 /* commit/abort the UPL on I/O success/failure */ 252 #define B_TDONE 0x80000000 /* buf_t that is part of a cluster level transaction has completed */ 253 254 /* Flags to low-level allocation routines. */ 255 #define B_CLRBUF 0x01 /* Request allocated buffer be cleared. */ 256 #define B_SYNC 0x02 /* Do all allocations synchronously. */ 257 #define B_NOBUFF 0x04 /* Do not allocate struct buf */ 258 259 /* 260 * ba_flags (Buffer Attribute flags) 261 * Some of these may be in-use only on embedded devices. 262 */ 263 #define BA_RAW_ENCRYPTED_IO 0x00000001 264 #define BA_THROTTLED_IO 0x00000002 265 #define BA_DELAYIDLESLEEP 0x00000004 /* Process is marked to delay idle sleep on disk IO */ 266 #define BA_NOCACHE 0x00000008 267 #define BA_META 0x00000010 268 #define BA_GREEDY_MODE 0x00000020 /* High speed writes that consume more storage */ 269 #define BA_QUICK_COMPLETE 0x00000040 /* Request quick completion at expense of storage efficiency */ 270 #define BA_PASSIVE 0x00000080 271 272 /* 273 * Note: IO_TIERs consume 0x0100, 0x0200, 0x0400, 0x0800 274 * These are now in-use by the I/O tiering system. 275 */ 276 #define BA_IO_TIER_MASK 0x00000f00 277 #define BA_IO_TIER_SHIFT 8 278 279 #define BA_ISOCHRONOUS 0x00001000 /* device specific isochronous throughput to media */ 280 281 #define BA_STRATEGY_TRACKED_IO 0x00002000 /* tracked by spec_strategy */ 282 #define BA_IO_TIER_UPGRADE 0x00004000 /* effective I/O tier is higher than BA_IO_TIER */ 283 #define BA_IO_SCHEDULED 0x00008000 /* buf is associated with a mount point that is io scheduled */ 284 #define BA_EXPEDITED_META_IO 0x00010000 /* metadata I/O which needs a high I/O tier */ 285 #define BA_WILL_VERIFY 0x00020000 /* Cluster layer will verify data */ 286 #define BA_ASYNC_VERIFY 0x00040000 /* Allowed to hand off to async threads */ 287 #define BA_VERIFY_VALID 0x00080000 /* Hash calculated by driver is valid */ 288 289 #define GET_BUFATTR_IO_TIER(bap) ((bap->ba_flags & BA_IO_TIER_MASK) >> BA_IO_TIER_SHIFT) 290 #define SET_BUFATTR_IO_TIER(bap, tier) \ 291 do { \ 292 (bap)->ba_flags &= (~BA_IO_TIER_MASK); \ 293 (bap)->ba_flags |= (((tier) << BA_IO_TIER_SHIFT) & BA_IO_TIER_MASK); \ 294 } while(0) 295 296 extern int niobuf_headers; /* The number of IO buffer headers for cluster IO */ 297 extern int nbuf_headers; /* The number of buffer headers */ 298 extern int max_nbuf_headers; /* The max number of buffer headers */ 299 extern int nbuf_hashelements; /* The number of elements in bufhash */ 300 extern struct buf *buf_headers; /* The buffer headers. */ 301 302 303 /* 304 * Definitions for the buffer free lists. 305 */ 306 307 enum bq_opts { 308 BQ_LOCKED = 0, /* super-blocks &c */ 309 BQ_LRU = 1, /* lru, useful buffers */ 310 BQ_AGE = 2, /* rubbish */ 311 BQ_EMPTY = 3, /* buffer headers with no memory */ 312 BQ_META = 4, /* buffer containing metadata */ 313 BQ_LAUNDRY = 5, /* buffers that need cleaning */ 314 BQUEUES = 6 /* number of free buffer queues */ 315 }; 316 317 #define CLUSTER_IO_BLOCK_SIZE 0x1000 318 319 __BEGIN_DECLS 320 321 buf_t alloc_io_buf(vnode_t, int); 322 void free_io_buf(buf_t); 323 324 int allocbuf(struct buf *, int); 325 void bufinit(void); 326 327 void buf_list_lock(void); 328 void buf_list_unlock(void); 329 330 void cluster_init(void); 331 332 uint32_t count_busy_buffers(void); 333 334 int buf_flushdirtyblks_skipinfo(vnode_t, int, int, const char *); 335 void buf_wait_for_shadow_io(vnode_t, daddr64_t); 336 337 #ifdef BUF_MAKE_PRIVATE 338 errno_t buf_make_private(buf_t bp); 339 #endif 340 341 #ifdef CONFIG_PROTECT 342 void buf_setcpoff(buf_t, uint64_t); 343 #endif 344 345 vnode_t buf_vnop_vnode(buf_t); 346 347 uint32_t get_num_bytes_for_verify_kind(vnode_verify_kind_t); 348 uint8_t * buf_verifyptr_with_size(buf_t bp, int size, uint32_t *len); 349 void buf_verify_free(buf_t bp); 350 351 __END_DECLS 352 353 354 /* 355 * Stats on usefulness of the buffer cache 356 */ 357 struct bufstats { 358 long bufs_incore; /* found incore */ 359 long bufs_busyincore; /* found incore. was busy */ 360 long bufs_vmhits; /* not incore. found in VM */ 361 long bufs_miss; /* not incore. not in VM */ 362 long bufs_sleeps; /* buffer starvation */ 363 long bufs_eblk; /* Calls to geteblk */ 364 uint32_t bufs_iobufmax; /* Max. number of IO buffers used */ 365 uint32_t bufs_iobufinuse; /* number of IO buffers in use */ 366 long bufs_iobufsleeps; /* IO buffer starvation */ 367 long bufs_iobufinuse_vdev; /* number of IO buffers in use by 368 * diskimages */ 369 }; 370 371 #endif /* KERNEL */ 372 #endif /* !_SYS_BUF_H_ */ 373