Lines Matching refs:cpu_data_ptr
317 cpu_data_t *cpu_data_ptr = getCpuDatap(); in cpu_sleep() local
319 cpu_data_ptr->cpu_active_thread = current_thread(); in cpu_sleep()
321 cpu_data_ptr->cpu_reset_handler = (uintptr_t) VM_KERNEL_STRIP_PTR(arm_init_cpu); in cpu_sleep()
323 cpu_data_ptr->cpu_reset_handler = (uintptr_t) start_cpu_paddr; in cpu_sleep()
325 cpu_data_ptr->cpu_flags |= SleepState; in cpu_sleep()
327 if (cpu_data_ptr->cpu_user_debug != NULL) { in cpu_sleep()
333 mt_cpu_down(cpu_data_ptr); in cpu_sleep()
343 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); in cpu_sleep()
345 bool deep_sleep = PE_cpu_down(cpu_data_ptr->cpu_id); in cpu_sleep()
346 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; in cpu_sleep()
348 cpu_data_ptr->cpu_reset_handler = (uintptr_t)0; in cpu_sleep()
352 …arm64_prepare_for_sleep(deep_sleep, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVec… in cpu_sleep()
358 PE_cpu_machine_quiesce(cpu_data_ptr->cpu_id); in cpu_sleep()
493 cpu_data_t *cpu_data_ptr = getCpuDatap(); in cpu_idle() local
503 } else if (__improbable(cpu_data_ptr->cpu_signal & SIGPdisabled)) { in cpu_idle()
512 uint32_t cid = cpu_data_ptr->cpu_cluster_id; in cpu_idle()
536 …KDBG(CPUPM_IDLE_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, wfe_timeout, !check_clust… in cpu_idle()
537 …ipending = wfe_to_deadline_or_interrupt(cid, wfe_deadline, cpu_data_ptr, false, check_cluster_reco… in cpu_idle()
538 KDBG(CPUPM_IDLE_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, wfe_deadline); in cpu_idle()
550 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_START, ipending, cpu_data_ptr->wfe_count, ~0ULL); in cpu_idle()
551 ipending = wfe_to_deadline_or_interrupt(cid, ~0ULL, cpu_data_ptr, false, false); in cpu_idle()
552 KDBG(CPUPM_IDLE_TIMER_WFE | DBG_FUNC_END, ipending, cpu_data_ptr->wfe_count, ~0ULL); in cpu_idle()
558 lastPop = cpu_data_ptr->rtcPop; in cpu_idle()
560 cpu_data_ptr->cpu_active_thread = current_thread(); in cpu_idle()
562 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) { in cpu_idle()
563 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, TRUE, &new_idle_timeout_ticks); in cpu_idle()
566 if (cpu_data_ptr->idle_timer_notify != NULL) { in cpu_idle()
569 cpu_data_ptr->idle_timer_deadline = 0x0ULL; in cpu_idle()
572 …clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline… in cpu_idle()
575 if (cpu_data_ptr->rtcPop != lastPop) { in cpu_idle()
582 mt_cpu_idle(cpu_data_ptr); in cpu_idle()
652 cpu_data_t *cpu_data_ptr = getCpuDatap(); in cpu_idle_exit() local
658 configure_coresight_registers(cpu_data_ptr); in cpu_idle_exit()
663 mt_cpu_run(cpu_data_ptr); in cpu_idle_exit()
666 if (wfi && (cpu_data_ptr->cpu_idle_notify != NULL)) { in cpu_idle_exit()
667 cpu_data_ptr->cpu_idle_notify(cpu_data_ptr->cpu_id, FALSE, &new_idle_timeout_ticks); in cpu_idle_exit()
670 if (cpu_data_ptr->idle_timer_notify != NULL) { in cpu_idle_exit()
673 cpu_data_ptr->idle_timer_deadline = 0x0ULL; in cpu_idle_exit()
676 …clock_absolutetime_interval_to_deadline(new_idle_timeout_ticks, &cpu_data_ptr->idle_timer_deadline… in cpu_idle_exit()
752 cpu_stack_alloc(cpu_data_t *cpu_data_ptr) in cpu_stack_alloc() argument
762 cpu_data_ptr->intstack_top = irq_stack + PAGE_SIZE + INTSTACK_SIZE; in cpu_stack_alloc()
763 cpu_data_ptr->istackptr = (void *)cpu_data_ptr->intstack_top; in cpu_stack_alloc()
770 cpu_data_ptr->excepstack_top = exc_stack + PAGE_SIZE + EXCEPSTACK_SIZE; in cpu_stack_alloc()
774 cpu_data_free(cpu_data_t *cpu_data_ptr) in cpu_data_free() argument
776 if ((cpu_data_ptr == NULL) || (cpu_data_ptr == &BootCpuData)) { in cpu_data_free()
780 int cpu_number = cpu_data_ptr->cpu_number; in cpu_data_free()
782 if (CpuDataEntries[cpu_number].cpu_data_vaddr == cpu_data_ptr) { in cpu_data_free()
788 cpu_data_ptr->intstack_top - INTSTACK_SIZE - PAGE_SIZE, in cpu_data_free()
791 cpu_data_ptr->excepstack_top - EXCEPSTACK_SIZE - PAGE_SIZE, in cpu_data_free()
796 cpu_data_init(cpu_data_t *cpu_data_ptr) in cpu_data_init() argument
800 cpu_data_ptr->cpu_flags = 0; in cpu_data_init()
801 cpu_data_ptr->cpu_int_state = 0; in cpu_data_init()
802 cpu_data_ptr->cpu_pending_ast = AST_NONE; in cpu_data_init()
803 cpu_data_ptr->cpu_cache_dispatch = NULL; in cpu_data_init()
804 cpu_data_ptr->rtcPop = EndOfAllTime; in cpu_data_init()
805 cpu_data_ptr->rtclock_datap = &RTClockData; in cpu_data_init()
806 cpu_data_ptr->cpu_user_debug = NULL; in cpu_data_init()
809 cpu_data_ptr->cpu_base_timebase = 0; in cpu_data_init()
810 cpu_data_ptr->cpu_idle_notify = NULL; in cpu_data_init()
811 cpu_data_ptr->cpu_idle_latency = 0x0ULL; in cpu_data_init()
812 cpu_data_ptr->cpu_idle_pop = 0x0ULL; in cpu_data_init()
813 cpu_data_ptr->cpu_reset_type = 0x0UL; in cpu_data_init()
814 cpu_data_ptr->cpu_reset_handler = 0x0UL; in cpu_data_init()
815 cpu_data_ptr->cpu_reset_assist = 0x0UL; in cpu_data_init()
816 cpu_data_ptr->cpu_regmap_paddr = 0x0ULL; in cpu_data_init()
817 cpu_data_ptr->cpu_phys_id = 0x0UL; in cpu_data_init()
818 cpu_data_ptr->cpu_l2_access_penalty = 0; in cpu_data_init()
819 cpu_data_ptr->cpu_cluster_type = CLUSTER_TYPE_SMP; in cpu_data_init()
820 cpu_data_ptr->cpu_cluster_id = 0; in cpu_data_init()
821 cpu_data_ptr->cpu_l2_id = 0; in cpu_data_init()
822 cpu_data_ptr->cpu_l2_size = 0; in cpu_data_init()
823 cpu_data_ptr->cpu_l3_id = 0; in cpu_data_init()
824 cpu_data_ptr->cpu_l3_size = 0; in cpu_data_init()
826 cpu_data_ptr->cpu_signal = SIGPdisabled; in cpu_data_init()
828 cpu_data_ptr->cpu_get_fiq_handler = NULL; in cpu_data_init()
829 cpu_data_ptr->cpu_tbd_hardware_addr = NULL; in cpu_data_init()
830 cpu_data_ptr->cpu_tbd_hardware_val = NULL; in cpu_data_init()
831 cpu_data_ptr->cpu_get_decrementer_func = NULL; in cpu_data_init()
832 cpu_data_ptr->cpu_set_decrementer_func = NULL; in cpu_data_init()
833 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; in cpu_data_init()
834 cpu_data_ptr->cpu_sleep_token_last = 0x00000000UL; in cpu_data_init()
835 cpu_data_ptr->cpu_xcall_p0 = NULL; in cpu_data_init()
836 cpu_data_ptr->cpu_xcall_p1 = NULL; in cpu_data_init()
837 cpu_data_ptr->cpu_imm_xcall_p0 = NULL; in cpu_data_init()
838 cpu_data_ptr->cpu_imm_xcall_p1 = NULL; in cpu_data_init()
841 cpu_data_ptr->coresight_base[i] = 0; in cpu_data_init()
845 pmap_cpu_data_t * pmap_cpu_data_ptr = &cpu_data_ptr->cpu_pmap_cpu_data; in cpu_data_init()
856 cpu_data_ptr->halt_status = CPU_NOT_HALTED; in cpu_data_init()
858 cpu_data_ptr->cpu_exc_vectors = (vm_offset_t)&exc_vectors_table; in cpu_data_init()
862 cpu_data_ptr->rop_key = 0; in cpu_data_init()
863 cpu_data_ptr->jop_key = ml_default_jop_pid(); in cpu_data_init()
868 cpu_data_register(cpu_data_t *cpu_data_ptr) in cpu_data_register() argument
870 int cpu = cpu_data_ptr->cpu_number; in cpu_data_register()
879 CpuDataEntries[cpu].cpu_data_vaddr = cpu_data_ptr; in cpu_data_register()
880 CpuDataEntries[cpu].cpu_data_paddr = (void *)ml_vtophys((vm_offset_t)cpu_data_ptr); in cpu_data_register()
905 cpu_data_t *cpu_data_ptr = CpuDataEntries[cpu].cpu_data_vaddr; in cpu_start() local
906 processor_t processor = PERCPU_GET_RELATIVE(processor, cpu_data, cpu_data_ptr); in cpu_start()
914 configure_coresight_registers(cpu_data_ptr); in cpu_start()
918 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) VM_KERNEL_STRIP_PTR(arm_init_cpu); in cpu_start()
920 cpu_data_ptr->cpu_reset_handler = (vm_offset_t) start_cpu_paddr; in cpu_start()
922 cpu_data_ptr->cpu_pmap_cpu_data.cpu_nested_pmap = NULL; in cpu_start()
931 cpu_data_ptr->cpu_active_thread = first_thread; in cpu_start()
932 first_thread->machine.CpuDatap = cpu_data_ptr; in cpu_start()
934 (char *)cpu_data_ptr - __PERCPU_ADDR(cpu_data); in cpu_start()
936 configure_coresight_registers(cpu_data_ptr); in cpu_start()
939 flush_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); in cpu_start()
951 switch (ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id]) { in cpu_start()
953 ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] = CTRR_LOCKING; in cpu_start()
957 assert_wait(&ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id], THREAD_UNINT); in cpu_start()
960 assert(ctrr_cluster_locked[cpu_data_ptr->cpu_cluster_id] != CTRR_LOCKING); in cpu_start()
968 (void) PE_cpu_start(cpu_data_ptr->cpu_id, (vm_offset_t)NULL, (vm_offset_t)NULL); in cpu_start()
1027 cpu_data_t *cpu_data_ptr = getCpuDatap(); in ml_arm_sleep() local
1029 if (cpu_data_ptr == &BootCpuData) { in ml_arm_sleep()
1038 if ((target_cdp == NULL) || (target_cdp == cpu_data_ptr)) { in ml_arm_sleep()
1057 cpu_data_ptr->cpu_sleep_token = ARM_CPU_ON_SLEEP_PATH; in ml_arm_sleep()
1059 if (cpu_data_ptr == &BootCpuData) { in ml_arm_sleep()
1085 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) { in ml_arm_sleep()
1086 …*(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM… in ml_arm_sleep()
1087 … *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; in ml_arm_sleep()
1115 …arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBas… in ml_arm_sleep()
1136 CleanPoU_DcacheRegion((vm_offset_t) cpu_data_ptr, sizeof(cpu_data_t)); in ml_arm_sleep()
1142 if (cpu_data_ptr->coresight_base[CORESIGHT_ED]) { in ml_arm_sleep()
1143 …*(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGLAR) = ARM… in ml_arm_sleep()
1144 … *(volatile uint32_t *)(cpu_data_ptr->coresight_base[CORESIGHT_ED] + ARM_DEBUG_OFFSET_DBGPRCR) = 0; in ml_arm_sleep()
1149 …arm64_prepare_for_sleep(true, cpu_data_ptr->cpu_number, ml_vtophys((vm_offset_t)&LowResetVectorBas… in ml_arm_sleep()
1162 cpu_data_t *cpu_data_ptr = getCpuDatap(); in cpu_machine_idle_init() local
1249 if (cpu_data_ptr == &BootCpuData) { in cpu_machine_idle_init()
1264 cpu_data_ptr->cpu_reset_handler = (uintptr_t) VM_KERNEL_STRIP_PTR(arm_init_idle_cpu); in cpu_machine_idle_init()
1266 cpu_data_ptr->cpu_reset_handler = resume_idle_cpu_paddr; in cpu_machine_idle_init()
1268 clean_dcache((vm_offset_t)cpu_data_ptr, sizeof(cpu_data_t), FALSE); in cpu_machine_idle_init()
1287 cpu_data_t *cpu_data_ptr = getCpuDatap(); in sleep_token_buffer_init() local
1292 if ((cpu_data_ptr == &BootCpuData) && (sleepTokenBuffer == (vm_offset_t) NULL)) { in sleep_token_buffer_init()