Lines Matching full:gt

40 void intel_gt_common_init_early(struct intel_gt *gt)  in intel_gt_common_init_early()  argument
42 spin_lock_init(gt->irq_lock); in intel_gt_common_init_early()
44 INIT_LIST_HEAD(&gt->closed_vma); in intel_gt_common_init_early()
45 spin_lock_init(&gt->closed_lock); in intel_gt_common_init_early()
47 init_llist_head(&gt->watchdog.list); in intel_gt_common_init_early()
48 INIT_WORK(&gt->watchdog.work, intel_gt_watchdog_work); in intel_gt_common_init_early()
50 intel_gt_init_buffer_pool(gt); in intel_gt_common_init_early()
51 intel_gt_init_reset(gt); in intel_gt_common_init_early()
52 intel_gt_init_requests(gt); in intel_gt_common_init_early()
53 intel_gt_init_timelines(gt); in intel_gt_common_init_early()
54 intel_gt_init_tlb(gt); in intel_gt_common_init_early()
55 intel_gt_pm_init_early(gt); in intel_gt_common_init_early()
57 intel_wopcm_init_early(&gt->wopcm); in intel_gt_common_init_early()
58 intel_uc_init_early(&gt->uc); in intel_gt_common_init_early()
59 intel_rps_init_early(&gt->rps); in intel_gt_common_init_early()
65 struct intel_gt *gt = to_gt(i915); in intel_root_gt_init_early() local
67 gt->i915 = i915; in intel_root_gt_init_early()
68 gt->uncore = &i915->uncore; in intel_root_gt_init_early()
69 gt->irq_lock = drmm_kzalloc(&i915->drm, sizeof(*gt->irq_lock), GFP_KERNEL); in intel_root_gt_init_early()
70 if (!gt->irq_lock) in intel_root_gt_init_early()
73 intel_gt_common_init_early(gt); in intel_root_gt_init_early()
78 static int intel_gt_probe_lmem(struct intel_gt *gt) in intel_gt_probe_lmem() argument
80 struct drm_i915_private *i915 = gt->i915; in intel_gt_probe_lmem()
81 unsigned int instance = gt->info.id; in intel_gt_probe_lmem()
86 mem = intel_gt_setup_lmem(gt); in intel_gt_probe_lmem()
92 gt_err(gt, "Failed to setup region(%d) type=%d\n", in intel_gt_probe_lmem()
109 int intel_gt_assign_ggtt(struct intel_gt *gt) in intel_gt_assign_ggtt() argument
111 /* Media GT shares primary GT's GGTT */ in intel_gt_assign_ggtt()
112 if (gt->type == GT_MEDIA) { in intel_gt_assign_ggtt()
113 gt->ggtt = to_gt(gt->i915)->ggtt; in intel_gt_assign_ggtt()
115 gt->ggtt = i915_ggtt_create(gt->i915); in intel_gt_assign_ggtt()
116 if (IS_ERR(gt->ggtt)) in intel_gt_assign_ggtt()
117 return PTR_ERR(gt->ggtt); in intel_gt_assign_ggtt()
120 list_add_tail(&gt->ggtt_link, &gt->ggtt->gt_list); in intel_gt_assign_ggtt()
125 int intel_gt_init_mmio(struct intel_gt *gt) in intel_gt_init_mmio() argument
127 intel_gt_init_clock_frequency(gt); in intel_gt_init_mmio()
129 intel_uc_init_mmio(&gt->uc); in intel_gt_init_mmio()
130 intel_sseu_info_init(gt); in intel_gt_init_mmio()
131 intel_gt_mcr_init(gt); in intel_gt_init_mmio()
133 return intel_engines_init_mmio(gt); in intel_gt_init_mmio()
136 static void init_unused_ring(struct intel_gt *gt, u32 base) in init_unused_ring() argument
138 struct intel_uncore *uncore = gt->uncore; in init_unused_ring()
146 static void init_unused_rings(struct intel_gt *gt) in init_unused_rings() argument
148 struct drm_i915_private *i915 = gt->i915; in init_unused_rings()
151 init_unused_ring(gt, PRB1_BASE); in init_unused_rings()
152 init_unused_ring(gt, SRB0_BASE); in init_unused_rings()
153 init_unused_ring(gt, SRB1_BASE); in init_unused_rings()
154 init_unused_ring(gt, SRB2_BASE); in init_unused_rings()
155 init_unused_ring(gt, SRB3_BASE); in init_unused_rings()
157 init_unused_ring(gt, SRB0_BASE); in init_unused_rings()
158 init_unused_ring(gt, SRB1_BASE); in init_unused_rings()
160 init_unused_ring(gt, PRB1_BASE); in init_unused_rings()
161 init_unused_ring(gt, PRB2_BASE); in init_unused_rings()
165 int intel_gt_init_hw(struct intel_gt *gt) in intel_gt_init_hw() argument
167 struct drm_i915_private *i915 = gt->i915; in intel_gt_init_hw()
168 struct intel_uncore *uncore = gt->uncore; in intel_gt_init_hw()
171 gt->last_init_time = ktime_get(); in intel_gt_init_hw()
185 /* Apply the GT workarounds... */ in intel_gt_init_hw()
186 intel_gt_apply_workarounds(gt); in intel_gt_init_hw()
188 intel_gt_verify_workarounds(gt, "init"); in intel_gt_init_hw()
190 intel_gt_init_swizzling(gt); in intel_gt_init_hw()
198 init_unused_rings(gt); in intel_gt_init_hw()
200 ret = i915_ppgtt_init_hw(gt); in intel_gt_init_hw()
202 gt_err(gt, "Enabling PPGTT failed (%d)\n", ret); in intel_gt_init_hw()
207 ret = intel_uc_init_hw(&gt->uc); in intel_gt_init_hw()
209 gt_probe_error(gt, "Enabling uc failed (%d)\n", ret); in intel_gt_init_hw()
213 intel_mocs_init(gt); in intel_gt_init_hw()
226 i915_reg_t intel_gt_perf_limit_reasons_reg(struct intel_gt *gt) in intel_gt_perf_limit_reasons_reg() argument
229 if (GRAPHICS_VER(gt->i915) < 11) in intel_gt_perf_limit_reasons_reg()
232 return gt->type == GT_MEDIA ? in intel_gt_perf_limit_reasons_reg()
237 intel_gt_clear_error_registers(struct intel_gt *gt, in intel_gt_clear_error_registers() argument
240 struct drm_i915_private *i915 = gt->i915; in intel_gt_clear_error_registers()
241 struct intel_uncore *uncore = gt->uncore; in intel_gt_clear_error_registers()
259 gt_dbg(gt, "EIR stuck: 0x%08x, masking\n", eir); in intel_gt_clear_error_registers()
266 intel_gt_mcr_multicast_rmw(gt, XEHP_RING_FAULT_REG, in intel_gt_clear_error_registers()
268 intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG); in intel_gt_clear_error_registers()
279 for_each_engine_masked(engine, gt, engine_mask, id) in intel_gt_clear_error_registers()
284 static void gen6_check_faults(struct intel_gt *gt) in gen6_check_faults() argument
290 for_each_engine(engine, gt, id) { in gen6_check_faults()
293 gt_dbg(gt, "Unexpected fault\n" in gen6_check_faults()
307 static void xehp_check_faults(struct intel_gt *gt) in xehp_check_faults() argument
318 fault = intel_gt_mcr_read_any(gt, XEHP_RING_FAULT_REG); in xehp_check_faults()
323 fault_data0 = intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA0); in xehp_check_faults()
324 fault_data1 = intel_gt_mcr_read_any(gt, XEHP_FAULT_TLB_DATA1); in xehp_check_faults()
329 gt_dbg(gt, "Unexpected fault\n" in xehp_check_faults()
343 static void gen8_check_faults(struct intel_gt *gt) in gen8_check_faults() argument
345 struct intel_uncore *uncore = gt->uncore; in gen8_check_faults()
349 if (GRAPHICS_VER(gt->i915) >= 12) { in gen8_check_faults()
370 gt_dbg(gt, "Unexpected fault\n" in gen8_check_faults()
384 void intel_gt_check_and_clear_faults(struct intel_gt *gt) in intel_gt_check_and_clear_faults() argument
386 struct drm_i915_private *i915 = gt->i915; in intel_gt_check_and_clear_faults()
390 xehp_check_faults(gt); in intel_gt_check_and_clear_faults()
392 gen8_check_faults(gt); in intel_gt_check_and_clear_faults()
394 gen6_check_faults(gt); in intel_gt_check_and_clear_faults()
398 intel_gt_clear_error_registers(gt, ALL_ENGINES); in intel_gt_check_and_clear_faults()
401 void intel_gt_flush_ggtt_writes(struct intel_gt *gt) in intel_gt_flush_ggtt_writes() argument
403 struct intel_uncore *uncore = gt->uncore; in intel_gt_flush_ggtt_writes()
427 if (INTEL_INFO(gt->i915)->has_coherent_ggtt) in intel_gt_flush_ggtt_writes()
430 intel_gt_chipset_flush(gt); in intel_gt_flush_ggtt_writes()
442 void intel_gt_chipset_flush(struct intel_gt *gt) in intel_gt_chipset_flush() argument
445 if (GRAPHICS_VER(gt->i915) < 6) in intel_gt_chipset_flush()
449 void intel_gt_driver_register(struct intel_gt *gt) in intel_gt_driver_register() argument
451 intel_gsc_init(&gt->gsc, gt->i915); in intel_gt_driver_register()
453 intel_rps_driver_register(&gt->rps); in intel_gt_driver_register()
455 intel_gt_debugfs_register(gt); in intel_gt_driver_register()
456 intel_gt_sysfs_register(gt); in intel_gt_driver_register()
459 static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size) in intel_gt_init_scratch() argument
461 struct drm_i915_private *i915 = gt->i915; in intel_gt_init_scratch()
474 gt_err(gt, "Failed to allocate scratch page\n"); in intel_gt_init_scratch()
478 vma = i915_vma_instance(obj, &gt->ggtt->vm, NULL); in intel_gt_init_scratch()
488 gt->scratch = i915_vma_make_unshrinkable(vma); in intel_gt_init_scratch()
497 static void intel_gt_fini_scratch(struct intel_gt *gt) in intel_gt_fini_scratch() argument
499 i915_vma_unpin_and_release(&gt->scratch, 0); in intel_gt_fini_scratch()
502 static struct i915_address_space *kernel_vm(struct intel_gt *gt) in kernel_vm() argument
504 if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING) in kernel_vm()
505 return &i915_ppgtt_create(gt, I915_BO_ALLOC_PM_EARLY)->vm; in kernel_vm()
507 return i915_vm_get(&gt->ggtt->vm); in kernel_vm()
510 static int __engines_record_defaults(struct intel_gt *gt) in __engines_record_defaults() argument
526 for_each_engine(engine, gt, id) { in __engines_record_defaults()
571 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) { in __engines_record_defaults()
609 intel_gt_set_wedged(gt); in __engines_record_defaults()
626 static int __engines_verify_workarounds(struct intel_gt *gt) in __engines_verify_workarounds() argument
635 for_each_engine(engine, gt, id) { in __engines_verify_workarounds()
641 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) in __engines_verify_workarounds()
647 static void __intel_gt_disable(struct intel_gt *gt) in __intel_gt_disable() argument
649 intel_gt_set_wedged_on_fini(gt); in __intel_gt_disable()
651 intel_gt_suspend_prepare(gt); in __intel_gt_disable()
652 intel_gt_suspend_late(gt); in __intel_gt_disable()
654 GEM_BUG_ON(intel_gt_pm_is_awake(gt)); in __intel_gt_disable()
657 int intel_gt_wait_for_idle(struct intel_gt *gt, long timeout) in intel_gt_wait_for_idle() argument
662 if (!intel_gt_pm_is_awake(gt)) in intel_gt_wait_for_idle()
665 while ((timeout = intel_gt_retire_requests_timeout(gt, timeout, in intel_gt_wait_for_idle()
678 return intel_uc_wait_for_idle(&gt->uc, remaining_timeout); in intel_gt_wait_for_idle()
681 int intel_gt_init(struct intel_gt *gt) in intel_gt_init() argument
685 err = i915_inject_probe_error(gt->i915, -ENODEV); in intel_gt_init()
689 intel_gt_init_workarounds(gt); in intel_gt_init()
698 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); in intel_gt_init()
700 err = intel_gt_init_scratch(gt, in intel_gt_init()
701 GRAPHICS_VER(gt->i915) == 2 ? SZ_256K : SZ_4K); in intel_gt_init()
705 intel_gt_pm_init(gt); in intel_gt_init()
707 gt->vm = kernel_vm(gt); in intel_gt_init()
708 if (!gt->vm) { in intel_gt_init()
713 intel_set_mocs_index(gt); in intel_gt_init()
715 err = intel_engines_init(gt); in intel_gt_init()
719 err = intel_uc_init(&gt->uc); in intel_gt_init()
723 err = intel_gt_resume(gt); in intel_gt_init()
727 err = intel_gt_init_hwconfig(gt); in intel_gt_init()
729 gt_err(gt, "Failed to retrieve hwconfig table: %pe\n", ERR_PTR(err)); in intel_gt_init()
731 err = __engines_record_defaults(gt); in intel_gt_init()
735 err = __engines_verify_workarounds(gt); in intel_gt_init()
739 err = i915_inject_probe_error(gt->i915, -EIO); in intel_gt_init()
743 intel_uc_init_late(&gt->uc); in intel_gt_init()
745 intel_migrate_init(&gt->migrate, gt); in intel_gt_init()
749 __intel_gt_disable(gt); in intel_gt_init()
750 intel_uc_fini_hw(&gt->uc); in intel_gt_init()
752 intel_uc_fini(&gt->uc); in intel_gt_init()
754 intel_engines_release(gt); in intel_gt_init()
755 i915_vm_put(fetch_and_zero(&gt->vm)); in intel_gt_init()
757 intel_gt_pm_fini(gt); in intel_gt_init()
758 intel_gt_fini_scratch(gt); in intel_gt_init()
761 intel_gt_set_wedged_on_init(gt); in intel_gt_init()
762 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); in intel_gt_init()
766 void intel_gt_driver_remove(struct intel_gt *gt) in intel_gt_driver_remove() argument
768 __intel_gt_disable(gt); in intel_gt_driver_remove()
770 intel_migrate_fini(&gt->migrate); in intel_gt_driver_remove()
771 intel_uc_driver_remove(&gt->uc); in intel_gt_driver_remove()
773 intel_engines_release(gt); in intel_gt_driver_remove()
775 intel_gt_flush_buffer_pool(gt); in intel_gt_driver_remove()
778 void intel_gt_driver_unregister(struct intel_gt *gt) in intel_gt_driver_unregister() argument
782 intel_gt_sysfs_unregister(gt); in intel_gt_driver_unregister()
783 intel_rps_driver_unregister(&gt->rps); in intel_gt_driver_unregister()
784 intel_gsc_fini(&gt->gsc); in intel_gt_driver_unregister()
797 * implications for subsystems outside of GT (i.e., PXP, HDCP), it's in intel_gt_driver_unregister()
807 intel_gsc_uc_flush_work(&gt->uc.gsc); in intel_gt_driver_unregister()
814 intel_gt_set_wedged_on_fini(gt); in intel_gt_driver_unregister()
817 with_intel_runtime_pm(gt->uncore->rpm, wakeref) in intel_gt_driver_unregister()
818 __intel_gt_reset(gt, ALL_ENGINES); in intel_gt_driver_unregister()
821 void intel_gt_driver_release(struct intel_gt *gt) in intel_gt_driver_release() argument
825 vm = fetch_and_zero(&gt->vm); in intel_gt_driver_release()
829 intel_wa_list_free(&gt->wa_list); in intel_gt_driver_release()
830 intel_gt_pm_fini(gt); in intel_gt_driver_release()
831 intel_gt_fini_scratch(gt); in intel_gt_driver_release()
832 intel_gt_fini_buffer_pool(gt); in intel_gt_driver_release()
833 intel_gt_fini_hwconfig(gt); in intel_gt_driver_release()
838 struct intel_gt *gt; in intel_gt_driver_late_release_all() local
844 for_each_gt(gt, i915, id) { in intel_gt_driver_late_release_all()
845 intel_uc_driver_late_release(&gt->uc); in intel_gt_driver_late_release_all()
846 intel_gt_fini_requests(gt); in intel_gt_driver_late_release_all()
847 intel_gt_fini_reset(gt); in intel_gt_driver_late_release_all()
848 intel_gt_fini_timelines(gt); in intel_gt_driver_late_release_all()
849 intel_gt_fini_tlb(gt); in intel_gt_driver_late_release_all()
850 intel_engines_free(gt); in intel_gt_driver_late_release_all()
854 static int intel_gt_tile_setup(struct intel_gt *gt, phys_addr_t phys_addr) in intel_gt_tile_setup() argument
858 if (!gt_is_root(gt)) { in intel_gt_tile_setup()
862 uncore = drmm_kzalloc(&gt->i915->drm, sizeof(*uncore), GFP_KERNEL); in intel_gt_tile_setup()
866 irq_lock = drmm_kzalloc(&gt->i915->drm, sizeof(*irq_lock), GFP_KERNEL); in intel_gt_tile_setup()
870 gt->uncore = uncore; in intel_gt_tile_setup()
871 gt->irq_lock = irq_lock; in intel_gt_tile_setup()
873 intel_gt_common_init_early(gt); in intel_gt_tile_setup()
876 intel_uncore_init_early(gt->uncore, gt); in intel_gt_tile_setup()
878 ret = intel_uncore_setup_mmio(gt->uncore, phys_addr); in intel_gt_tile_setup()
882 gt->phys_addr = phys_addr; in intel_gt_tile_setup()
890 struct intel_gt *gt = to_gt(i915); in intel_gt_probe_all() local
901 * We always have at least one primary GT on any device in intel_gt_probe_all()
905 gt->i915 = i915; in intel_gt_probe_all()
906 gt->name = "Primary GT"; in intel_gt_probe_all()
907 gt->info.engine_mask = INTEL_INFO(i915)->platform_engine_mask; in intel_gt_probe_all()
909 gt_dbg(gt, "Setting up %s\n", gt->name); in intel_gt_probe_all()
910 ret = intel_gt_tile_setup(gt, phys_addr); in intel_gt_probe_all()
914 i915->gt[0] = gt; in intel_gt_probe_all()
922 gt = drmm_kzalloc(&i915->drm, sizeof(*gt), GFP_KERNEL); in intel_gt_probe_all()
923 if (!gt) { in intel_gt_probe_all()
928 gt->i915 = i915; in intel_gt_probe_all()
929 gt->name = gtdef->name; in intel_gt_probe_all()
930 gt->type = gtdef->type; in intel_gt_probe_all()
931 gt->info.engine_mask = gtdef->engine_mask; in intel_gt_probe_all()
932 gt->info.id = i; in intel_gt_probe_all()
934 gt_dbg(gt, "Setting up %s\n", gt->name); in intel_gt_probe_all()
945 ret = intel_gt_tile_setup(gt, phys_addr + gtdef->mapping_base); in intel_gt_probe_all()
949 ret = intel_sa_mediagt_setup(gt, phys_addr + gtdef->mapping_base, in intel_gt_probe_all()
954 /* Primary GT should not appear in extra GT list */ in intel_gt_probe_all()
963 i915->gt[i] = gt; in intel_gt_probe_all()
975 struct intel_gt *gt; in intel_gt_tiles_init() local
979 for_each_gt(gt, i915, id) { in intel_gt_tiles_init()
980 ret = intel_gt_probe_lmem(gt); in intel_gt_tiles_init()
996 enum i915_map_type intel_gt_coherent_map_type(struct intel_gt *gt, in intel_gt_coherent_map_type() argument
1002 * version 13.0 when the object is on the Media GT in intel_gt_coherent_map_type()
1004 if (i915_gem_object_is_lmem(obj) || intel_gt_needs_wa_22016122933(gt)) in intel_gt_coherent_map_type()
1006 if (HAS_LLC(gt->i915) || always_coherent) in intel_gt_coherent_map_type()