617 						 struct its_cmd_desc *desc)  in its_build_mapd_cmd()  argument
620 	u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);  in its_build_mapd_cmd()
622 	itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);  in its_build_mapd_cmd()
626 	its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);  in its_build_mapd_cmd()
629 	its_encode_valid(cmd, desc->its_mapd_cmd.valid);  in its_build_mapd_cmd()
638 						 struct its_cmd_desc *desc)  in its_build_mapc_cmd()  argument
641 	its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);  in its_build_mapc_cmd()
642 	its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);  in its_build_mapc_cmd()
643 	its_encode_valid(cmd, desc->its_mapc_cmd.valid);  in its_build_mapc_cmd()
647 	return desc->its_mapc_cmd.col;  in its_build_mapc_cmd()
652 						  struct its_cmd_desc *desc)  in its_build_mapti_cmd()  argument
656 	col = dev_event_to_col(desc->its_mapti_cmd.dev,  in its_build_mapti_cmd()
657 			       desc->its_mapti_cmd.event_id);  in its_build_mapti_cmd()
660 	its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);  in its_build_mapti_cmd()
661 	its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);  in its_build_mapti_cmd()
662 	its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);  in its_build_mapti_cmd()
672 						 struct its_cmd_desc *desc)  in its_build_movi_cmd()  argument
676 	col = dev_event_to_col(desc->its_movi_cmd.dev,  in its_build_movi_cmd()
677 			       desc->its_movi_cmd.event_id);  in its_build_movi_cmd()
680 	its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);  in its_build_movi_cmd()
681 	its_encode_event_id(cmd, desc->its_movi_cmd.event_id);  in its_build_movi_cmd()
682 	its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);  in its_build_movi_cmd()
691 						    struct its_cmd_desc *desc)  in its_build_discard_cmd()  argument
695 	col = dev_event_to_col(desc->its_discard_cmd.dev,  in its_build_discard_cmd()
696 			       desc->its_discard_cmd.event_id);  in its_build_discard_cmd()
699 	its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);  in its_build_discard_cmd()
700 	its_encode_event_id(cmd, desc->its_discard_cmd.event_id);  in its_build_discard_cmd()
709 						struct its_cmd_desc *desc)  in its_build_inv_cmd()  argument
713 	col = dev_event_to_col(desc->its_inv_cmd.dev,  in its_build_inv_cmd()
714 			       desc->its_inv_cmd.event_id);  in its_build_inv_cmd()
717 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);  in its_build_inv_cmd()
718 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);  in its_build_inv_cmd()
727 						struct its_cmd_desc *desc)  in its_build_int_cmd()  argument
731 	col = dev_event_to_col(desc->its_int_cmd.dev,  in its_build_int_cmd()
732 			       desc->its_int_cmd.event_id);  in its_build_int_cmd()
735 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);  in its_build_int_cmd()
736 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);  in its_build_int_cmd()
745 						  struct its_cmd_desc *desc)  in its_build_clear_cmd()  argument
749 	col = dev_event_to_col(desc->its_clear_cmd.dev,  in its_build_clear_cmd()
750 			       desc->its_clear_cmd.event_id);  in its_build_clear_cmd()
753 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);  in its_build_clear_cmd()
754 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);  in its_build_clear_cmd()
763 						   struct its_cmd_desc *desc)  in its_build_invall_cmd()  argument
766 	its_encode_collection(cmd, desc->its_invall_cmd.col->col_id);  in its_build_invall_cmd()
770 	return desc->its_invall_cmd.col;  in its_build_invall_cmd()
775 					     struct its_cmd_desc *desc)  in its_build_vinvall_cmd()  argument
778 	its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);  in its_build_vinvall_cmd()
782 	return valid_vpe(its, desc->its_vinvall_cmd.vpe);  in its_build_vinvall_cmd()
787 					   struct its_cmd_desc *desc)  in its_build_vmapp_cmd()  argument
789 	struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe);  in its_build_vmapp_cmd()
795 	its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);  in its_build_vmapp_cmd()
796 	its_encode_valid(cmd, desc->its_vmapp_cmd.valid);  in its_build_vmapp_cmd()
798 	if (!desc->its_vmapp_cmd.valid) {  in its_build_vmapp_cmd()
799 		alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count);  in its_build_vmapp_cmd()
812 	vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));  in its_build_vmapp_cmd()
813 	target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;  in its_build_vmapp_cmd()
819 	alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count);  in its_build_vmapp_cmd()
824 	vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page));  in its_build_vmapp_cmd()
836 	its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi);  in its_build_vmapp_cmd()
846 					    struct its_cmd_desc *desc)  in its_build_vmapti_cmd()  argument
850 	if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled)  in its_build_vmapti_cmd()
851 		db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;  in its_build_vmapti_cmd()
856 	its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);  in its_build_vmapti_cmd()
857 	its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);  in its_build_vmapti_cmd()
858 	its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);  in its_build_vmapti_cmd()
860 	its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);  in its_build_vmapti_cmd()
864 	return valid_vpe(its, desc->its_vmapti_cmd.vpe);  in its_build_vmapti_cmd()
869 					   struct its_cmd_desc *desc)  in its_build_vmovi_cmd()  argument
873 	if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled)  in its_build_vmovi_cmd()
874 		db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;  in its_build_vmovi_cmd()
879 	its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);  in its_build_vmovi_cmd()
880 	its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);  in its_build_vmovi_cmd()
881 	its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);  in its_build_vmovi_cmd()
887 	return valid_vpe(its, desc->its_vmovi_cmd.vpe);  in its_build_vmovi_cmd()
892 					   struct its_cmd_desc *desc)  in its_build_vmovp_cmd()  argument
896 	target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;  in its_build_vmovp_cmd()
898 	its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);  in its_build_vmovp_cmd()
899 	its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);  in its_build_vmovp_cmd()
900 	its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);  in its_build_vmovp_cmd()
905 		its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi);  in its_build_vmovp_cmd()
910 	return valid_vpe(its, desc->its_vmovp_cmd.vpe);  in its_build_vmovp_cmd()
915 					  struct its_cmd_desc *desc)  in its_build_vinv_cmd()  argument
919 	map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev,  in its_build_vinv_cmd()
920 				    desc->its_inv_cmd.event_id);  in its_build_vinv_cmd()
923 	its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);  in its_build_vinv_cmd()
924 	its_encode_event_id(cmd, desc->its_inv_cmd.event_id);  in its_build_vinv_cmd()
933 					  struct its_cmd_desc *desc)  in its_build_vint_cmd()  argument
937 	map = dev_event_to_vlpi_map(desc->its_int_cmd.dev,  in its_build_vint_cmd()
938 				    desc->its_int_cmd.event_id);  in its_build_vint_cmd()
941 	its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);  in its_build_vint_cmd()
942 	its_encode_event_id(cmd, desc->its_int_cmd.event_id);  in its_build_vint_cmd()
951 					    struct its_cmd_desc *desc)  in its_build_vclear_cmd()  argument
955 	map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev,  in its_build_vclear_cmd()
956 				    desc->its_clear_cmd.event_id);  in its_build_vclear_cmd()
959 	its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);  in its_build_vclear_cmd()
960 	its_encode_event_id(cmd, desc->its_clear_cmd.event_id);  in its_build_vclear_cmd()
969 					   struct its_cmd_desc *desc)  in its_build_invdb_cmd()  argument
975 	its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id);  in its_build_invdb_cmd()
979 	return valid_vpe(its, desc->its_invdb_cmd.vpe);  in its_build_invdb_cmd()
984 					  struct its_cmd_desc *desc)  in its_build_vsgi_cmd()  argument
990 	its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id);  in its_build_vsgi_cmd()
991 	its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi);  in its_build_vsgi_cmd()
992 	its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority);  in its_build_vsgi_cmd()
993 	its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group);  in its_build_vsgi_cmd()
994 	its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear);  in its_build_vsgi_cmd()
995 	its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable);  in its_build_vsgi_cmd()
999 	return valid_vpe(its, desc->its_vsgi_cmd.vpe);  in its_build_vsgi_cmd()
1123 	  struct its_cmd_desc *desc)					\
1137 	sync_obj = builder(its, cmd, desc);				\
1186 	struct its_cmd_desc desc;  in BUILD_SINGLE_CMD_FUNC()  local
1188 	desc.its_int_cmd.dev = dev;  in BUILD_SINGLE_CMD_FUNC()
1189 	desc.its_int_cmd.event_id = event_id;  in BUILD_SINGLE_CMD_FUNC()
1191 	its_send_single_command(dev->its, its_build_int_cmd, &desc);  in BUILD_SINGLE_CMD_FUNC()
1196 	struct its_cmd_desc desc;  in its_send_clear()  local
1198 	desc.its_clear_cmd.dev = dev;  in its_send_clear()
1199 	desc.its_clear_cmd.event_id = event_id;  in its_send_clear()
1201 	its_send_single_command(dev->its, its_build_clear_cmd, &desc);  in its_send_clear()
1206 	struct its_cmd_desc desc;  in its_send_inv()  local
1208 	desc.its_inv_cmd.dev = dev;  in its_send_inv()
1209 	desc.its_inv_cmd.event_id = event_id;  in its_send_inv()
1211 	its_send_single_command(dev->its, its_build_inv_cmd, &desc);  in its_send_inv()
1216 	struct its_cmd_desc desc;  in its_send_mapd()  local
1218 	desc.its_mapd_cmd.dev = dev;  in its_send_mapd()
1219 	desc.its_mapd_cmd.valid = !!valid;  in its_send_mapd()
1221 	its_send_single_command(dev->its, its_build_mapd_cmd, &desc);  in its_send_mapd()
1227 	struct its_cmd_desc desc;  in its_send_mapc()  local
1229 	desc.its_mapc_cmd.col = col;  in its_send_mapc()
1230 	desc.its_mapc_cmd.valid = !!valid;  in its_send_mapc()
1232 	its_send_single_command(its, its_build_mapc_cmd, &desc);  in its_send_mapc()
1237 	struct its_cmd_desc desc;  in its_send_mapti()  local
1239 	desc.its_mapti_cmd.dev = dev;  in its_send_mapti()
1240 	desc.its_mapti_cmd.phys_id = irq_id;  in its_send_mapti()
1241 	desc.its_mapti_cmd.event_id = id;  in its_send_mapti()
1243 	its_send_single_command(dev->its, its_build_mapti_cmd, &desc);  in its_send_mapti()
1249 	struct its_cmd_desc desc;  in its_send_movi()  local
1251 	desc.its_movi_cmd.dev = dev;  in its_send_movi()
1252 	desc.its_movi_cmd.col = col;  in its_send_movi()
1253 	desc.its_movi_cmd.event_id = id;  in its_send_movi()
1255 	its_send_single_command(dev->its, its_build_movi_cmd, &desc);  in its_send_movi()
1260 	struct its_cmd_desc desc;  in its_send_discard()  local
1262 	desc.its_discard_cmd.dev = dev;  in its_send_discard()
1263 	desc.its_discard_cmd.event_id = id;  in its_send_discard()
1265 	its_send_single_command(dev->its, its_build_discard_cmd, &desc);  in its_send_discard()
1270 	struct its_cmd_desc desc;  in its_send_invall()  local
1272 	desc.its_invall_cmd.col = col;  in its_send_invall()
1274 	its_send_single_command(its, its_build_invall_cmd, &desc);  in its_send_invall()
1280 	struct its_cmd_desc desc;  in its_send_vmapti()  local
1282 	desc.its_vmapti_cmd.vpe = map->vpe;  in its_send_vmapti()
1283 	desc.its_vmapti_cmd.dev = dev;  in its_send_vmapti()
1284 	desc.its_vmapti_cmd.virt_id = map->vintid;  in its_send_vmapti()
1285 	desc.its_vmapti_cmd.event_id = id;  in its_send_vmapti()
1286 	desc.its_vmapti_cmd.db_enabled = map->db_enabled;  in its_send_vmapti()
1288 	its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);  in its_send_vmapti()
1294 	struct its_cmd_desc desc;  in its_send_vmovi()  local
1296 	desc.its_vmovi_cmd.vpe = map->vpe;  in its_send_vmovi()
1297 	desc.its_vmovi_cmd.dev = dev;  in its_send_vmovi()
1298 	desc.its_vmovi_cmd.event_id = id;  in its_send_vmovi()
1299 	desc.its_vmovi_cmd.db_enabled = map->db_enabled;  in its_send_vmovi()
1301 	its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);  in its_send_vmovi()
1307 	struct its_cmd_desc desc;  in its_send_vmapp()  local
1309 	desc.its_vmapp_cmd.vpe = vpe;  in its_send_vmapp()
1310 	desc.its_vmapp_cmd.valid = valid;  in its_send_vmapp()
1311 	desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];  in its_send_vmapp()
1313 	its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);  in its_send_vmapp()
1318 	struct its_cmd_desc desc = {};  in its_send_vmovp()  local
1323 	desc.its_vmovp_cmd.vpe = vpe;  in its_send_vmovp()
1327 		desc.its_vmovp_cmd.col = &its->collections[col_id];  in its_send_vmovp()
1328 		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);  in its_send_vmovp()
1342 	desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;  in its_send_vmovp()
1343 	desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm);  in its_send_vmovp()
1353 		desc.its_vmovp_cmd.col = &its->collections[col_id];  in its_send_vmovp()
1354 		its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);  in its_send_vmovp()
1362 	struct its_cmd_desc desc;  in its_send_vinvall()  local
1364 	desc.its_vinvall_cmd.vpe = vpe;  in its_send_vinvall()
1365 	its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);  in its_send_vinvall()
1370 	struct its_cmd_desc desc;  in its_send_vinv()  local
1376 	desc.its_inv_cmd.dev = dev;  in its_send_vinv()
1377 	desc.its_inv_cmd.event_id = event_id;  in its_send_vinv()
1379 	its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc);  in its_send_vinv()
1384 	struct its_cmd_desc desc;  in its_send_vint()  local
1390 	desc.its_int_cmd.dev = dev;  in its_send_vint()
1391 	desc.its_int_cmd.event_id = event_id;  in its_send_vint()
1393 	its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc);  in its_send_vint()
1398 	struct its_cmd_desc desc;  in its_send_vclear()  local
1404 	desc.its_clear_cmd.dev = dev;  in its_send_vclear()
1405 	desc.its_clear_cmd.event_id = event_id;  in its_send_vclear()
1407 	its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc);  in its_send_vclear()
1412 	struct its_cmd_desc desc;  in its_send_invdb()  local
1414 	desc.its_invdb_cmd.vpe = vpe;  in its_send_invdb()
1415 	its_send_single_vcommand(its, its_build_invdb_cmd, &desc);  in its_send_invdb()
3594 	err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev));  in its_irq_domain_alloc()
4217 	struct its_cmd_desc desc;  in its_configure_sgi()  local
4219 	desc.its_vsgi_cmd.vpe = vpe;  in its_configure_sgi()
4220 	desc.its_vsgi_cmd.sgi = d->hwirq;  in its_configure_sgi()
4221 	desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority;  in its_configure_sgi()
4222 	desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled;  in its_configure_sgi()
4223 	desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group;  in its_configure_sgi()
4224 	desc.its_vsgi_cmd.clear = clear;  in its_configure_sgi()
4231 	its_send_single_vcommand(find_4_1_its(), its_build_vsgi_cmd, &desc);  in its_configure_sgi()
4770 		.desc	= "ITS: Cavium errata 22375, 24313",
4778 		.desc	= "ITS: Cavium erratum 23144",
4786 		.desc	= "ITS: QDF2400 erratum 0065",
4799 		.desc	= "ITS: Socionext Synquacer pre-ITS",
4807 		.desc	= "ITS: Hip07 erratum 161600802",
4815 		.desc   = "ITS: Rockchip erratum RK3588001",
4822 		.desc   = "ITS: non-coherent attribute",