161 	int (*invalidate_tlb)(struct apple_dart_stream_map *stream_map);
292 #define for_each_stream_map(i, base, stream_map)                               \  argument
293 	for (i = 0, stream_map = &(base)->stream_maps[0];                      \
294 	     i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
295 	     stream_map = &(base)->stream_maps[++i])
306 apple_dart_hw_enable_translation(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_enable_translation()  argument
308 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_enable_translation()
311 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_enable_translation()
315 static void apple_dart_hw_disable_dma(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_disable_dma()  argument
317 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_disable_dma()
320 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_disable_dma()
325 apple_dart_hw_enable_bypass(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_enable_bypass()  argument
327 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_enable_bypass()
330 	WARN_ON(!stream_map->dart->supports_bypass);  in apple_dart_hw_enable_bypass()
331 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_enable_bypass()
336 static void apple_dart_hw_set_ttbr(struct apple_dart_stream_map *stream_map,  in apple_dart_hw_set_ttbr()  argument
339 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_set_ttbr()
343 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_set_ttbr()
349 static void apple_dart_hw_clear_ttbr(struct apple_dart_stream_map *stream_map,  in apple_dart_hw_clear_ttbr()  argument
352 	struct apple_dart *dart = stream_map->dart;  in apple_dart_hw_clear_ttbr()
355 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams)  in apple_dart_hw_clear_ttbr()
360 apple_dart_hw_clear_all_ttbrs(struct apple_dart_stream_map *stream_map)  in apple_dart_hw_clear_all_ttbrs()  argument
364 	for (i = 0; i < stream_map->dart->hw->ttbr_count; ++i)  in apple_dart_hw_clear_all_ttbrs()
365 		apple_dart_hw_clear_ttbr(stream_map, i);  in apple_dart_hw_clear_all_ttbrs()
369 apple_dart_t8020_hw_stream_command(struct apple_dart_stream_map *stream_map,  in apple_dart_t8020_hw_stream_command()  argument
376 	spin_lock_irqsave(&stream_map->dart->lock, flags);  in apple_dart_t8020_hw_stream_command()
378 	writel(stream_map->sidmap[0], stream_map->dart->regs + DART_T8020_STREAM_SELECT);  in apple_dart_t8020_hw_stream_command()
379 	writel(command, stream_map->dart->regs + DART_T8020_STREAM_COMMAND);  in apple_dart_t8020_hw_stream_command()
382 		stream_map->dart->regs + DART_T8020_STREAM_COMMAND, command_reg,  in apple_dart_t8020_hw_stream_command()
386 	spin_unlock_irqrestore(&stream_map->dart->lock, flags);  in apple_dart_t8020_hw_stream_command()
389 		dev_err(stream_map->dart->dev,  in apple_dart_t8020_hw_stream_command()
391 			command, stream_map->sidmap[0]);  in apple_dart_t8020_hw_stream_command()
399 apple_dart_t8110_hw_tlb_command(struct apple_dart_stream_map *stream_map,  in apple_dart_t8110_hw_tlb_command()  argument
402 	struct apple_dart *dart = stream_map->dart;  in apple_dart_t8110_hw_tlb_command()
409 	for_each_set_bit(sid, stream_map->sidmap, dart->num_streams) {  in apple_dart_t8110_hw_tlb_command()
427 		dev_err(stream_map->dart->dev,  in apple_dart_t8110_hw_tlb_command()
437 apple_dart_t8020_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)  in apple_dart_t8020_hw_invalidate_tlb()  argument
440 		stream_map, DART_T8020_STREAM_COMMAND_INVALIDATE);  in apple_dart_t8020_hw_invalidate_tlb()
444 apple_dart_t8110_hw_invalidate_tlb(struct apple_dart_stream_map *stream_map)  in apple_dart_t8110_hw_invalidate_tlb()  argument
447 		stream_map, DART_T8110_TLB_CMD_OP_FLUSH_SID);  in apple_dart_t8110_hw_invalidate_tlb()
453 	struct apple_dart_stream_map stream_map;  in apple_dart_hw_reset()  local
463 	stream_map.dart = dart;  in apple_dart_hw_reset()
464 	bitmap_zero(stream_map.sidmap, DART_MAX_STREAMS);  in apple_dart_hw_reset()
465 	bitmap_set(stream_map.sidmap, 0, dart->num_streams);  in apple_dart_hw_reset()
466 	apple_dart_hw_disable_dma(&stream_map);  in apple_dart_hw_reset()
467 	apple_dart_hw_clear_all_ttbrs(&stream_map);  in apple_dart_hw_reset()
479 	return dart->hw->invalidate_tlb(&stream_map);  in apple_dart_hw_reset()
486 	struct apple_dart_stream_map stream_map;  in apple_dart_domain_flush_tlb()  local
489 		stream_map.dart = domain_stream_map->dart;  in apple_dart_domain_flush_tlb()
491 		for (j = 0; j < BITS_TO_LONGS(stream_map.dart->num_streams); j++)  in apple_dart_domain_flush_tlb()
492 			stream_map.sidmap[j] = atomic_long_read(&domain_stream_map->sidmap[j]);  in apple_dart_domain_flush_tlb()
494 		stream_map.dart->hw->invalidate_tlb(&stream_map);  in apple_dart_domain_flush_tlb()
555 			     struct apple_dart_stream_map *stream_map)  in apple_dart_setup_translation()  argument
562 		apple_dart_hw_set_ttbr(stream_map, i,  in apple_dart_setup_translation()
564 	for (; i < stream_map->dart->hw->ttbr_count; ++i)  in apple_dart_setup_translation()
565 		apple_dart_hw_clear_ttbr(stream_map, i);  in apple_dart_setup_translation()
567 	apple_dart_hw_enable_translation(stream_map);  in apple_dart_setup_translation()
568 	stream_map->dart->hw->invalidate_tlb(stream_map);  in apple_dart_setup_translation()
658 	struct apple_dart_stream_map *stream_map;  in apple_dart_attach_dev()  local
679 		for_each_stream_map(i, cfg, stream_map)  in apple_dart_attach_dev()
680 			apple_dart_setup_translation(dart_domain, stream_map);  in apple_dart_attach_dev()
683 		for_each_stream_map(i, cfg, stream_map)  in apple_dart_attach_dev()
684 			apple_dart_hw_disable_dma(stream_map);  in apple_dart_attach_dev()
687 		for_each_stream_map(i, cfg, stream_map)  in apple_dart_attach_dev()
688 			apple_dart_hw_enable_bypass(stream_map);  in apple_dart_attach_dev()
698 	struct apple_dart_stream_map *stream_map;  in apple_dart_probe_device()  local
704 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_probe_device()
706 			dev, stream_map->dart->dev,  in apple_dart_probe_device()
801 	struct apple_dart_stream_map *stream_map;  in apple_dart_release_group()  local
806 	for_each_stream_map(i, group_master_cfg, stream_map)  in apple_dart_release_group()
807 		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)  in apple_dart_release_group()
808 			stream_map->dart->sid2group[sid] = NULL;  in apple_dart_release_group()
841 	struct apple_dart_stream_map *stream_map;  in apple_dart_device_group()  local
848 	for_each_stream_map(i, cfg, stream_map) {  in apple_dart_device_group()
849 		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams) {  in apple_dart_device_group()
851 				stream_map->dart->sid2group[sid];  in apple_dart_device_group()
901 	for_each_stream_map(i, cfg, stream_map)  in apple_dart_device_group()
902 		for_each_set_bit(sid, stream_map->sidmap, stream_map->dart->num_streams)  in apple_dart_device_group()
903 			stream_map->dart->sid2group[sid] = group;  in apple_dart_device_group()