149 void ni_tio_write(struct ni_gpct *counter, unsigned int value,  in ni_tio_write()  argument
153 		counter->counter_dev->write(counter, value, reg);  in ni_tio_write()
162 unsigned int ni_tio_read(struct ni_gpct *counter, enum ni_gpct_register reg)  in ni_tio_read()  argument
165 		return counter->counter_dev->read(counter, reg);  in ni_tio_read()
170 static void ni_tio_reset_count_and_disarm(struct ni_gpct *counter)  in ni_tio_reset_count_and_disarm()  argument
172 	unsigned int cidx = counter->counter_index;  in ni_tio_reset_count_and_disarm()
174 	ni_tio_write(counter, GI_RESET(cidx), NITIO_RESET_REG(cidx));  in ni_tio_reset_count_and_disarm()
177 static int ni_tio_clock_period_ps(const struct ni_gpct *counter,  in ni_tio_clock_period_ps()  argument
201 		*period_ps = counter->clock_period_ps;  in ni_tio_clock_period_ps()
221 static void ni_tio_set_bits_transient(struct ni_gpct *counter,  in ni_tio_set_bits_transient()  argument
226 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_bits_transient()
227 	unsigned int chip = counter->chip_index;  in ni_tio_set_bits_transient()
236 		ni_tio_write(counter, regs[reg] | transient, reg);  in ni_tio_set_bits_transient()
252 void ni_tio_set_bits(struct ni_gpct *counter, enum ni_gpct_register reg,  in ni_tio_set_bits()  argument
255 	ni_tio_set_bits_transient(counter, reg, mask, value, 0x0);  in ni_tio_set_bits()
268 unsigned int ni_tio_get_soft_copy(const struct ni_gpct *counter,  in ni_tio_get_soft_copy()  argument
271 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_get_soft_copy()
272 	unsigned int chip = counter->chip_index;  in ni_tio_get_soft_copy()
285 static unsigned int ni_tio_clock_src_modifiers(const struct ni_gpct *counter)  in ni_tio_clock_src_modifiers()  argument
287 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_clock_src_modifiers()
288 	unsigned int cidx = counter->counter_index;  in ni_tio_clock_src_modifiers()
290 		ni_tio_get_soft_copy(counter, NITIO_CNT_MODE_REG(cidx));  in ni_tio_clock_src_modifiers()
293 	if (ni_tio_get_soft_copy(counter, NITIO_INPUT_SEL_REG(cidx)) &  in ni_tio_clock_src_modifiers()
303 static int ni_m_series_clock_src_select(const struct ni_gpct *counter,  in ni_m_series_clock_src_select()  argument
306 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_m_series_clock_src_select()
307 	unsigned int cidx = counter->counter_index;  in ni_m_series_clock_src_select()
308 	unsigned int chip = counter->chip_index;  in ni_m_series_clock_src_select()
314 	src = GI_BITS_TO_SRC(ni_tio_get_soft_copy(counter,  in ni_m_series_clock_src_select()
365 	clock_source |= ni_tio_clock_src_modifiers(counter);  in ni_m_series_clock_src_select()
370 static int ni_660x_clock_src_select(const struct ni_gpct *counter,  in ni_660x_clock_src_select()  argument
374 	unsigned int cidx = counter->counter_index;  in ni_660x_clock_src_select()
378 	src = GI_BITS_TO_SRC(ni_tio_get_soft_copy(counter,  in ni_660x_clock_src_select()
423 	clock_source |= ni_tio_clock_src_modifiers(counter);  in ni_660x_clock_src_select()
428 static int ni_tio_generic_clock_src_select(const struct ni_gpct *counter,  in ni_tio_generic_clock_src_select()  argument
431 	switch (counter->counter_dev->variant) {  in ni_tio_generic_clock_src_select()
435 		return ni_m_series_clock_src_select(counter, clk_src);  in ni_tio_generic_clock_src_select()
437 		return ni_660x_clock_src_select(counter, clk_src);  in ni_tio_generic_clock_src_select()
441 static void ni_tio_set_sync_mode(struct ni_gpct *counter)  in ni_tio_set_sync_mode()  argument
443 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_sync_mode()
444 	unsigned int cidx = counter->counter_index;  in ni_tio_set_sync_mode()
469 	mode = ni_tio_get_soft_copy(counter, reg);  in ni_tio_set_sync_mode()
482 	ret = ni_tio_generic_clock_src_select(counter, &clk_src);  in ni_tio_set_sync_mode()
485 	ret = ni_tio_clock_period_ps(counter, clk_src, &ps);  in ni_tio_set_sync_mode()
495 	ni_tio_set_bits(counter, reg, mask, bits);  in ni_tio_set_sync_mode()
498 static int ni_tio_set_counter_mode(struct ni_gpct *counter, unsigned int mode)  in ni_tio_set_counter_mode()  argument
500 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_counter_mode()
501 	unsigned int cidx = counter->counter_index;  in ni_tio_set_counter_mode()
528 	ni_tio_set_bits(counter, NITIO_MODE_REG(cidx),  in ni_tio_set_counter_mode()
538 		ni_tio_set_bits(counter, NITIO_CNT_MODE_REG(cidx),  in ni_tio_set_counter_mode()
541 		ni_tio_set_sync_mode(counter);  in ni_tio_set_counter_mode()
544 	ni_tio_set_bits(counter, NITIO_CMD_REG(cidx), GI_CNT_DIR_MASK,  in ni_tio_set_counter_mode()
551 	ni_tio_set_bits(counter, NITIO_INPUT_SEL_REG(cidx),  in ni_tio_set_counter_mode()
558 int ni_tio_arm(struct ni_gpct *counter, bool arm, unsigned int start_trigger)  in ni_tio_arm()  argument
560 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_arm()
561 	unsigned int cidx = counter->counter_index;  in ni_tio_arm()
604 			ni_tio_set_bits(counter, NITIO_CNT_MODE_REG(cidx),  in ni_tio_arm()
609 	ni_tio_set_bits_transient(counter, NITIO_CMD_REG(cidx),  in ni_tio_arm()
723 static void ni_tio_set_source_subselect(struct ni_gpct *counter,  in ni_tio_set_source_subselect()  argument
726 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_source_subselect()
727 	unsigned int cidx = counter->counter_index;  in ni_tio_set_source_subselect()
728 	unsigned int chip = counter->chip_index;  in ni_tio_set_source_subselect()
748 	ni_tio_write(counter, counter_dev->regs[chip][second_gate_reg],  in ni_tio_set_source_subselect()
752 static int ni_tio_set_clock_src(struct ni_gpct *counter,  in ni_tio_set_clock_src()  argument
756 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_clock_src()
757 	unsigned int cidx = counter->counter_index;  in ni_tio_set_clock_src()
781 	ni_tio_set_bits(counter, NITIO_INPUT_SEL_REG(cidx),  in ni_tio_set_clock_src()
783 	ni_tio_set_source_subselect(counter, clock_source);  in ni_tio_set_clock_src()
799 		ni_tio_set_bits(counter, NITIO_CNT_MODE_REG(cidx),  in ni_tio_set_clock_src()
803 	counter->clock_period_ps = period_ns * 1000;  in ni_tio_set_clock_src()
804 	ni_tio_set_sync_mode(counter);  in ni_tio_set_clock_src()
808 static int ni_tio_get_clock_src(struct ni_gpct *counter,  in ni_tio_get_clock_src()  argument
815 	ret = ni_tio_generic_clock_src_select(counter, clock_source);  in ni_tio_get_clock_src()
818 	ret = ni_tio_clock_period_ps(counter, *clock_source, &temp64);  in ni_tio_get_clock_src()
826 static inline void ni_tio_set_gate_raw(struct ni_gpct *counter,  in ni_tio_set_gate_raw()  argument
829 	ni_tio_set_bits(counter, NITIO_INPUT_SEL_REG(counter->counter_index),  in ni_tio_set_gate_raw()
833 static inline void ni_tio_set_gate2_raw(struct ni_gpct *counter,  in ni_tio_set_gate2_raw()  argument
836 	ni_tio_set_bits(counter, NITIO_GATE2_REG(counter->counter_index),  in ni_tio_set_gate2_raw()
841 static inline void ni_tio_set_gate_mode(struct ni_gpct *counter,  in ni_tio_set_gate_mode()  argument
860 	ni_tio_set_bits(counter, NITIO_MODE_REG(counter->counter_index),  in ni_tio_set_gate_mode()
872 static inline void ni_tio_set_gate2_mode(struct ni_gpct *counter,  in ni_tio_set_gate2_mode()  argument
892 	ni_tio_set_bits(counter, NITIO_GATE2_REG(counter->counter_index),  in ni_tio_set_gate2_mode()
896 static int ni_660x_set_gate(struct ni_gpct *counter, unsigned int gate_source)  in ni_660x_set_gate()  argument
931 	ni_tio_set_gate_raw(counter, gate_sel);  in ni_660x_set_gate()
935 static int ni_m_set_gate(struct ni_gpct *counter, unsigned int gate_source)  in ni_m_set_gate()  argument
971 	ni_tio_set_gate_raw(counter, gate_sel);  in ni_m_set_gate()
975 static int ni_660x_set_gate2(struct ni_gpct *counter, unsigned int gate_source)  in ni_660x_set_gate2()  argument
1011 	ni_tio_set_gate2_raw(counter, gate2_sel);  in ni_660x_set_gate2()
1015 static int ni_m_set_gate2(struct ni_gpct *counter, unsigned int gate_source)  in ni_m_set_gate2()  argument
1021 	ni_tio_set_gate2_raw(counter, gate_source);  in ni_m_set_gate2()
1025 int ni_tio_set_gate_src_raw(struct ni_gpct *counter,  in ni_tio_set_gate_src_raw()  argument
1028 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_gate_src_raw()
1033 		ni_tio_set_gate_mode(counter, NI_GPCT_DISABLED_GATE_SELECT);  in ni_tio_set_gate_src_raw()
1035 		ni_tio_set_gate_raw(counter, src);  in ni_tio_set_gate_src_raw()
1037 		ni_tio_set_gate_mode(counter, src);  in ni_tio_set_gate_src_raw()
1044 		ni_tio_set_gate2_mode(counter, NI_GPCT_DISABLED_GATE_SELECT);  in ni_tio_set_gate_src_raw()
1046 		ni_tio_set_gate2_raw(counter, src);  in ni_tio_set_gate_src_raw()
1048 		ni_tio_set_gate2_mode(counter, src);  in ni_tio_set_gate_src_raw()
1057 int ni_tio_set_gate_src(struct ni_gpct *counter,  in ni_tio_set_gate_src()  argument
1060 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_gate_src()
1072 		ni_tio_set_gate_mode(counter, NI_GPCT_DISABLED_GATE_SELECT);  in ni_tio_set_gate_src()
1077 			ret = ni_m_set_gate(counter, chan);  in ni_tio_set_gate_src()
1080 			ret = ni_660x_set_gate(counter, chan);  in ni_tio_set_gate_src()
1088 		ni_tio_set_gate_mode(counter, src);  in ni_tio_set_gate_src()
1095 		ni_tio_set_gate2_mode(counter, NI_GPCT_DISABLED_GATE_SELECT);  in ni_tio_set_gate_src()
1099 			ret = ni_m_set_gate2(counter, chan);  in ni_tio_set_gate_src()
1102 			ret = ni_660x_set_gate2(counter, chan);  in ni_tio_set_gate_src()
1110 		ni_tio_set_gate2_mode(counter, src);  in ni_tio_set_gate_src()
1119 static int ni_tio_set_other_src(struct ni_gpct *counter, unsigned int index,  in ni_tio_set_other_src()  argument
1122 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_set_other_src()
1123 	unsigned int cidx = counter->counter_index;  in ni_tio_set_other_src()
1124 	unsigned int chip = counter->chip_index;  in ni_tio_set_other_src()
1152 	ni_tio_write(counter, counter_dev->regs[chip][abz_reg], abz_reg);  in ni_tio_set_other_src()
1156 static int ni_tio_get_other_src(struct ni_gpct *counter, unsigned int index,  in ni_tio_get_other_src()  argument
1159 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_get_other_src()
1160 	unsigned int cidx = counter->counter_index;  in ni_tio_get_other_src()
1185 	*source = (ni_tio_get_soft_copy(counter, abz_reg) >> shift) & mask;  in ni_tio_get_other_src()
1343 static inline unsigned int ni_tio_get_gate_mode(struct ni_gpct *counter)  in ni_tio_get_gate_mode()  argument
1345 	unsigned int mode = ni_tio_get_soft_copy(counter,  in ni_tio_get_gate_mode()
1346 				NITIO_MODE_REG(counter->counter_index));  in ni_tio_get_gate_mode()
1359 static inline unsigned int ni_tio_get_gate2_mode(struct ni_gpct *counter)  in ni_tio_get_gate2_mode()  argument
1361 	unsigned int mode = ni_tio_get_soft_copy(counter,  in ni_tio_get_gate2_mode()
1362 				NITIO_GATE2_REG(counter->counter_index));  in ni_tio_get_gate2_mode()
1373 static inline unsigned int ni_tio_get_gate_val(struct ni_gpct *counter)  in ni_tio_get_gate_val()  argument
1375 	return GI_BITS_TO_GATE(ni_tio_get_soft_copy(counter,  in ni_tio_get_gate_val()
1376 		NITIO_INPUT_SEL_REG(counter->counter_index)));  in ni_tio_get_gate_val()
1379 static inline unsigned int ni_tio_get_gate2_val(struct ni_gpct *counter)  in ni_tio_get_gate2_val()  argument
1381 	return GI_BITS_TO_GATE2(ni_tio_get_soft_copy(counter,  in ni_tio_get_gate2_val()
1382 		NITIO_GATE2_REG(counter->counter_index)));  in ni_tio_get_gate2_val()
1385 static int ni_tio_get_gate_src(struct ni_gpct *counter, unsigned int gate_index,  in ni_tio_get_gate_src()  argument
1393 		gate = ni_tio_get_gate_val(counter);  in ni_tio_get_gate_src()
1394 		switch (counter->counter_dev->variant) {  in ni_tio_get_gate_src()
1406 		*gate_source |= ni_tio_get_gate_mode(counter);  in ni_tio_get_gate_src()
1409 		gate = ni_tio_get_gate2_val(counter);  in ni_tio_get_gate_src()
1410 		switch (counter->counter_dev->variant) {  in ni_tio_get_gate_src()
1422 		*gate_source |= ni_tio_get_gate2_mode(counter);  in ni_tio_get_gate_src()
1430 static int ni_tio_get_gate_src_raw(struct ni_gpct *counter,  in ni_tio_get_gate_src_raw()  argument
1436 		*gate_source = ni_tio_get_gate_mode(counter)  in ni_tio_get_gate_src_raw()
1437 			     | ni_tio_get_gate_val(counter);  in ni_tio_get_gate_src_raw()
1440 		*gate_source = ni_tio_get_gate2_mode(counter)  in ni_tio_get_gate_src_raw()
1441 			     | ni_tio_get_gate2_val(counter);  in ni_tio_get_gate_src_raw()
1454 	struct ni_gpct *counter = s->private;  in ni_tio_insn_config()  local
1455 	unsigned int cidx = counter->counter_index;  in ni_tio_insn_config()
1461 		ret = ni_tio_set_counter_mode(counter, data[1]);  in ni_tio_insn_config()
1464 		ret = ni_tio_arm(counter, true, data[1]);  in ni_tio_insn_config()
1467 		ret = ni_tio_arm(counter, false, 0);  in ni_tio_insn_config()
1471 		status = ni_tio_read(counter, NITIO_SHARED_STATUS_REG(cidx));  in ni_tio_insn_config()
1480 		ret = ni_tio_set_clock_src(counter, data[1], data[2]);  in ni_tio_insn_config()
1483 		ret = ni_tio_get_clock_src(counter, &data[1], &data[2]);  in ni_tio_insn_config()
1486 		ret = ni_tio_set_gate_src(counter, data[1], data[2]);  in ni_tio_insn_config()
1489 		ret = ni_tio_get_gate_src(counter, data[1], &data[2]);  in ni_tio_insn_config()
1492 		ret = ni_tio_set_other_src(counter, data[1], data[2]);  in ni_tio_insn_config()
1495 		ni_tio_reset_count_and_disarm(counter);  in ni_tio_insn_config()
1518 	struct ni_gpct *counter = &counter_dev->counters[ctr_index];  in ni_tio_get_routing()  local
1523 		ret = ni_tio_get_other_src(counter, dest, &reg);  in ni_tio_get_routing()
1525 		ret = ni_tio_get_gate_src_raw(counter, 0, &reg);  in ni_tio_get_routing()
1527 		ret = ni_tio_get_gate_src_raw(counter, 1, &reg);  in ni_tio_get_routing()
1560 	struct ni_gpct *counter = &counter_dev->counters[ctr_index];  in ni_tio_set_routing()  local
1564 		ret = ni_tio_set_other_src(counter, dest, reg);  in ni_tio_set_routing()
1566 		ret = ni_tio_set_gate_src_raw(counter, 0, reg);  in ni_tio_set_routing()
1568 		ret = ni_tio_set_gate_src_raw(counter, 1, reg);  in ni_tio_set_routing()
1608 	struct ni_gpct *counter = s->private;  in ni_tio_read_sw_save_reg()  local
1609 	unsigned int cidx = counter->counter_index;  in ni_tio_read_sw_save_reg()
1612 	ni_tio_set_bits(counter, NITIO_CMD_REG(cidx), GI_SAVE_TRACE, 0);  in ni_tio_read_sw_save_reg()
1613 	ni_tio_set_bits(counter, NITIO_CMD_REG(cidx),  in ni_tio_read_sw_save_reg()
1625 	val = ni_tio_read(counter, NITIO_SW_SAVE_REG(cidx));  in ni_tio_read_sw_save_reg()
1626 	if (val != ni_tio_read(counter, NITIO_SW_SAVE_REG(cidx)))  in ni_tio_read_sw_save_reg()
1627 		val = ni_tio_read(counter, NITIO_SW_SAVE_REG(cidx));  in ni_tio_read_sw_save_reg()
1637 	struct ni_gpct *counter = s->private;  in ni_tio_insn_read()  local
1638 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_insn_read()
1640 	unsigned int cidx = counter->counter_index;  in ni_tio_insn_read()
1641 	unsigned int chip = counter->chip_index;  in ni_tio_insn_read()
1663 static unsigned int ni_tio_next_load_register(struct ni_gpct *counter)  in ni_tio_next_load_register()  argument
1665 	unsigned int cidx = counter->counter_index;  in ni_tio_next_load_register()
1666 	unsigned int bits = ni_tio_read(counter, NITIO_SHARED_STATUS_REG(cidx));  in ni_tio_next_load_register()
1678 	struct ni_gpct *counter = s->private;  in ni_tio_insn_write()  local
1679 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_insn_write()
1681 	unsigned int cidx = counter->counter_index;  in ni_tio_insn_write()
1682 	unsigned int chip = counter->chip_index;  in ni_tio_insn_write()
1700 		load_reg = ni_tio_next_load_register(counter);  in ni_tio_insn_write()
1701 		ni_tio_write(counter, load_val, load_reg);  in ni_tio_insn_write()
1702 		ni_tio_set_bits_transient(counter, NITIO_CMD_REG(cidx),  in ni_tio_insn_write()
1705 		ni_tio_write(counter, counter_dev->regs[chip][load_reg],  in ni_tio_insn_write()
1710 		ni_tio_write(counter, load_val, NITIO_LOADA_REG(cidx));  in ni_tio_insn_write()
1714 		ni_tio_write(counter, load_val, NITIO_LOADB_REG(cidx));  in ni_tio_insn_write()
1723 void ni_tio_init_counter(struct ni_gpct *counter)  in ni_tio_init_counter()  argument
1725 	struct ni_gpct_device *counter_dev = counter->counter_dev;  in ni_tio_init_counter()
1726 	unsigned int cidx = counter->counter_index;  in ni_tio_init_counter()
1727 	unsigned int chip = counter->chip_index;  in ni_tio_init_counter()
1729 	ni_tio_reset_count_and_disarm(counter);  in ni_tio_init_counter()
1733 	ni_tio_write(counter, 0x0, NITIO_AUTO_INC_REG(cidx));  in ni_tio_init_counter()
1735 	ni_tio_set_bits(counter, NITIO_CMD_REG(cidx),  in ni_tio_init_counter()
1738 	ni_tio_set_bits(counter, NITIO_MODE_REG(cidx), ~0, 0);  in ni_tio_init_counter()
1741 	ni_tio_write(counter, 0x0, NITIO_LOADA_REG(cidx));  in ni_tio_init_counter()
1744 	ni_tio_write(counter, 0x0, NITIO_LOADB_REG(cidx));  in ni_tio_init_counter()
1746 	ni_tio_set_bits(counter, NITIO_INPUT_SEL_REG(cidx), ~0, 0);  in ni_tio_init_counter()
1749 		ni_tio_set_bits(counter, NITIO_CNT_MODE_REG(cidx), ~0, 0);  in ni_tio_init_counter()
1753 		ni_tio_write(counter, 0x0, NITIO_GATE2_REG(cidx));  in ni_tio_init_counter()
1756 	ni_tio_set_bits(counter, NITIO_DMA_CFG_REG(cidx), ~0, 0x0);  in ni_tio_init_counter()
1758 	ni_tio_set_bits(counter, NITIO_INT_ENA_REG(cidx), ~0, 0x0);  in ni_tio_init_counter()
1764 			 void (*write)(struct ni_gpct *counter,  in ni_gpct_device_construct()  argument
1767 			 unsigned int (*read)(struct ni_gpct *counter,  in ni_gpct_device_construct()  argument
1775 	struct ni_gpct *counter;  in ni_gpct_device_construct()  local
1796 	counter_dev->counters = kcalloc(num_counters, sizeof(*counter),  in ni_gpct_device_construct()
1808 		counter = &counter_dev->counters[i];  in ni_gpct_device_construct()
1809 		counter->counter_dev = counter_dev;  in ni_gpct_device_construct()
1810 		counter->chip_index = i / counters_per_chip;  in ni_gpct_device_construct()
1811 		counter->counter_index = i % counters_per_chip;  in ni_gpct_device_construct()
1812 		spin_lock_init(&counter->lock);  in ni_gpct_device_construct()