1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) STMicroelectronics 2022 - All Rights Reserved
4  * Author: Gabriel Fernandez <gabriel.fernandez@foss.st.com> for STMicroelectronics.
5  */
6 
7 #include <linux/clk.h>
8 #include <linux/delay.h>
9 #include <linux/device.h>
10 #include <linux/err.h>
11 #include <linux/io.h>
12 #include <linux/of.h>
13 #include <linux/of_address.h>
14 #include <linux/slab.h>
15 #include <linux/spinlock.h>
16 
17 #include "clk-stm32-core.h"
18 #include "reset-stm32.h"
19 
20 static DEFINE_SPINLOCK(rlock);
21 
22 static int stm32_rcc_clock_init(struct device *dev,
23 				const struct of_device_id *match,
24 				void __iomem *base)
25 {
26 	const struct stm32_rcc_match_data *data = match->data;
27 	struct clk_hw_onecell_data *clk_data = data->hw_clks;
28 	struct device_node *np = dev_of_node(dev);
29 	struct clk_hw **hws;
30 	int n, max_binding;
31 
32 	max_binding =  data->maxbinding;
33 
34 	clk_data = devm_kzalloc(dev, struct_size(clk_data, hws, max_binding), GFP_KERNEL);
35 	if (!clk_data)
36 		return -ENOMEM;
37 
38 	clk_data->num = max_binding;
39 
40 	hws = clk_data->hws;
41 
42 	for (n = 0; n < max_binding; n++)
43 		hws[n] = ERR_PTR(-ENOENT);
44 
45 	for (n = 0; n < data->num_clocks; n++) {
46 		const struct clock_config *cfg_clock = &data->tab_clocks[n];
47 		struct clk_hw *hw = ERR_PTR(-ENOENT);
48 
49 		if (data->check_security &&
50 		    data->check_security(base, cfg_clock))
51 			continue;
52 
53 		if (cfg_clock->func)
54 			hw = (*cfg_clock->func)(dev, data, base, &rlock,
55 						cfg_clock);
56 
57 		if (IS_ERR(hw)) {
58 			dev_err(dev, "Can't register clk %d: %ld\n", n,
59 				PTR_ERR(hw));
60 			return PTR_ERR(hw);
61 		}
62 
63 		if (cfg_clock->id != NO_ID)
64 			hws[cfg_clock->id] = hw;
65 	}
66 
67 	return of_clk_add_hw_provider(np, of_clk_hw_onecell_get, clk_data);
68 }
69 
70 int stm32_rcc_init(struct device *dev, const struct of_device_id *match_data,
71 		   void __iomem *base)
72 {
73 	const struct of_device_id *match;
74 	int err;
75 
76 	match = of_match_node(match_data, dev_of_node(dev));
77 	if (!match) {
78 		dev_err(dev, "match data not found\n");
79 		return -ENODEV;
80 	}
81 
82 	/* RCC Reset Configuration */
83 	err = stm32_rcc_reset_init(dev, match, base);
84 	if (err) {
85 		pr_err("stm32 reset failed to initialize\n");
86 		return err;
87 	}
88 
89 	/* RCC Clock Configuration */
90 	err = stm32_rcc_clock_init(dev, match, base);
91 	if (err) {
92 		pr_err("stm32 clock failed to initialize\n");
93 		return err;
94 	}
95 
96 	return 0;
97 }
98 
99 static u8 stm32_mux_get_parent(void __iomem *base,
100 			       struct clk_stm32_clock_data *data,
101 			       u16 mux_id)
102 {
103 	const struct stm32_mux_cfg *mux = &data->muxes[mux_id];
104 	u32 mask = BIT(mux->width) - 1;
105 	u32 val;
106 
107 	val = readl(base + mux->offset) >> mux->shift;
108 	val &= mask;
109 
110 	return val;
111 }
112 
113 static int stm32_mux_set_parent(void __iomem *base,
114 				struct clk_stm32_clock_data *data,
115 				u16 mux_id, u8 index)
116 {
117 	const struct stm32_mux_cfg *mux = &data->muxes[mux_id];
118 
119 	u32 mask = BIT(mux->width) - 1;
120 	u32 reg = readl(base + mux->offset);
121 	u32 val = index << mux->shift;
122 
123 	reg &= ~(mask << mux->shift);
124 	reg |= val;
125 
126 	writel(reg, base + mux->offset);
127 
128 	return 0;
129 }
130 
131 static void stm32_gate_endisable(void __iomem *base,
132 				 struct clk_stm32_clock_data *data,
133 				 u16 gate_id, int enable)
134 {
135 	const struct stm32_gate_cfg *gate = &data->gates[gate_id];
136 	void __iomem *addr = base + gate->offset;
137 
138 	if (enable) {
139 		if (data->gate_cpt[gate_id]++ > 0)
140 			return;
141 
142 		if (gate->set_clr != 0)
143 			writel(BIT(gate->bit_idx), addr);
144 		else
145 			writel(readl(addr) | BIT(gate->bit_idx), addr);
146 	} else {
147 		if (--data->gate_cpt[gate_id] > 0)
148 			return;
149 
150 		if (gate->set_clr != 0)
151 			writel(BIT(gate->bit_idx), addr + gate->set_clr);
152 		else
153 			writel(readl(addr) & ~BIT(gate->bit_idx), addr);
154 	}
155 }
156 
157 static void stm32_gate_disable_unused(void __iomem *base,
158 				      struct clk_stm32_clock_data *data,
159 				      u16 gate_id)
160 {
161 	const struct stm32_gate_cfg *gate = &data->gates[gate_id];
162 	void __iomem *addr = base + gate->offset;
163 
164 	if (data->gate_cpt[gate_id] > 0)
165 		return;
166 
167 	if (gate->set_clr != 0)
168 		writel(BIT(gate->bit_idx), addr + gate->set_clr);
169 	else
170 		writel(readl(addr) & ~BIT(gate->bit_idx), addr);
171 }
172 
173 static int stm32_gate_is_enabled(void __iomem *base,
174 				 struct clk_stm32_clock_data *data,
175 				 u16 gate_id)
176 {
177 	const struct stm32_gate_cfg *gate = &data->gates[gate_id];
178 
179 	return (readl(base + gate->offset) & BIT(gate->bit_idx)) != 0;
180 }
181 
182 static unsigned int _get_table_div(const struct clk_div_table *table,
183 				   unsigned int val)
184 {
185 	const struct clk_div_table *clkt;
186 
187 	for (clkt = table; clkt->div; clkt++)
188 		if (clkt->val == val)
189 			return clkt->div;
190 	return 0;
191 }
192 
193 static unsigned int _get_div(const struct clk_div_table *table,
194 			     unsigned int val, unsigned long flags, u8 width)
195 {
196 	if (flags & CLK_DIVIDER_ONE_BASED)
197 		return val;
198 	if (flags & CLK_DIVIDER_POWER_OF_TWO)
199 		return 1 << val;
200 	if (table)
201 		return _get_table_div(table, val);
202 	return val + 1;
203 }
204 
205 static unsigned long stm32_divider_get_rate(void __iomem *base,
206 					    struct clk_stm32_clock_data *data,
207 					    u16 div_id,
208 					    unsigned long parent_rate)
209 {
210 	const struct stm32_div_cfg *divider = &data->dividers[div_id];
211 	unsigned int val;
212 	unsigned int div;
213 
214 	val =  readl(base + divider->offset) >> divider->shift;
215 	val &= clk_div_mask(divider->width);
216 	div = _get_div(divider->table, val, divider->flags, divider->width);
217 
218 	if (!div) {
219 		WARN(!(divider->flags & CLK_DIVIDER_ALLOW_ZERO),
220 		     "%d: Zero divisor and CLK_DIVIDER_ALLOW_ZERO not set\n",
221 		     div_id);
222 		return parent_rate;
223 	}
224 
225 	return DIV_ROUND_UP_ULL((u64)parent_rate, div);
226 }
227 
228 static int stm32_divider_set_rate(void __iomem *base,
229 				  struct clk_stm32_clock_data *data,
230 				  u16 div_id, unsigned long rate,
231 				  unsigned long parent_rate)
232 {
233 	const struct stm32_div_cfg *divider = &data->dividers[div_id];
234 	int value;
235 	u32 val;
236 
237 	value = divider_get_val(rate, parent_rate, divider->table,
238 				divider->width, divider->flags);
239 	if (value < 0)
240 		return value;
241 
242 	if (divider->flags & CLK_DIVIDER_HIWORD_MASK) {
243 		val = clk_div_mask(divider->width) << (divider->shift + 16);
244 	} else {
245 		val = readl(base + divider->offset);
246 		val &= ~(clk_div_mask(divider->width) << divider->shift);
247 	}
248 
249 	val |= (u32)value << divider->shift;
250 
251 	writel(val, base + divider->offset);
252 
253 	return 0;
254 }
255 
256 static u8 clk_stm32_mux_get_parent(struct clk_hw *hw)
257 {
258 	struct clk_stm32_mux *mux = to_clk_stm32_mux(hw);
259 
260 	return stm32_mux_get_parent(mux->base, mux->clock_data, mux->mux_id);
261 }
262 
263 static int clk_stm32_mux_set_parent(struct clk_hw *hw, u8 index)
264 {
265 	struct clk_stm32_mux *mux = to_clk_stm32_mux(hw);
266 	unsigned long flags = 0;
267 
268 	spin_lock_irqsave(mux->lock, flags);
269 
270 	stm32_mux_set_parent(mux->base, mux->clock_data, mux->mux_id, index);
271 
272 	spin_unlock_irqrestore(mux->lock, flags);
273 
274 	return 0;
275 }
276 
277 const struct clk_ops clk_stm32_mux_ops = {
278 	.get_parent	= clk_stm32_mux_get_parent,
279 	.set_parent	= clk_stm32_mux_set_parent,
280 };
281 
282 static void clk_stm32_gate_endisable(struct clk_hw *hw, int enable)
283 {
284 	struct clk_stm32_gate *gate = to_clk_stm32_gate(hw);
285 	unsigned long flags = 0;
286 
287 	spin_lock_irqsave(gate->lock, flags);
288 
289 	stm32_gate_endisable(gate->base, gate->clock_data, gate->gate_id, enable);
290 
291 	spin_unlock_irqrestore(gate->lock, flags);
292 }
293 
294 static int clk_stm32_gate_enable(struct clk_hw *hw)
295 {
296 	clk_stm32_gate_endisable(hw, 1);
297 
298 	return 0;
299 }
300 
301 static void clk_stm32_gate_disable(struct clk_hw *hw)
302 {
303 	clk_stm32_gate_endisable(hw, 0);
304 }
305 
306 static int clk_stm32_gate_is_enabled(struct clk_hw *hw)
307 {
308 	struct clk_stm32_gate *gate = to_clk_stm32_gate(hw);
309 
310 	return stm32_gate_is_enabled(gate->base, gate->clock_data, gate->gate_id);
311 }
312 
313 static void clk_stm32_gate_disable_unused(struct clk_hw *hw)
314 {
315 	struct clk_stm32_gate *gate = to_clk_stm32_gate(hw);
316 	unsigned long flags = 0;
317 
318 	spin_lock_irqsave(gate->lock, flags);
319 
320 	stm32_gate_disable_unused(gate->base, gate->clock_data, gate->gate_id);
321 
322 	spin_unlock_irqrestore(gate->lock, flags);
323 }
324 
325 const struct clk_ops clk_stm32_gate_ops = {
326 	.enable		= clk_stm32_gate_enable,
327 	.disable	= clk_stm32_gate_disable,
328 	.is_enabled	= clk_stm32_gate_is_enabled,
329 	.disable_unused	= clk_stm32_gate_disable_unused,
330 };
331 
332 static int clk_stm32_divider_set_rate(struct clk_hw *hw, unsigned long rate,
333 				      unsigned long parent_rate)
334 {
335 	struct clk_stm32_div *div = to_clk_stm32_divider(hw);
336 	unsigned long flags = 0;
337 	int ret;
338 
339 	if (div->div_id == NO_STM32_DIV)
340 		return rate;
341 
342 	spin_lock_irqsave(div->lock, flags);
343 
344 	ret = stm32_divider_set_rate(div->base, div->clock_data, div->div_id, rate, parent_rate);
345 
346 	spin_unlock_irqrestore(div->lock, flags);
347 
348 	return ret;
349 }
350 
351 static long clk_stm32_divider_round_rate(struct clk_hw *hw, unsigned long rate,
352 					 unsigned long *prate)
353 {
354 	struct clk_stm32_div *div = to_clk_stm32_divider(hw);
355 	const struct stm32_div_cfg *divider;
356 
357 	if (div->div_id == NO_STM32_DIV)
358 		return rate;
359 
360 	divider = &div->clock_data->dividers[div->div_id];
361 
362 	/* if read only, just return current value */
363 	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
364 		u32 val;
365 
366 		val =  readl(div->base + divider->offset) >> divider->shift;
367 		val &= clk_div_mask(divider->width);
368 
369 		return divider_ro_round_rate(hw, rate, prate, divider->table,
370 				divider->width, divider->flags,
371 				val);
372 	}
373 
374 	return divider_round_rate_parent(hw, clk_hw_get_parent(hw),
375 					 rate, prate, divider->table,
376 					 divider->width, divider->flags);
377 }
378 
379 static unsigned long clk_stm32_divider_recalc_rate(struct clk_hw *hw,
380 						   unsigned long parent_rate)
381 {
382 	struct clk_stm32_div *div = to_clk_stm32_divider(hw);
383 
384 	if (div->div_id == NO_STM32_DIV)
385 		return parent_rate;
386 
387 	return stm32_divider_get_rate(div->base, div->clock_data, div->div_id, parent_rate);
388 }
389 
390 const struct clk_ops clk_stm32_divider_ops = {
391 	.recalc_rate	= clk_stm32_divider_recalc_rate,
392 	.round_rate	= clk_stm32_divider_round_rate,
393 	.set_rate	= clk_stm32_divider_set_rate,
394 };
395 
396 static int clk_stm32_composite_set_rate(struct clk_hw *hw, unsigned long rate,
397 					unsigned long parent_rate)
398 {
399 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
400 	unsigned long flags = 0;
401 	int ret;
402 
403 	if (composite->div_id == NO_STM32_DIV)
404 		return rate;
405 
406 	spin_lock_irqsave(composite->lock, flags);
407 
408 	ret = stm32_divider_set_rate(composite->base, composite->clock_data,
409 				     composite->div_id, rate, parent_rate);
410 
411 	spin_unlock_irqrestore(composite->lock, flags);
412 
413 	return ret;
414 }
415 
416 static unsigned long clk_stm32_composite_recalc_rate(struct clk_hw *hw,
417 						     unsigned long parent_rate)
418 {
419 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
420 
421 	if (composite->div_id == NO_STM32_DIV)
422 		return parent_rate;
423 
424 	return stm32_divider_get_rate(composite->base, composite->clock_data,
425 				      composite->div_id, parent_rate);
426 }
427 
428 static long clk_stm32_composite_round_rate(struct clk_hw *hw, unsigned long rate,
429 					   unsigned long *prate)
430 {
431 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
432 
433 	const struct stm32_div_cfg *divider;
434 
435 	if (composite->div_id == NO_STM32_DIV)
436 		return rate;
437 
438 	divider = &composite->clock_data->dividers[composite->div_id];
439 
440 	/* if read only, just return current value */
441 	if (divider->flags & CLK_DIVIDER_READ_ONLY) {
442 		u32 val;
443 
444 		val =  readl(composite->base + divider->offset) >> divider->shift;
445 		val &= clk_div_mask(divider->width);
446 
447 		return divider_ro_round_rate(hw, rate, prate, divider->table,
448 				divider->width, divider->flags,
449 				val);
450 	}
451 
452 	return divider_round_rate_parent(hw, clk_hw_get_parent(hw),
453 					 rate, prate, divider->table,
454 					 divider->width, divider->flags);
455 }
456 
457 static u8 clk_stm32_composite_get_parent(struct clk_hw *hw)
458 {
459 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
460 
461 	return stm32_mux_get_parent(composite->base, composite->clock_data, composite->mux_id);
462 }
463 
464 static int clk_stm32_composite_set_parent(struct clk_hw *hw, u8 index)
465 {
466 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
467 	unsigned long flags = 0;
468 
469 	spin_lock_irqsave(composite->lock, flags);
470 
471 	stm32_mux_set_parent(composite->base, composite->clock_data, composite->mux_id, index);
472 
473 	spin_unlock_irqrestore(composite->lock, flags);
474 
475 	if (composite->clock_data->is_multi_mux) {
476 		struct clk_hw *other_mux_hw = composite->clock_data->is_multi_mux(hw);
477 
478 		if (other_mux_hw) {
479 			struct clk_hw *hwp = clk_hw_get_parent_by_index(hw, index);
480 
481 			clk_hw_reparent(other_mux_hw, hwp);
482 		}
483 	}
484 
485 	return 0;
486 }
487 
488 static int clk_stm32_composite_is_enabled(struct clk_hw *hw)
489 {
490 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
491 
492 	if (composite->gate_id == NO_STM32_GATE)
493 		return (__clk_get_enable_count(hw->clk) > 0);
494 
495 	return stm32_gate_is_enabled(composite->base, composite->clock_data, composite->gate_id);
496 }
497 
498 #define MUX_SAFE_POSITION 0
499 
500 static int clk_stm32_has_safe_mux(struct clk_hw *hw)
501 {
502 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
503 	const struct stm32_mux_cfg *mux = &composite->clock_data->muxes[composite->mux_id];
504 
505 	return !!(mux->flags & MUX_SAFE);
506 }
507 
508 static void clk_stm32_set_safe_position_mux(struct clk_hw *hw)
509 {
510 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
511 
512 	if (!clk_stm32_composite_is_enabled(hw)) {
513 		unsigned long flags = 0;
514 
515 		if (composite->clock_data->is_multi_mux) {
516 			struct clk_hw *other_mux_hw = NULL;
517 
518 			other_mux_hw = composite->clock_data->is_multi_mux(hw);
519 
520 			if (!other_mux_hw || clk_stm32_composite_is_enabled(other_mux_hw))
521 				return;
522 		}
523 
524 		spin_lock_irqsave(composite->lock, flags);
525 
526 		stm32_mux_set_parent(composite->base, composite->clock_data,
527 				     composite->mux_id, MUX_SAFE_POSITION);
528 
529 		spin_unlock_irqrestore(composite->lock, flags);
530 	}
531 }
532 
533 static void clk_stm32_safe_restore_position_mux(struct clk_hw *hw)
534 {
535 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
536 	int sel = clk_hw_get_parent_index(hw);
537 	unsigned long flags = 0;
538 
539 	spin_lock_irqsave(composite->lock, flags);
540 
541 	stm32_mux_set_parent(composite->base, composite->clock_data, composite->mux_id, sel);
542 
543 	spin_unlock_irqrestore(composite->lock, flags);
544 }
545 
546 static void clk_stm32_composite_gate_endisable(struct clk_hw *hw, int enable)
547 {
548 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
549 	unsigned long flags = 0;
550 
551 	spin_lock_irqsave(composite->lock, flags);
552 
553 	stm32_gate_endisable(composite->base, composite->clock_data, composite->gate_id, enable);
554 
555 	spin_unlock_irqrestore(composite->lock, flags);
556 }
557 
558 static int clk_stm32_composite_gate_enable(struct clk_hw *hw)
559 {
560 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
561 
562 	if (composite->gate_id == NO_STM32_GATE)
563 		return 0;
564 
565 	clk_stm32_composite_gate_endisable(hw, 1);
566 
567 	if (composite->mux_id != NO_STM32_MUX && clk_stm32_has_safe_mux(hw))
568 		clk_stm32_safe_restore_position_mux(hw);
569 
570 	return 0;
571 }
572 
573 static void clk_stm32_composite_gate_disable(struct clk_hw *hw)
574 {
575 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
576 
577 	if (composite->gate_id == NO_STM32_GATE)
578 		return;
579 
580 	clk_stm32_composite_gate_endisable(hw, 0);
581 
582 	if (composite->mux_id != NO_STM32_MUX && clk_stm32_has_safe_mux(hw))
583 		clk_stm32_set_safe_position_mux(hw);
584 }
585 
586 static void clk_stm32_composite_disable_unused(struct clk_hw *hw)
587 {
588 	struct clk_stm32_composite *composite = to_clk_stm32_composite(hw);
589 	unsigned long flags = 0;
590 
591 	if (composite->gate_id == NO_STM32_GATE)
592 		return;
593 
594 	spin_lock_irqsave(composite->lock, flags);
595 
596 	stm32_gate_disable_unused(composite->base, composite->clock_data, composite->gate_id);
597 
598 	spin_unlock_irqrestore(composite->lock, flags);
599 }
600 
601 const struct clk_ops clk_stm32_composite_ops = {
602 	.set_rate	= clk_stm32_composite_set_rate,
603 	.recalc_rate	= clk_stm32_composite_recalc_rate,
604 	.round_rate	= clk_stm32_composite_round_rate,
605 	.get_parent	= clk_stm32_composite_get_parent,
606 	.set_parent	= clk_stm32_composite_set_parent,
607 	.enable		= clk_stm32_composite_gate_enable,
608 	.disable	= clk_stm32_composite_gate_disable,
609 	.is_enabled	= clk_stm32_composite_is_enabled,
610 	.disable_unused	= clk_stm32_composite_disable_unused,
611 };
612 
613 struct clk_hw *clk_stm32_mux_register(struct device *dev,
614 				      const struct stm32_rcc_match_data *data,
615 				      void __iomem *base,
616 				      spinlock_t *lock,
617 				      const struct clock_config *cfg)
618 {
619 	struct clk_stm32_mux *mux = cfg->clock_cfg;
620 	struct clk_hw *hw = &mux->hw;
621 	int err;
622 
623 	mux->base = base;
624 	mux->lock = lock;
625 	mux->clock_data = data->clock_data;
626 
627 	err = clk_hw_register(dev, hw);
628 	if (err)
629 		return ERR_PTR(err);
630 
631 	return hw;
632 }
633 
634 struct clk_hw *clk_stm32_gate_register(struct device *dev,
635 				       const struct stm32_rcc_match_data *data,
636 				       void __iomem *base,
637 				       spinlock_t *lock,
638 				       const struct clock_config *cfg)
639 {
640 	struct clk_stm32_gate *gate = cfg->clock_cfg;
641 	struct clk_hw *hw = &gate->hw;
642 	int err;
643 
644 	gate->base = base;
645 	gate->lock = lock;
646 	gate->clock_data = data->clock_data;
647 
648 	err = clk_hw_register(dev, hw);
649 	if (err)
650 		return ERR_PTR(err);
651 
652 	return hw;
653 }
654 
655 struct clk_hw *clk_stm32_div_register(struct device *dev,
656 				      const struct stm32_rcc_match_data *data,
657 				      void __iomem *base,
658 				      spinlock_t *lock,
659 				      const struct clock_config *cfg)
660 {
661 	struct clk_stm32_div *div = cfg->clock_cfg;
662 	struct clk_hw *hw = &div->hw;
663 	int err;
664 
665 	div->base = base;
666 	div->lock = lock;
667 	div->clock_data = data->clock_data;
668 
669 	err = clk_hw_register(dev, hw);
670 	if (err)
671 		return ERR_PTR(err);
672 
673 	return hw;
674 }
675 
676 struct clk_hw *clk_stm32_composite_register(struct device *dev,
677 					    const struct stm32_rcc_match_data *data,
678 					    void __iomem *base,
679 					    spinlock_t *lock,
680 					    const struct clock_config *cfg)
681 {
682 	struct clk_stm32_composite *composite = cfg->clock_cfg;
683 	struct clk_hw *hw = &composite->hw;
684 	int err;
685 
686 	composite->base = base;
687 	composite->lock = lock;
688 	composite->clock_data = data->clock_data;
689 
690 	err = clk_hw_register(dev, hw);
691 	if (err)
692 		return ERR_PTR(err);
693 
694 	return hw;
695 }
696