1 /*
2  * SPDX-License-Identifier: MIT
3  *
4  * Copyright © 2019 Intel Corporation
5  */
6 
7 #include <linux/wait_bit.h>
8 
9 #include "intel_runtime_pm.h"
10 #include "intel_wakeref.h"
11 
12 static void rpm_get(struct intel_wakeref *wf)
13 {
14 	wf->wakeref = intel_runtime_pm_get(wf->rpm);
15 }
16 
17 static void rpm_put(struct intel_wakeref *wf)
18 {
19 	intel_wakeref_t wakeref = fetch_and_zero(&wf->wakeref);
20 
21 	intel_runtime_pm_put(wf->rpm, wakeref);
22 	INTEL_WAKEREF_BUG_ON(!wakeref);
23 }
24 
25 int __intel_wakeref_get_first(struct intel_wakeref *wf)
26 {
27 	/*
28 	 * Treat get/put as different subclasses, as we may need to run
29 	 * the put callback from under the shrinker and do not want to
30 	 * cross-contanimate that callback with any extra work performed
31 	 * upon acquiring the wakeref.
32 	 */
33 	mutex_lock_nested(&wf->mutex, SINGLE_DEPTH_NESTING);
34 	if (!atomic_read(&wf->count)) {
35 		int err;
36 
37 		rpm_get(wf);
38 
39 		err = wf->ops->get(wf);
40 		if (unlikely(err)) {
41 			rpm_put(wf);
42 			mutex_unlock(&wf->mutex);
43 			return err;
44 		}
45 
46 		smp_mb__before_atomic(); /* release wf->count */
47 	}
48 	atomic_inc(&wf->count);
49 	mutex_unlock(&wf->mutex);
50 
51 	INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
52 	return 0;
53 }
54 
55 static void ____intel_wakeref_put_last(struct intel_wakeref *wf)
56 {
57 	INTEL_WAKEREF_BUG_ON(atomic_read(&wf->count) <= 0);
58 	if (unlikely(!atomic_dec_and_test(&wf->count)))
59 		goto unlock;
60 
61 	/* ops->put() must reschedule its own release on error/deferral */
62 	if (likely(!wf->ops->put(wf))) {
63 		rpm_put(wf);
64 		wake_up_var(&wf->wakeref);
65 	}
66 
67 unlock:
68 	mutex_unlock(&wf->mutex);
69 }
70 
71 void __intel_wakeref_put_last(struct intel_wakeref *wf, unsigned long flags)
72 {
73 	INTEL_WAKEREF_BUG_ON(work_pending(&wf->work));
74 
75 	/* Assume we are not in process context and so cannot sleep. */
76 	if (flags & INTEL_WAKEREF_PUT_ASYNC || !mutex_trylock(&wf->mutex)) {
77 		schedule_work(&wf->work);
78 		return;
79 	}
80 
81 	____intel_wakeref_put_last(wf);
82 }
83 
84 static void __intel_wakeref_put_work(struct work_struct *wrk)
85 {
86 	struct intel_wakeref *wf = container_of(wrk, typeof(*wf), work);
87 
88 	if (atomic_add_unless(&wf->count, -1, 1))
89 		return;
90 
91 	mutex_lock(&wf->mutex);
92 	____intel_wakeref_put_last(wf);
93 }
94 
95 void __intel_wakeref_init(struct intel_wakeref *wf,
96 			  struct intel_runtime_pm *rpm,
97 			  const struct intel_wakeref_ops *ops,
98 			  struct intel_wakeref_lockclass *key)
99 {
100 	wf->rpm = rpm;
101 	wf->ops = ops;
102 
103 	__mutex_init(&wf->mutex, "wakeref.mutex", &key->mutex);
104 	atomic_set(&wf->count, 0);
105 	wf->wakeref = 0;
106 
107 	INIT_WORK(&wf->work, __intel_wakeref_put_work);
108 	lockdep_init_map(&wf->work.lockdep_map, "wakeref.work", &key->work, 0);
109 }
110 
111 int intel_wakeref_wait_for_idle(struct intel_wakeref *wf)
112 {
113 	int err;
114 
115 	might_sleep();
116 
117 	err = wait_var_event_killable(&wf->wakeref,
118 				      !intel_wakeref_is_active(wf));
119 	if (err)
120 		return err;
121 
122 	intel_wakeref_unlock_wait(wf);
123 	return 0;
124 }
125 
126 static void wakeref_auto_timeout(struct timer_list *t)
127 {
128 	struct intel_wakeref_auto *wf = from_timer(wf, t, timer);
129 	intel_wakeref_t wakeref;
130 	unsigned long flags;
131 
132 	if (!refcount_dec_and_lock_irqsave(&wf->count, &wf->lock, &flags))
133 		return;
134 
135 	wakeref = fetch_and_zero(&wf->wakeref);
136 	spin_unlock_irqrestore(&wf->lock, flags);
137 
138 	intel_runtime_pm_put(wf->rpm, wakeref);
139 }
140 
141 void intel_wakeref_auto_init(struct intel_wakeref_auto *wf,
142 			     struct intel_runtime_pm *rpm)
143 {
144 	spin_lock_init(&wf->lock);
145 	timer_setup(&wf->timer, wakeref_auto_timeout, 0);
146 	refcount_set(&wf->count, 0);
147 	wf->wakeref = 0;
148 	wf->rpm = rpm;
149 }
150 
151 void intel_wakeref_auto(struct intel_wakeref_auto *wf, unsigned long timeout)
152 {
153 	unsigned long flags;
154 
155 	if (!timeout) {
156 		if (del_timer_sync(&wf->timer))
157 			wakeref_auto_timeout(&wf->timer);
158 		return;
159 	}
160 
161 	/* Our mission is that we only extend an already active wakeref */
162 	assert_rpm_wakelock_held(wf->rpm);
163 
164 	if (!refcount_inc_not_zero(&wf->count)) {
165 		spin_lock_irqsave(&wf->lock, flags);
166 		if (!refcount_inc_not_zero(&wf->count)) {
167 			INTEL_WAKEREF_BUG_ON(wf->wakeref);
168 			wf->wakeref = intel_runtime_pm_get_if_in_use(wf->rpm);
169 			refcount_set(&wf->count, 1);
170 		}
171 		spin_unlock_irqrestore(&wf->lock, flags);
172 	}
173 
174 	/*
175 	 * If we extend a pending timer, we will only get a single timer
176 	 * callback and so need to cancel the local inc by running the
177 	 * elided callback to keep the wf->count balanced.
178 	 */
179 	if (mod_timer(&wf->timer, jiffies + timeout))
180 		wakeref_auto_timeout(&wf->timer);
181 }
182 
183 void intel_wakeref_auto_fini(struct intel_wakeref_auto *wf)
184 {
185 	intel_wakeref_auto(wf, 0);
186 	INTEL_WAKEREF_BUG_ON(wf->wakeref);
187 }
188