25 #include <linux/uio.h> |
25 #include <linux/uio.h> |
26 #include <linux/kref.h> |
26 #include <linux/kref.h> |
27 #include <linux/completion.h> |
27 #include <linux/completion.h> |
28 #include <linux/rcupdate.h> |
28 #include <linux/rcupdate.h> |
29 #include <linux/dma-mapping.h> |
29 #include <linux/dma-mapping.h> |
30 |
|
31 /** |
|
32 * enum dma_state - resource PNP/power management state |
|
33 * @DMA_RESOURCE_SUSPEND: DMA device going into low power state |
|
34 * @DMA_RESOURCE_RESUME: DMA device returning to full power |
|
35 * @DMA_RESOURCE_AVAILABLE: DMA device available to the system |
|
36 * @DMA_RESOURCE_REMOVED: DMA device removed from the system |
|
37 */ |
|
38 enum dma_state { |
|
39 DMA_RESOURCE_SUSPEND, |
|
40 DMA_RESOURCE_RESUME, |
|
41 DMA_RESOURCE_AVAILABLE, |
|
42 DMA_RESOURCE_REMOVED, |
|
43 }; |
|
44 |
|
45 /** |
|
46 * enum dma_state_client - state of the channel in the client |
|
47 * @DMA_ACK: client would like to use, or was using this channel |
|
48 * @DMA_DUP: client has already seen this channel, or is not using this channel |
|
49 * @DMA_NAK: client does not want to see any more channels |
|
50 */ |
|
51 enum dma_state_client { |
|
52 DMA_ACK, |
|
53 DMA_DUP, |
|
54 DMA_NAK, |
|
55 }; |
|
56 |
30 |
57 /** |
31 /** |
58 * typedef dma_cookie_t - an opaque DMA cookie |
32 * typedef dma_cookie_t - an opaque DMA cookie |
59 * |
33 * |
60 * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code |
34 * if dma_cookie_t is >0 it's a DMA request cookie, <0 it's an error code |
130 * See linux/cpumask.h |
94 * See linux/cpumask.h |
131 */ |
95 */ |
132 typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t; |
96 typedef struct { DECLARE_BITMAP(bits, DMA_TX_TYPE_END); } dma_cap_mask_t; |
133 |
97 |
134 /** |
98 /** |
135 * struct dma_slave - Information about a DMA slave |
|
136 * @dev: device acting as DMA slave |
|
137 * @dma_dev: required DMA master device. If non-NULL, the client can not be |
|
138 * bound to other masters than this. |
|
139 * @tx_reg: physical address of data register used for |
|
140 * memory-to-peripheral transfers |
|
141 * @rx_reg: physical address of data register used for |
|
142 * peripheral-to-memory transfers |
|
143 * @reg_width: peripheral register width |
|
144 * |
|
145 * If dma_dev is non-NULL, the client can not be bound to other DMA |
|
146 * masters than the one corresponding to this device. The DMA master |
|
147 * driver may use this to determine if there is controller-specific |
|
148 * data wrapped around this struct. Drivers of platform code that sets |
|
149 * the dma_dev field must therefore make sure to use an appropriate |
|
150 * controller-specific dma slave structure wrapping this struct. |
|
151 */ |
|
152 struct dma_slave { |
|
153 struct device *dev; |
|
154 struct device *dma_dev; |
|
155 dma_addr_t tx_reg; |
|
156 dma_addr_t rx_reg; |
|
157 enum dma_slave_width reg_width; |
|
158 }; |
|
159 |
|
160 /** |
|
161 * struct dma_chan_percpu - the per-CPU part of struct dma_chan |
99 * struct dma_chan_percpu - the per-CPU part of struct dma_chan |
162 * @refcount: local_t used for open-coded "bigref" counting |
|
163 * @memcpy_count: transaction counter |
100 * @memcpy_count: transaction counter |
164 * @bytes_transferred: byte counter |
101 * @bytes_transferred: byte counter |
165 */ |
102 */ |
166 |
103 |
167 struct dma_chan_percpu { |
104 struct dma_chan_percpu { |
168 local_t refcount; |
|
169 /* stats */ |
105 /* stats */ |
170 unsigned long memcpy_count; |
106 unsigned long memcpy_count; |
171 unsigned long bytes_transferred; |
107 unsigned long bytes_transferred; |
172 }; |
108 }; |
173 |
109 |
174 /** |
110 /** |
175 * struct dma_chan - devices supply DMA channels, clients use them |
111 * struct dma_chan - devices supply DMA channels, clients use them |
176 * @device: ptr to the dma device who supplies this channel, always !%NULL |
112 * @device: ptr to the dma device who supplies this channel, always !%NULL |
177 * @cookie: last cookie value returned to client |
113 * @cookie: last cookie value returned to client |
178 * @chan_id: channel ID for sysfs |
114 * @chan_id: channel ID for sysfs |
179 * @class_dev: class device for sysfs |
115 * @dev: class device for sysfs |
180 * @refcount: kref, used in "bigref" slow-mode |
|
181 * @slow_ref: indicates that the DMA channel is free |
|
182 * @rcu: the DMA channel's RCU head |
|
183 * @device_node: used to add this to the device chan list |
116 * @device_node: used to add this to the device chan list |
184 * @local: per-cpu pointer to a struct dma_chan_percpu |
117 * @local: per-cpu pointer to a struct dma_chan_percpu |
185 * @client-count: how many clients are using this channel |
118 * @client-count: how many clients are using this channel |
|
119 * @table_count: number of appearances in the mem-to-mem allocation table |
|
120 * @private: private data for certain client-channel associations |
186 */ |
121 */ |
187 struct dma_chan { |
122 struct dma_chan { |
188 struct dma_device *device; |
123 struct dma_device *device; |
189 dma_cookie_t cookie; |
124 dma_cookie_t cookie; |
190 |
125 |
191 /* sysfs */ |
126 /* sysfs */ |
192 int chan_id; |
127 int chan_id; |
193 struct device dev; |
128 struct dma_chan_dev *dev; |
194 |
|
195 struct kref refcount; |
|
196 int slow_ref; |
|
197 struct rcu_head rcu; |
|
198 |
129 |
199 struct list_head device_node; |
130 struct list_head device_node; |
200 struct dma_chan_percpu *local; |
131 struct dma_chan_percpu *local; |
201 int client_count; |
132 int client_count; |
202 }; |
133 int table_count; |
203 |
134 void *private; |
204 #define to_dma_chan(p) container_of(p, struct dma_chan, dev) |
135 }; |
|
136 |
|
137 /** |
|
138 * struct dma_chan_dev - relate sysfs device node to backing channel device |
|
139 * @chan - driver channel device |
|
140 * @device - sysfs device |
|
141 * @dev_id - parent dma_device dev_id |
|
142 * @idr_ref - reference count to gate release of dma_device dev_id |
|
143 */ |
|
144 struct dma_chan_dev { |
|
145 struct dma_chan *chan; |
|
146 struct device device; |
|
147 int dev_id; |
|
148 atomic_t *idr_ref; |
|
149 }; |
|
150 |
|
151 static inline const char *dma_chan_name(struct dma_chan *chan) |
|
152 { |
|
153 return dev_name(&chan->dev->device); |
|
154 } |
205 |
155 |
206 void dma_chan_cleanup(struct kref *kref); |
156 void dma_chan_cleanup(struct kref *kref); |
207 |
157 |
208 static inline void dma_chan_get(struct dma_chan *chan) |
158 /** |
209 { |
159 * typedef dma_filter_fn - callback filter for dma_request_channel |
210 if (unlikely(chan->slow_ref)) |
160 * @chan: channel to be reviewed |
211 kref_get(&chan->refcount); |
161 * @filter_param: opaque parameter passed through dma_request_channel |
212 else { |
162 * |
213 local_inc(&(per_cpu_ptr(chan->local, get_cpu())->refcount)); |
163 * When this optional parameter is specified in a call to dma_request_channel a |
214 put_cpu(); |
164 * suitable channel is passed to this routine for further dispositioning before |
215 } |
165 * being returned. Where 'suitable' indicates a non-busy channel that |
216 } |
166 * satisfies the given capability mask. It returns 'true' to indicate that the |
217 |
167 * channel is suitable. |
218 static inline void dma_chan_put(struct dma_chan *chan) |
168 */ |
219 { |
169 typedef bool (*dma_filter_fn)(struct dma_chan *chan, void *filter_param); |
220 if (unlikely(chan->slow_ref)) |
|
221 kref_put(&chan->refcount, dma_chan_cleanup); |
|
222 else { |
|
223 local_dec(&(per_cpu_ptr(chan->local, get_cpu())->refcount)); |
|
224 put_cpu(); |
|
225 } |
|
226 } |
|
227 |
|
228 /* |
|
229 * typedef dma_event_callback - function pointer to a DMA event callback |
|
230 * For each channel added to the system this routine is called for each client. |
|
231 * If the client would like to use the channel it returns '1' to signal (ack) |
|
232 * the dmaengine core to take out a reference on the channel and its |
|
233 * corresponding device. A client must not 'ack' an available channel more |
|
234 * than once. When a channel is removed all clients are notified. If a client |
|
235 * is using the channel it must 'ack' the removal. A client must not 'ack' a |
|
236 * removed channel more than once. |
|
237 * @client - 'this' pointer for the client context |
|
238 * @chan - channel to be acted upon |
|
239 * @state - available or removed |
|
240 */ |
|
241 struct dma_client; |
|
242 typedef enum dma_state_client (*dma_event_callback) (struct dma_client *client, |
|
243 struct dma_chan *chan, enum dma_state state); |
|
244 |
|
245 /** |
|
246 * struct dma_client - info on the entity making use of DMA services |
|
247 * @event_callback: func ptr to call when something happens |
|
248 * @cap_mask: only return channels that satisfy the requested capabilities |
|
249 * a value of zero corresponds to any capability |
|
250 * @slave: data for preparing slave transfer. Must be non-NULL iff the |
|
251 * DMA_SLAVE capability is requested. |
|
252 * @global_node: list_head for global dma_client_list |
|
253 */ |
|
254 struct dma_client { |
|
255 dma_event_callback event_callback; |
|
256 dma_cap_mask_t cap_mask; |
|
257 struct dma_slave *slave; |
|
258 struct list_head global_node; |
|
259 }; |
|
260 |
170 |
261 typedef void (*dma_async_tx_callback)(void *dma_async_param); |
171 typedef void (*dma_async_tx_callback)(void *dma_async_param); |
262 /** |
172 /** |
263 * struct dma_async_tx_descriptor - async transaction descriptor |
173 * struct dma_async_tx_descriptor - async transaction descriptor |
264 * ---dma generic offload fields--- |
174 * ---dma generic offload fields--- |
311 * @device_prep_dma_zero_sum: prepares a zero_sum operation |
219 * @device_prep_dma_zero_sum: prepares a zero_sum operation |
312 * @device_prep_dma_memset: prepares a memset operation |
220 * @device_prep_dma_memset: prepares a memset operation |
313 * @device_prep_dma_interrupt: prepares an end of chain interrupt operation |
221 * @device_prep_dma_interrupt: prepares an end of chain interrupt operation |
314 * @device_prep_slave_sg: prepares a slave dma operation |
222 * @device_prep_slave_sg: prepares a slave dma operation |
315 * @device_terminate_all: terminate all pending operations |
223 * @device_terminate_all: terminate all pending operations |
|
224 * @device_is_tx_complete: poll for transaction completion |
316 * @device_issue_pending: push pending transactions to hardware |
225 * @device_issue_pending: push pending transactions to hardware |
317 */ |
226 */ |
318 struct dma_device { |
227 struct dma_device { |
319 |
228 |
320 unsigned int chancnt; |
229 unsigned int chancnt; |
321 struct list_head channels; |
230 struct list_head channels; |
322 struct list_head global_node; |
231 struct list_head global_node; |
323 dma_cap_mask_t cap_mask; |
232 dma_cap_mask_t cap_mask; |
324 int max_xor; |
233 int max_xor; |
325 |
234 |
326 struct kref refcount; |
|
327 struct completion done; |
|
328 |
|
329 int dev_id; |
235 int dev_id; |
330 struct device *dev; |
236 struct device *dev; |
331 |
237 |
332 int (*device_alloc_chan_resources)(struct dma_chan *chan, |
238 int (*device_alloc_chan_resources)(struct dma_chan *chan); |
333 struct dma_client *client); |
|
334 void (*device_free_chan_resources)(struct dma_chan *chan); |
239 void (*device_free_chan_resources)(struct dma_chan *chan); |
335 |
240 |
336 struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( |
241 struct dma_async_tx_descriptor *(*device_prep_dma_memcpy)( |
337 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, |
242 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, |
338 size_t len, unsigned long flags); |
243 size_t len, unsigned long flags); |
360 void (*device_issue_pending)(struct dma_chan *chan); |
265 void (*device_issue_pending)(struct dma_chan *chan); |
361 }; |
266 }; |
362 |
267 |
363 /* --- public DMA engine API --- */ |
268 /* --- public DMA engine API --- */ |
364 |
269 |
365 void dma_async_client_register(struct dma_client *client); |
270 #ifdef CONFIG_DMA_ENGINE |
366 void dma_async_client_unregister(struct dma_client *client); |
271 void dmaengine_get(void); |
367 void dma_async_client_chan_request(struct dma_client *client); |
272 void dmaengine_put(void); |
|
273 #else |
|
274 static inline void dmaengine_get(void) |
|
275 { |
|
276 } |
|
277 static inline void dmaengine_put(void) |
|
278 { |
|
279 } |
|
280 #endif |
|
281 |
|
282 #ifdef CONFIG_NET_DMA |
|
283 #define net_dmaengine_get() dmaengine_get() |
|
284 #define net_dmaengine_put() dmaengine_put() |
|
285 #else |
|
286 static inline void net_dmaengine_get(void) |
|
287 { |
|
288 } |
|
289 static inline void net_dmaengine_put(void) |
|
290 { |
|
291 } |
|
292 #endif |
|
293 |
368 dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan, |
294 dma_cookie_t dma_async_memcpy_buf_to_buf(struct dma_chan *chan, |
369 void *dest, void *src, size_t len); |
295 void *dest, void *src, size_t len); |
370 dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan, |
296 dma_cookie_t dma_async_memcpy_buf_to_pg(struct dma_chan *chan, |
371 struct page *page, unsigned int offset, void *kdata, size_t len); |
297 struct page *page, unsigned int offset, void *kdata, size_t len); |
372 dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan, |
298 dma_cookie_t dma_async_memcpy_pg_to_pg(struct dma_chan *chan, |
473 } |
410 } |
474 return DMA_IN_PROGRESS; |
411 return DMA_IN_PROGRESS; |
475 } |
412 } |
476 |
413 |
477 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie); |
414 enum dma_status dma_sync_wait(struct dma_chan *chan, dma_cookie_t cookie); |
|
415 #ifdef CONFIG_DMA_ENGINE |
|
416 enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx); |
|
417 void dma_issue_pending_all(void); |
|
418 #else |
|
419 static inline enum dma_status dma_wait_for_async_tx(struct dma_async_tx_descriptor *tx) |
|
420 { |
|
421 return DMA_SUCCESS; |
|
422 } |
|
423 static inline void dma_issue_pending_all(void) |
|
424 { |
|
425 do { } while (0); |
|
426 } |
|
427 #endif |
478 |
428 |
479 /* --- DMA device --- */ |
429 /* --- DMA device --- */ |
480 |
430 |
481 int dma_async_device_register(struct dma_device *device); |
431 int dma_async_device_register(struct dma_device *device); |
482 void dma_async_device_unregister(struct dma_device *device); |
432 void dma_async_device_unregister(struct dma_device *device); |
|
433 void dma_run_dependencies(struct dma_async_tx_descriptor *tx); |
|
434 struct dma_chan *dma_find_channel(enum dma_transaction_type tx_type); |
|
435 #define dma_request_channel(mask, x, y) __dma_request_channel(&(mask), x, y) |
|
436 struct dma_chan *__dma_request_channel(dma_cap_mask_t *mask, dma_filter_fn fn, void *fn_param); |
|
437 void dma_release_channel(struct dma_chan *chan); |
483 |
438 |
484 /* --- Helper iov-locking functions --- */ |
439 /* --- Helper iov-locking functions --- */ |
485 |
440 |
486 struct dma_page_list { |
441 struct dma_page_list { |
487 char __user *base_address; |
442 char __user *base_address; |