1 /***********************license start***************
2  * Author: Cavium Networks
3  *
4  * Contact: support@caviumnetworks.com
5  * This file is part of the OCTEON SDK
6  *
7  * Copyright (c) 2003-2008 Cavium Networks
8  *
9  * This file is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License, Version 2, as
11  * published by the Free Software Foundation.
12  *
13  * This file is distributed in the hope that it will be useful, but
14  * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty
15  * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or
16  * NONINFRINGEMENT.  See the GNU General Public License for more
17  * details.
18  *
19  * You should have received a copy of the GNU General Public License
20  * along with this file; if not, write to the Free Software
21  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
22  * or visit http://www.gnu.org/licenses/.
23  *
24  * This file may also be available under a different license from Cavium.
25  * Contact Cavium Networks for more information
26  ***********************license end**************************************/
27 
28 /*
29  *
30  * Support functions for managing command queues used for
31  * various hardware blocks.
32  *
33  * The common command queue infrastructure abstracts out the
34  * software necessary for adding to Octeon's chained queue
35  * structures. These structures are used for commands to the
36  * PKO, ZIP, DFA, RAID, and DMA engine blocks. Although each
37  * hardware unit takes commands and CSRs of different types,
38  * they all use basic linked command buffers to store the
39  * pending request. In general, users of the CVMX API don't
40  * call cvmx-cmd-queue functions directly. Instead the hardware
41  * unit specific wrapper should be used. The wrappers perform
42  * unit specific validation and CSR writes to submit the
43  * commands.
44  *
45  * Even though most software will never directly interact with
46  * cvmx-cmd-queue, knowledge of its internal working can help
47  * in diagnosing performance problems and help with debugging.
48  *
49  * Command queue pointers are stored in a global named block
50  * called "cvmx_cmd_queues". Except for the PKO queues, each
51  * hardware queue is stored in its own cache line to reduce SMP
52  * contention on spin locks. The PKO queues are stored such that
53  * every 16th queue is next to each other in memory. This scheme
54  * allows for queues being in separate cache lines when there
55  * are low number of queues per port. With 16 queues per port,
56  * the first queue for each port is in the same cache area. The
57  * second queues for each port are in another area, etc. This
58  * allows software to implement very efficient lockless PKO with
59  * 16 queues per port using a minimum of cache lines per core.
60  * All queues for a given core will be isolated in the same
61  * cache area.
62  *
63  * In addition to the memory pointer layout, cvmx-cmd-queue
64  * provides an optimized fair ll/sc locking mechanism for the
65  * queues. The lock uses a "ticket / now serving" model to
66  * maintain fair order on contended locks. In addition, it uses
67  * predicted locking time to limit cache contention. When a core
68  * know it must wait in line for a lock, it spins on the
69  * internal cycle counter to completely eliminate any causes of
70  * bus traffic.
71  *
72  */
73 
74 #ifndef __CVMX_CMD_QUEUE_H__
75 #define __CVMX_CMD_QUEUE_H__
76 
77 #include <linux/prefetch.h>
78 
79 #include <asm/compiler.h>
80 
81 #include <asm/octeon/cvmx-fpa.h>
82 /**
83  * By default we disable the max depth support. Most programs
84  * don't use it and it slows down the command queue processing
85  * significantly.
86  */
87 #ifndef CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH
88 #define CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH 0
89 #endif
90 
91 /**
92  * Enumeration representing all hardware blocks that use command
93  * queues. Each hardware block has up to 65536 sub identifiers for
94  * multiple command queues. Not all chips support all hardware
95  * units.
96  */
97 typedef enum {
98 	CVMX_CMD_QUEUE_PKO_BASE = 0x00000,
99 
100 #define CVMX_CMD_QUEUE_PKO(queue) \
101 	((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_PKO_BASE + (0xffff&(queue))))
102 
103 	CVMX_CMD_QUEUE_ZIP = 0x10000,
104 	CVMX_CMD_QUEUE_DFA = 0x20000,
105 	CVMX_CMD_QUEUE_RAID = 0x30000,
106 	CVMX_CMD_QUEUE_DMA_BASE = 0x40000,
107 
108 #define CVMX_CMD_QUEUE_DMA(queue) \
109 	((cvmx_cmd_queue_id_t)(CVMX_CMD_QUEUE_DMA_BASE + (0xffff&(queue))))
110 
111 	CVMX_CMD_QUEUE_END = 0x50000,
112 } cvmx_cmd_queue_id_t;
113 
114 /**
115  * Command write operations can fail if the command queue needs
116  * a new buffer and the associated FPA pool is empty. It can also
117  * fail if the number of queued command words reaches the maximum
118  * set at initialization.
119  */
120 typedef enum {
121 	CVMX_CMD_QUEUE_SUCCESS = 0,
122 	CVMX_CMD_QUEUE_NO_MEMORY = -1,
123 	CVMX_CMD_QUEUE_FULL = -2,
124 	CVMX_CMD_QUEUE_INVALID_PARAM = -3,
125 	CVMX_CMD_QUEUE_ALREADY_SETUP = -4,
126 } cvmx_cmd_queue_result_t;
127 
128 typedef struct {
129 	/* You have lock when this is your ticket */
130 	uint8_t now_serving;
131 	uint64_t unused1:24;
132 	/* Maximum outstanding command words */
133 	uint32_t max_depth;
134 	/* FPA pool buffers come from */
135 	uint64_t fpa_pool:3;
136 	/* Top of command buffer pointer shifted 7 */
137 	uint64_t base_ptr_div128:29;
138 	uint64_t unused2:6;
139 	/* FPA buffer size in 64bit words minus 1 */
140 	uint64_t pool_size_m1:13;
141 	/* Number of commands already used in buffer */
142 	uint64_t index:13;
143 } __cvmx_cmd_queue_state_t;
144 
145 /**
146  * This structure contains the global state of all command queues.
147  * It is stored in a bootmem named block and shared by all
148  * applications running on Octeon. Tickets are stored in a differnet
149  * cache line that queue information to reduce the contention on the
150  * ll/sc used to get a ticket. If this is not the case, the update
151  * of queue state causes the ll/sc to fail quite often.
152  */
153 typedef struct {
154 	uint64_t ticket[(CVMX_CMD_QUEUE_END >> 16) * 256];
155 	__cvmx_cmd_queue_state_t state[(CVMX_CMD_QUEUE_END >> 16) * 256];
156 } __cvmx_cmd_queue_all_state_t;
157 
158 /**
159  * Initialize a command queue for use. The initial FPA buffer is
160  * allocated and the hardware unit is configured to point to the
161  * new command queue.
162  *
163  * @queue_id:  Hardware command queue to initialize.
164  * @max_depth: Maximum outstanding commands that can be queued.
165  * @fpa_pool:  FPA pool the command queues should come from.
166  * @pool_size: Size of each buffer in the FPA pool (bytes)
167  *
168  * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
169  */
170 cvmx_cmd_queue_result_t cvmx_cmd_queue_initialize(cvmx_cmd_queue_id_t queue_id,
171 						  int max_depth, int fpa_pool,
172 						  int pool_size);
173 
174 /**
175  * Shutdown a queue a free it's command buffers to the FPA. The
176  * hardware connected to the queue must be stopped before this
177  * function is called.
178  *
179  * @queue_id: Queue to shutdown
180  *
181  * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
182  */
183 cvmx_cmd_queue_result_t cvmx_cmd_queue_shutdown(cvmx_cmd_queue_id_t queue_id);
184 
185 /**
186  * Return the number of command words pending in the queue. This
187  * function may be relatively slow for some hardware units.
188  *
189  * @queue_id: Hardware command queue to query
190  *
191  * Returns Number of outstanding commands
192  */
193 int cvmx_cmd_queue_length(cvmx_cmd_queue_id_t queue_id);
194 
195 /**
196  * Return the command buffer to be written to. The purpose of this
197  * function is to allow CVMX routine access t othe low level buffer
198  * for initial hardware setup. User applications should not call this
199  * function directly.
200  *
201  * @queue_id: Command queue to query
202  *
203  * Returns Command buffer or NULL on failure
204  */
205 void *cvmx_cmd_queue_buffer(cvmx_cmd_queue_id_t queue_id);
206 
207 /**
208  * Get the index into the state arrays for the supplied queue id.
209  *
210  * @queue_id: Queue ID to get an index for
211  *
212  * Returns Index into the state arrays
213  */
__cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)214 static inline int __cvmx_cmd_queue_get_index(cvmx_cmd_queue_id_t queue_id)
215 {
216 	/*
217 	 * Warning: This code currently only works with devices that
218 	 * have 256 queues or less. Devices with more than 16 queues
219 	 * are laid out in memory to allow cores quick access to
220 	 * every 16th queue. This reduces cache thrashing when you are
221 	 * running 16 queues per port to support lockless operation.
222 	 */
223 	int unit = queue_id >> 16;
224 	int q = (queue_id >> 4) & 0xf;
225 	int core = queue_id & 0xf;
226 	return unit * 256 + core * 16 + q;
227 }
228 
229 /**
230  * Lock the supplied queue so nobody else is updating it at the same
231  * time as us.
232  *
233  * @queue_id: Queue ID to lock
234  * @qptr:     Pointer to the queue's global state
235  */
__cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id,__cvmx_cmd_queue_state_t * qptr)236 static inline void __cvmx_cmd_queue_lock(cvmx_cmd_queue_id_t queue_id,
237 					 __cvmx_cmd_queue_state_t *qptr)
238 {
239 	extern __cvmx_cmd_queue_all_state_t
240 	    *__cvmx_cmd_queue_state_ptr;
241 	int tmp;
242 	int my_ticket;
243 	prefetch(qptr);
244 	asm volatile (
245 		".set push\n"
246 		".set noreorder\n"
247 		"1:\n"
248 		/* Atomic add one to ticket_ptr */
249 		"ll	%[my_ticket], %[ticket_ptr]\n"
250 		/* and store the original value */
251 		"li	%[ticket], 1\n"
252 		/* in my_ticket */
253 		"baddu	%[ticket], %[my_ticket]\n"
254 		"sc	%[ticket], %[ticket_ptr]\n"
255 		"beqz	%[ticket], 1b\n"
256 		" nop\n"
257 		/* Load the current now_serving ticket */
258 		"lbu	%[ticket], %[now_serving]\n"
259 		"2:\n"
260 		/* Jump out if now_serving == my_ticket */
261 		"beq	%[ticket], %[my_ticket], 4f\n"
262 		/* Find out how many tickets are in front of me */
263 		" subu	 %[ticket], %[my_ticket], %[ticket]\n"
264 		/* Use tickets in front of me minus one to delay */
265 		"subu  %[ticket], 1\n"
266 		/* Delay will be ((tickets in front)-1)*32 loops */
267 		"cins	%[ticket], %[ticket], 5, 7\n"
268 		"3:\n"
269 		/* Loop here until our ticket might be up */
270 		"bnez	%[ticket], 3b\n"
271 		" subu	%[ticket], 1\n"
272 		/* Jump back up to check out ticket again */
273 		"b	2b\n"
274 		/* Load the current now_serving ticket */
275 		" lbu	%[ticket], %[now_serving]\n"
276 		"4:\n"
277 		".set pop\n" :
278 		[ticket_ptr] "=" GCC_OFF_SMALL_ASM()(__cvmx_cmd_queue_state_ptr->ticket[__cvmx_cmd_queue_get_index(queue_id)]),
279 		[now_serving] "=m"(qptr->now_serving), [ticket] "=r"(tmp),
280 		[my_ticket] "=r"(my_ticket)
281 	    );
282 }
283 
284 /**
285  * Unlock the queue, flushing all writes.
286  *
287  * @qptr:   Queue to unlock
288  */
__cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t * qptr)289 static inline void __cvmx_cmd_queue_unlock(__cvmx_cmd_queue_state_t *qptr)
290 {
291 	qptr->now_serving++;
292 	CVMX_SYNCWS;
293 }
294 
295 /**
296  * Get the queue state structure for the given queue id
297  *
298  * @queue_id: Queue id to get
299  *
300  * Returns Queue structure or NULL on failure
301  */
302 static inline __cvmx_cmd_queue_state_t
__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)303     *__cvmx_cmd_queue_get_state(cvmx_cmd_queue_id_t queue_id)
304 {
305 	extern __cvmx_cmd_queue_all_state_t
306 	    *__cvmx_cmd_queue_state_ptr;
307 	return &__cvmx_cmd_queue_state_ptr->
308 	    state[__cvmx_cmd_queue_get_index(queue_id)];
309 }
310 
311 /**
312  * Write an arbitrary number of command words to a command queue.
313  * This is a generic function; the fixed number of command word
314  * functions yield higher performance.
315  *
316  * @queue_id:  Hardware command queue to write to
317  * @use_locking:
318  *		    Use internal locking to ensure exclusive access for queue
319  *		    updates. If you don't use this locking you must ensure
320  *		    exclusivity some other way. Locking is strongly recommended.
321  * @cmd_count: Number of command words to write
322  * @cmds:      Array of commands to write
323  *
324  * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
325  */
cvmx_cmd_queue_write(cvmx_cmd_queue_id_t queue_id,int use_locking,int cmd_count,uint64_t * cmds)326 static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write(cvmx_cmd_queue_id_t
327 							   queue_id,
328 							   int use_locking,
329 							   int cmd_count,
330 							   uint64_t *cmds)
331 {
332 	__cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
333 
334 	/* Make sure nobody else is updating the same queue */
335 	if (likely(use_locking))
336 		__cvmx_cmd_queue_lock(queue_id, qptr);
337 
338 	/*
339 	 * If a max queue length was specified then make sure we don't
340 	 * exceed it. If any part of the command would be below the
341 	 * limit we allow it.
342 	 */
343 	if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
344 		if (unlikely
345 		    (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
346 			if (likely(use_locking))
347 				__cvmx_cmd_queue_unlock(qptr);
348 			return CVMX_CMD_QUEUE_FULL;
349 		}
350 	}
351 
352 	/*
353 	 * Normally there is plenty of room in the current buffer for
354 	 * the command.
355 	 */
356 	if (likely(qptr->index + cmd_count < qptr->pool_size_m1)) {
357 		uint64_t *ptr =
358 		    (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
359 						  base_ptr_div128 << 7);
360 		ptr += qptr->index;
361 		qptr->index += cmd_count;
362 		while (cmd_count--)
363 			*ptr++ = *cmds++;
364 	} else {
365 		uint64_t *ptr;
366 		int count;
367 		/*
368 		 * We need a new command buffer. Fail if there isn't
369 		 * one available.
370 		 */
371 		uint64_t *new_buffer =
372 		    (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
373 		if (unlikely(new_buffer == NULL)) {
374 			if (likely(use_locking))
375 				__cvmx_cmd_queue_unlock(qptr);
376 			return CVMX_CMD_QUEUE_NO_MEMORY;
377 		}
378 		ptr =
379 		    (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
380 						  base_ptr_div128 << 7);
381 		/*
382 		 * Figure out how many command words will fit in this
383 		 * buffer. One location will be needed for the next
384 		 * buffer pointer.
385 		 */
386 		count = qptr->pool_size_m1 - qptr->index;
387 		ptr += qptr->index;
388 		cmd_count -= count;
389 		while (count--)
390 			*ptr++ = *cmds++;
391 		*ptr = cvmx_ptr_to_phys(new_buffer);
392 		/*
393 		 * The current buffer is full and has a link to the
394 		 * next buffer. Time to write the rest of the commands
395 		 * into the new buffer.
396 		 */
397 		qptr->base_ptr_div128 = *ptr >> 7;
398 		qptr->index = cmd_count;
399 		ptr = new_buffer;
400 		while (cmd_count--)
401 			*ptr++ = *cmds++;
402 	}
403 
404 	/* All updates are complete. Release the lock and return */
405 	if (likely(use_locking))
406 		__cvmx_cmd_queue_unlock(qptr);
407 	return CVMX_CMD_QUEUE_SUCCESS;
408 }
409 
410 /**
411  * Simple function to write two command words to a command
412  * queue.
413  *
414  * @queue_id: Hardware command queue to write to
415  * @use_locking:
416  *		   Use internal locking to ensure exclusive access for queue
417  *		   updates. If you don't use this locking you must ensure
418  *		   exclusivity some other way. Locking is strongly recommended.
419  * @cmd1:     Command
420  * @cmd2:     Command
421  *
422  * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
423  */
cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t queue_id,int use_locking,uint64_t cmd1,uint64_t cmd2)424 static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write2(cvmx_cmd_queue_id_t
425 							    queue_id,
426 							    int use_locking,
427 							    uint64_t cmd1,
428 							    uint64_t cmd2)
429 {
430 	__cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
431 
432 	/* Make sure nobody else is updating the same queue */
433 	if (likely(use_locking))
434 		__cvmx_cmd_queue_lock(queue_id, qptr);
435 
436 	/*
437 	 * If a max queue length was specified then make sure we don't
438 	 * exceed it. If any part of the command would be below the
439 	 * limit we allow it.
440 	 */
441 	if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
442 		if (unlikely
443 		    (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
444 			if (likely(use_locking))
445 				__cvmx_cmd_queue_unlock(qptr);
446 			return CVMX_CMD_QUEUE_FULL;
447 		}
448 	}
449 
450 	/*
451 	 * Normally there is plenty of room in the current buffer for
452 	 * the command.
453 	 */
454 	if (likely(qptr->index + 2 < qptr->pool_size_m1)) {
455 		uint64_t *ptr =
456 		    (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
457 						  base_ptr_div128 << 7);
458 		ptr += qptr->index;
459 		qptr->index += 2;
460 		ptr[0] = cmd1;
461 		ptr[1] = cmd2;
462 	} else {
463 		uint64_t *ptr;
464 		/*
465 		 * Figure out how many command words will fit in this
466 		 * buffer. One location will be needed for the next
467 		 * buffer pointer.
468 		 */
469 		int count = qptr->pool_size_m1 - qptr->index;
470 		/*
471 		 * We need a new command buffer. Fail if there isn't
472 		 * one available.
473 		 */
474 		uint64_t *new_buffer =
475 		    (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
476 		if (unlikely(new_buffer == NULL)) {
477 			if (likely(use_locking))
478 				__cvmx_cmd_queue_unlock(qptr);
479 			return CVMX_CMD_QUEUE_NO_MEMORY;
480 		}
481 		count--;
482 		ptr =
483 		    (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
484 						  base_ptr_div128 << 7);
485 		ptr += qptr->index;
486 		*ptr++ = cmd1;
487 		if (likely(count))
488 			*ptr++ = cmd2;
489 		*ptr = cvmx_ptr_to_phys(new_buffer);
490 		/*
491 		 * The current buffer is full and has a link to the
492 		 * next buffer. Time to write the rest of the commands
493 		 * into the new buffer.
494 		 */
495 		qptr->base_ptr_div128 = *ptr >> 7;
496 		qptr->index = 0;
497 		if (unlikely(count == 0)) {
498 			qptr->index = 1;
499 			new_buffer[0] = cmd2;
500 		}
501 	}
502 
503 	/* All updates are complete. Release the lock and return */
504 	if (likely(use_locking))
505 		__cvmx_cmd_queue_unlock(qptr);
506 	return CVMX_CMD_QUEUE_SUCCESS;
507 }
508 
509 /**
510  * Simple function to write three command words to a command
511  * queue.
512  *
513  * @queue_id: Hardware command queue to write to
514  * @use_locking:
515  *		   Use internal locking to ensure exclusive access for queue
516  *		   updates. If you don't use this locking you must ensure
517  *		   exclusivity some other way. Locking is strongly recommended.
518  * @cmd1:     Command
519  * @cmd2:     Command
520  * @cmd3:     Command
521  *
522  * Returns CVMX_CMD_QUEUE_SUCCESS or a failure code
523  */
cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t queue_id,int use_locking,uint64_t cmd1,uint64_t cmd2,uint64_t cmd3)524 static inline cvmx_cmd_queue_result_t cvmx_cmd_queue_write3(cvmx_cmd_queue_id_t
525 							    queue_id,
526 							    int use_locking,
527 							    uint64_t cmd1,
528 							    uint64_t cmd2,
529 							    uint64_t cmd3)
530 {
531 	__cvmx_cmd_queue_state_t *qptr = __cvmx_cmd_queue_get_state(queue_id);
532 
533 	/* Make sure nobody else is updating the same queue */
534 	if (likely(use_locking))
535 		__cvmx_cmd_queue_lock(queue_id, qptr);
536 
537 	/*
538 	 * If a max queue length was specified then make sure we don't
539 	 * exceed it. If any part of the command would be below the
540 	 * limit we allow it.
541 	 */
542 	if (CVMX_CMD_QUEUE_ENABLE_MAX_DEPTH && unlikely(qptr->max_depth)) {
543 		if (unlikely
544 		    (cvmx_cmd_queue_length(queue_id) > (int)qptr->max_depth)) {
545 			if (likely(use_locking))
546 				__cvmx_cmd_queue_unlock(qptr);
547 			return CVMX_CMD_QUEUE_FULL;
548 		}
549 	}
550 
551 	/*
552 	 * Normally there is plenty of room in the current buffer for
553 	 * the command.
554 	 */
555 	if (likely(qptr->index + 3 < qptr->pool_size_m1)) {
556 		uint64_t *ptr =
557 		    (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
558 						  base_ptr_div128 << 7);
559 		ptr += qptr->index;
560 		qptr->index += 3;
561 		ptr[0] = cmd1;
562 		ptr[1] = cmd2;
563 		ptr[2] = cmd3;
564 	} else {
565 		uint64_t *ptr;
566 		/*
567 		 * Figure out how many command words will fit in this
568 		 * buffer. One location will be needed for the next
569 		 * buffer pointer
570 		 */
571 		int count = qptr->pool_size_m1 - qptr->index;
572 		/*
573 		 * We need a new command buffer. Fail if there isn't
574 		 * one available
575 		 */
576 		uint64_t *new_buffer =
577 		    (uint64_t *) cvmx_fpa_alloc(qptr->fpa_pool);
578 		if (unlikely(new_buffer == NULL)) {
579 			if (likely(use_locking))
580 				__cvmx_cmd_queue_unlock(qptr);
581 			return CVMX_CMD_QUEUE_NO_MEMORY;
582 		}
583 		count--;
584 		ptr =
585 		    (uint64_t *) cvmx_phys_to_ptr((uint64_t) qptr->
586 						  base_ptr_div128 << 7);
587 		ptr += qptr->index;
588 		*ptr++ = cmd1;
589 		if (count) {
590 			*ptr++ = cmd2;
591 			if (count > 1)
592 				*ptr++ = cmd3;
593 		}
594 		*ptr = cvmx_ptr_to_phys(new_buffer);
595 		/*
596 		 * The current buffer is full and has a link to the
597 		 * next buffer. Time to write the rest of the commands
598 		 * into the new buffer.
599 		 */
600 		qptr->base_ptr_div128 = *ptr >> 7;
601 		qptr->index = 0;
602 		ptr = new_buffer;
603 		if (count == 0) {
604 			*ptr++ = cmd2;
605 			qptr->index++;
606 		}
607 		if (count < 2) {
608 			*ptr++ = cmd3;
609 			qptr->index++;
610 		}
611 	}
612 
613 	/* All updates are complete. Release the lock and return */
614 	if (likely(use_locking))
615 		__cvmx_cmd_queue_unlock(qptr);
616 	return CVMX_CMD_QUEUE_SUCCESS;
617 }
618 
619 #endif /* __CVMX_CMD_QUEUE_H__ */
620