xref: /openbmc/qemu/tests/tcg/hexagon/scatter_gather.c (revision c3679385)

/*
 *  Copyright(c) 2019-2023 Qualcomm Innovation Center, Inc. All Rights Reserved.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License as published by
 *  the Free Software Foundation; either version 2 of the License, or
 *  (at your option) any later version.
 *
 *  This program is distributed in the hope that it will be useful,
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *  GNU General Public License for more details.
 *
 *  You should have received a copy of the GNU General Public License
 *  along with this program; if not, see <http://www.gnu.org/licenses/>.
 */

/*
 * This example tests the HVX scatter/gather instructions
 *
 * See section 5.13 of the V68 HVX Programmer's Reference
 *
 * There are 3 main classes operations
 *     _16                 16-bit elements and 16-bit offsets
 *     _32                 32-bit elements and 32-bit offsets
 *     _16_32              16-bit elements and 32-bit offsets
 *
 * There are also masked and accumulate versions
 */

#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include <inttypes.h>

typedef long HVX_Vector       __attribute__((__vector_size__(128)))
                              __attribute__((aligned(128)));
typedef long HVX_VectorPair   __attribute__((__vector_size__(256)))
                              __attribute__((aligned(128)));
typedef long HVX_VectorPred   __attribute__((__vector_size__(128)))
                              __attribute__((aligned(128)));

int err;

/* define the number of rows/cols in a square matrix */
#define MATRIX_SIZE 64

/* define the size of the scatter buffer */
#define SCATTER_BUFFER_SIZE (MATRIX_SIZE * MATRIX_SIZE)

/* fake vtcm - put buffers together and force alignment */
static struct {
    unsigned short vscatter16[SCATTER_BUFFER_SIZE];
    unsigned short vgather16[MATRIX_SIZE];
    unsigned int   vscatter32[SCATTER_BUFFER_SIZE];
    unsigned int   vgather32[MATRIX_SIZE];
    unsigned short vscatter16_32[SCATTER_BUFFER_SIZE];
    unsigned short vgather16_32[MATRIX_SIZE];
} vtcm __attribute__((aligned(0x10000)));

/* declare the arrays of reference values */
unsigned short vscatter16_ref[SCATTER_BUFFER_SIZE];
unsigned short vgather16_ref[MATRIX_SIZE];
unsigned int   vscatter32_ref[SCATTER_BUFFER_SIZE];
unsigned int   vgather32_ref[MATRIX_SIZE];
unsigned short vscatter16_32_ref[SCATTER_BUFFER_SIZE];
unsigned short vgather16_32_ref[MATRIX_SIZE];

/* declare the arrays of offsets */
unsigned short half_offsets[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned int   word_offsets[MATRIX_SIZE] __attribute__((aligned(128)));

/* declare the arrays of values */
unsigned short half_values[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned short half_values_acc[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned short half_values_masked[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned int   word_values[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned int   word_values_acc[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned int   word_values_masked[MATRIX_SIZE] __attribute__((aligned(128)));

/* declare the arrays of predicates */
unsigned short half_predicates[MATRIX_SIZE] __attribute__((aligned(128)));
unsigned int   word_predicates[MATRIX_SIZE] __attribute__((aligned(128)));

/* make this big enough for all the operations */
const size_t region_len = sizeof(vtcm);

/* optionally add sync instructions */
#define SYNC_VECTOR 1

static void sync_scatter(void *addr)
{
#if SYNC_VECTOR
    /*
     * Do the scatter release followed by a dummy load to complete the
     * synchronization.  Normally the dummy load would be deferred as
     * long as possible to minimize stalls.
     */
    asm volatile("vmem(%0 + #0):scatter_release\n" : : "r"(addr));
    /* use volatile to force the load */
    volatile HVX_Vector vDummy = *(HVX_Vector *)addr; vDummy = vDummy;
#endif
}

static void sync_gather(void *addr)
{
#if SYNC_VECTOR
    /* use volatile to force the load */
    volatile HVX_Vector vDummy = *(HVX_Vector *)addr; vDummy = vDummy;
#endif
}

/* optionally print the results */
#define PRINT_DATA 0

#define FILL_CHAR       '.'

/* fill vtcm scratch with ee */
void prefill_vtcm_scratch(void)
{
    memset(&vtcm, FILL_CHAR, sizeof(vtcm));
}

/* create byte offsets to be a diagonal of the matrix with 16 bit elements */
void create_offsets_values_preds_16(void)
{
    unsigned short half_element = 0;
    unsigned short half_element_masked = 0;
    char letter = 'A';
    char letter_masked = '@';

    for (int i = 0; i < MATRIX_SIZE; i++) {
        half_offsets[i] = i * (2 * MATRIX_SIZE + 2);

        half_element = 0;
        half_element_masked = 0;
        for (int j = 0; j < 2; j++) {
            half_element |= letter << j * 8;
            half_element_masked |= letter_masked << j * 8;
        }

        half_values[i] = half_element;
        half_values_acc[i] = ((i % 10) << 8) + (i % 10);
        half_values_masked[i] = half_element_masked;

        letter++;
        /* reset to 'A' */
        if (letter == 'M') {
            letter = 'A';
        }

        half_predicates[i] = (i % 3 == 0 || i % 5 == 0) ? ~0 : 0;
    }
}

/* create byte offsets to be a diagonal of the matrix with 32 bit elements */
void create_offsets_values_preds_32(void)
{
    unsigned int word_element = 0;
    unsigned int word_element_masked = 0;
    char letter = 'A';
    char letter_masked = '&';

    for (int i = 0; i < MATRIX_SIZE; i++) {
        word_offsets[i] = i * (4 * MATRIX_SIZE + 4);

        word_element = 0;
        word_element_masked = 0;
        for (int j = 0; j < 4; j++) {
            word_element |= letter << j * 8;
            word_element_masked |= letter_masked << j * 8;
        }

        word_values[i] = word_element;
        word_values_acc[i] = ((i % 10) << 8) + (i % 10);
        word_values_masked[i] = word_element_masked;

        letter++;
        /* reset to 'A' */
        if (letter == 'M') {
            letter = 'A';
        }

        word_predicates[i] = (i % 4 == 0 || i % 7 == 0) ? ~0 : 0;
    }
}

/*
 * create byte offsets to be a diagonal of the matrix with 16 bit elements
 * and 32 bit offsets
 */
void create_offsets_values_preds_16_32(void)
{
    unsigned short half_element = 0;
    unsigned short half_element_masked = 0;
    char letter = 'D';
    char letter_masked = '$';

    for (int i = 0; i < MATRIX_SIZE; i++) {
        word_offsets[i] = i * (2 * MATRIX_SIZE + 2);

        half_element = 0;
        half_element_masked = 0;
        for (int j = 0; j < 2; j++) {
            half_element |= letter << j * 8;
            half_element_masked |= letter_masked << j * 8;
        }

        half_values[i] = half_element;
        half_values_acc[i] = ((i % 10) << 8) + (i % 10);
        half_values_masked[i] = half_element_masked;

        letter++;
        /* reset to 'A' */
        if (letter == 'P') {
            letter = 'D';
        }

        half_predicates[i] = (i % 2 == 0 || i % 13 == 0) ? ~0 : 0;
    }
}

/* scatter the 16 bit elements using HVX */
void vector_scatter_16(void)
{
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%3 + #0)\n\t"
         "vscatter(%0, m0, v0.h).h = v1\n\t"
         : : "r"(vtcm.vscatter16), "r"(region_len),
             "r"(half_offsets), "r"(half_values)
         : "m0", "v0", "v1", "memory");

    sync_scatter(vtcm.vscatter16);
}

/* scatter-accumulate the 16 bit elements using HVX */
void vector_scatter_16_acc(void)
{
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%3 + #0)\n\t"
         "vscatter(%0, m0, v0.h).h += v1\n\t"
         : : "r"(vtcm.vscatter16), "r"(region_len),
             "r"(half_offsets), "r"(half_values_acc)
         : "m0", "v0", "v1", "memory");

    sync_scatter(vtcm.vscatter16);
}

/* masked scatter the 16 bit elements using HVX */
void vector_scatter_16_masked(void)
{
    asm ("r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "v1 = vmem(%4 + #0)\n\t"
         "if (q0) vscatter(%1, m0, v0.h).h = v1\n\t"
         : : "r"(half_predicates), "r"(vtcm.vscatter16), "r"(region_len),
             "r"(half_offsets), "r"(half_values_masked)
         : "r1", "q0", "m0", "q0", "v0", "v1", "memory");

    sync_scatter(vtcm.vscatter16);
}

/* scatter the 32 bit elements using HVX */
void vector_scatter_32(void)
{
    HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
    HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
    HVX_Vector *valueslo = (HVX_Vector *)word_values;
    HVX_Vector *valueshi = (HVX_Vector *)&word_values[MATRIX_SIZE / 2];

    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%3 + #0)\n\t"
         "vscatter(%0, m0, v0.w).w = v1\n\t"
         : : "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetslo), "r"(valueslo)
         : "m0", "v0", "v1", "memory");
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%3 + #0)\n\t"
         "vscatter(%0, m0, v0.w).w = v1\n\t"
         : : "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetshi), "r"(valueshi)
         : "m0", "v0", "v1", "memory");

    sync_scatter(vtcm.vscatter32);
}

/* scatter-accumulate the 32 bit elements using HVX */
void vector_scatter_32_acc(void)
{
    HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
    HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
    HVX_Vector *valueslo = (HVX_Vector *)word_values_acc;
    HVX_Vector *valueshi = (HVX_Vector *)&word_values_acc[MATRIX_SIZE / 2];

    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%3 + #0)\n\t"
         "vscatter(%0, m0, v0.w).w += v1\n\t"
         : : "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetslo), "r"(valueslo)
         : "m0", "v0", "v1", "memory");
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%3 + #0)\n\t"
         "vscatter(%0, m0, v0.w).w += v1\n\t"
         : : "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetshi), "r"(valueshi)
         : "m0", "v0", "v1", "memory");

    sync_scatter(vtcm.vscatter32);
}

/* masked scatter the 32 bit elements using HVX */
void vector_scatter_32_masked(void)
{
    HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
    HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
    HVX_Vector *valueslo = (HVX_Vector *)word_values_masked;
    HVX_Vector *valueshi = (HVX_Vector *)&word_values_masked[MATRIX_SIZE / 2];
    HVX_Vector *predslo = (HVX_Vector *)word_predicates;
    HVX_Vector *predshi = (HVX_Vector *)&word_predicates[MATRIX_SIZE / 2];

    asm ("r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "v1 = vmem(%4 + #0)\n\t"
         "if (q0) vscatter(%1, m0, v0.w).w = v1\n\t"
         : : "r"(predslo), "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetslo), "r"(valueslo)
         : "r1", "q0", "m0", "q0", "v0", "v1", "memory");
    asm ("r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "v1 = vmem(%4 + #0)\n\t"
         "if (q0) vscatter(%1, m0, v0.w).w = v1\n\t"
         : : "r"(predshi), "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetshi), "r"(valueshi)
         : "r1", "q0", "m0", "q0", "v0", "v1", "memory");

    sync_scatter(vtcm.vscatter32);
}

/* scatter the 16 bit elements with 32 bit offsets using HVX */
void vector_scatter_16_32(void)
{
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%2 + #1)\n\t"
         "v2 = vmem(%3 + #0)\n\t"
         "v2.h = vshuff(v2.h)\n\t"  /* shuffle the values for the scatter */
         "vscatter(%0, m0, v1:0.w).h = v2\n\t"
         : : "r"(vtcm.vscatter16_32), "r"(region_len),
             "r"(word_offsets), "r"(half_values)
         : "m0", "v0", "v1", "v2", "memory");

    sync_scatter(vtcm.vscatter16_32);
}

/* scatter-accumulate the 16 bit elements with 32 bit offsets using HVX */
void vector_scatter_16_32_acc(void)
{
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%2 + #1)\n\t"
         "v2 = vmem(%3 + #0)\n\t" \
         "v2.h = vshuff(v2.h)\n\t"  /* shuffle the values for the scatter */
         "vscatter(%0, m0, v1:0.w).h += v2\n\t"
         : : "r"(vtcm.vscatter16_32), "r"(region_len),
             "r"(word_offsets), "r"(half_values_acc)
         : "m0", "v0", "v1", "v2", "memory");

    sync_scatter(vtcm.vscatter16_32);
}

/* masked scatter the 16 bit elements with 32 bit offsets using HVX */
void vector_scatter_16_32_masked(void)
{
    asm ("r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "v0.h = vshuff(v0.h)\n\t"  /* shuffle the predicates */
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "v1 = vmem(%3 + #1)\n\t"
         "v2 = vmem(%4 + #0)\n\t" \
         "v2.h = vshuff(v2.h)\n\t"  /* shuffle the values for the scatter */
         "if (q0) vscatter(%1, m0, v1:0.w).h = v2\n\t"
         : : "r"(half_predicates), "r"(vtcm.vscatter16_32), "r"(region_len),
             "r"(word_offsets), "r"(half_values_masked)
         : "r1", "q0", "m0", "v0", "v1", "v2", "memory");

    sync_scatter(vtcm.vscatter16_32);
}

/* gather the elements from the scatter16 buffer using HVX */
void vector_gather_16(void)
{
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "{ vtmp.h = vgather(%0, m0, v0.h).h\n\t"
         "  vmem(%3 + #0) = vtmp.new }\n\t"
         : : "r"(vtcm.vscatter16), "r"(region_len),
             "r"(half_offsets), "r"(vtcm.vgather16)
         : "m0", "v0", "memory");

    sync_gather(vtcm.vgather16);
}

static unsigned short gather_16_masked_init(void)
{
    char letter = '?';
    return letter | (letter << 8);
}

/* masked gather the elements from the scatter16 buffer using HVX */
void vector_gather_16_masked(void)
{
    unsigned short init = gather_16_masked_init();

    asm ("v0.h = vsplat(%5)\n\t"
         "vmem(%4 + #0) = v0\n\t"  /* initialize the write area */
         "r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "{ if (q0) vtmp.h = vgather(%1, m0, v0.h).h\n\t"
         "  vmem(%4 + #0) = vtmp.new }\n\t"
         : : "r"(half_predicates), "r"(vtcm.vscatter16), "r"(region_len),
             "r"(half_offsets), "r"(vtcm.vgather16), "r"(init)
         : "r1", "q0", "m0", "v0", "memory");

    sync_gather(vtcm.vgather16);
}

/* gather the elements from the scatter32 buffer using HVX */
void vector_gather_32(void)
{
    HVX_Vector *vgatherlo = (HVX_Vector *)vtcm.vgather32;
    HVX_Vector *vgatherhi = (HVX_Vector *)&vtcm.vgather32[MATRIX_SIZE / 2];
    HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
    HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];

    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "{ vtmp.w = vgather(%0, m0, v0.w).w\n\t"
         "  vmem(%3 + #0) = vtmp.new }\n\t"
         : : "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetslo), "r"(vgatherlo)
         : "m0", "v0", "memory");
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "{ vtmp.w = vgather(%0, m0, v0.w).w\n\t"
         "  vmem(%3 + #0) = vtmp.new }\n\t"
         : : "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetshi), "r"(vgatherhi)
         : "m0", "v0", "memory");

    sync_gather(vgatherlo);
    sync_gather(vgatherhi);
}

static unsigned int gather_32_masked_init(void)
{
    char letter = '?';
    return letter | (letter << 8) | (letter << 16) | (letter << 24);
}

/* masked gather the elements from the scatter32 buffer using HVX */
void vector_gather_32_masked(void)
{
    unsigned int init = gather_32_masked_init();
    HVX_Vector *vgatherlo = (HVX_Vector *)vtcm.vgather32;
    HVX_Vector *vgatherhi = (HVX_Vector *)&vtcm.vgather32[MATRIX_SIZE / 2];
    HVX_Vector *offsetslo = (HVX_Vector *)word_offsets;
    HVX_Vector *offsetshi = (HVX_Vector *)&word_offsets[MATRIX_SIZE / 2];
    HVX_Vector *predslo = (HVX_Vector *)word_predicates;
    HVX_Vector *predshi = (HVX_Vector *)&word_predicates[MATRIX_SIZE / 2];

    asm ("v0.h = vsplat(%5)\n\t"
         "vmem(%4 + #0) = v0\n\t"  /* initialize the write area */
         "r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "{ if (q0) vtmp.w = vgather(%1, m0, v0.w).w\n\t"
         "  vmem(%4 + #0) = vtmp.new }\n\t"
         : : "r"(predslo), "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetslo), "r"(vgatherlo), "r"(init)
         : "r1", "q0", "m0", "v0", "memory");
    asm ("v0.h = vsplat(%5)\n\t"
         "vmem(%4 + #0) = v0\n\t"  /* initialize the write area */
         "r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "{ if (q0) vtmp.w = vgather(%1, m0, v0.w).w\n\t"
         "  vmem(%4 + #0) = vtmp.new }\n\t"
         : : "r"(predshi), "r"(vtcm.vscatter32), "r"(region_len),
             "r"(offsetshi), "r"(vgatherhi), "r"(init)
         : "r1", "q0", "m0", "v0", "memory");

    sync_gather(vgatherlo);
    sync_gather(vgatherhi);
}

/* gather the elements from the scatter16_32 buffer using HVX */
void vector_gather_16_32(void)
{
    asm ("m0 = %1\n\t"
         "v0 = vmem(%2 + #0)\n\t"
         "v1 = vmem(%2 + #1)\n\t"
         "{ vtmp.h = vgather(%0, m0, v1:0.w).h\n\t"
         "  vmem(%3 + #0) = vtmp.new }\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "v0.h = vdeal(v0.h)\n\t"  /* deal the elements to get the order back */
         "vmem(%3 + #0) = v0\n\t"
         : : "r"(vtcm.vscatter16_32), "r"(region_len),
             "r"(word_offsets), "r"(vtcm.vgather16_32)
         : "m0", "v0", "v1", "memory");

    sync_gather(vtcm.vgather16_32);
}

/* masked gather the elements from the scatter16_32 buffer using HVX */
void vector_gather_16_32_masked(void)
{
    unsigned short init = gather_16_masked_init();

    asm ("v0.h = vsplat(%5)\n\t"
         "vmem(%4 + #0) = v0\n\t"  /* initialize the write area */
         "r1 = #-1\n\t"
         "v0 = vmem(%0 + #0)\n\t"
         "v0.h = vshuff(v0.h)\n\t"  /* shuffle the predicates */
         "q0 = vand(v0, r1)\n\t"
         "m0 = %2\n\t"
         "v0 = vmem(%3 + #0)\n\t"
         "v1 = vmem(%3 + #1)\n\t"
         "{ if (q0) vtmp.h = vgather(%1, m0, v1:0.w).h\n\t"
         "  vmem(%4 + #0) = vtmp.new }\n\t"
         "v0 = vmem(%4 + #0)\n\t"
         "v0.h = vdeal(v0.h)\n\t"  /* deal the elements to get the order back */
         "vmem(%4 + #0) = v0\n\t"
         : : "r"(half_predicates), "r"(vtcm.vscatter16_32), "r"(region_len),
             "r"(word_offsets), "r"(vtcm.vgather16_32), "r"(init)
         : "r1", "q0", "m0", "v0", "v1", "memory");

    sync_gather(vtcm.vgather16_32);
}

static void check_buffer(const char *name, void *c, void *r, size_t size)
{
    char *check = (char *)c;
    char *ref = (char *)r;
    for (int i = 0; i < size; i++) {
        if (check[i] != ref[i]) {
            printf("ERROR %s [%d]: 0x%x (%c) != 0x%x (%c)\n", name, i,
                   check[i], check[i], ref[i], ref[i]);
            err++;
        }
    }
}

/*
 * These scalar functions are the C equivalents of the vector functions that
 * use HVX
 */

/* scatter the 16 bit elements using C */
void scalar_scatter_16(unsigned short *vscatter16)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vscatter16[half_offsets[i] / 2] = half_values[i];
    }
}

void check_scatter_16()
{
    memset(vscatter16_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned short));
    scalar_scatter_16(vscatter16_ref);
    check_buffer(__func__, vtcm.vscatter16, vscatter16_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}

/* scatter the 16 bit elements using C */
void scalar_scatter_16_acc(unsigned short *vscatter16)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vscatter16[half_offsets[i] / 2] += half_values_acc[i];
    }
}

/* scatter-accumulate the 16 bit elements using C */
void check_scatter_16_acc()
{
    memset(vscatter16_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned short));
    scalar_scatter_16(vscatter16_ref);
    scalar_scatter_16_acc(vscatter16_ref);
    check_buffer(__func__, vtcm.vscatter16, vscatter16_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}

/* masked scatter the 16 bit elements using C */
void scalar_scatter_16_masked(unsigned short *vscatter16)
{
    for (int i = 0; i < MATRIX_SIZE; i++) {
        if (half_predicates[i]) {
            vscatter16[half_offsets[i] / 2] = half_values_masked[i];
        }
    }

}

void check_scatter_16_masked()
{
    memset(vscatter16_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned short));
    scalar_scatter_16(vscatter16_ref);
    scalar_scatter_16_acc(vscatter16_ref);
    scalar_scatter_16_masked(vscatter16_ref);
    check_buffer(__func__, vtcm.vscatter16, vscatter16_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}

/* scatter the 32 bit elements using C */
void scalar_scatter_32(unsigned int *vscatter32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vscatter32[word_offsets[i] / 4] = word_values[i];
    }
}

void check_scatter_32()
{
    memset(vscatter32_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned int));
    scalar_scatter_32(vscatter32_ref);
    check_buffer(__func__, vtcm.vscatter32, vscatter32_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned int));
}

/* scatter-accumulate the 32 bit elements using C */
void scalar_scatter_32_acc(unsigned int *vscatter32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vscatter32[word_offsets[i] / 4] += word_values_acc[i];
    }
}

void check_scatter_32_acc()
{
    memset(vscatter32_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned int));
    scalar_scatter_32(vscatter32_ref);
    scalar_scatter_32_acc(vscatter32_ref);
    check_buffer(__func__, vtcm.vscatter32, vscatter32_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned int));
}

/* masked scatter the 32 bit elements using C */
void scalar_scatter_32_masked(unsigned int *vscatter32)
{
    for (int i = 0; i < MATRIX_SIZE; i++) {
        if (word_predicates[i]) {
            vscatter32[word_offsets[i] / 4] = word_values_masked[i];
        }
    }
}

void check_scatter_32_masked()
{
    memset(vscatter32_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned int));
    scalar_scatter_32(vscatter32_ref);
    scalar_scatter_32_acc(vscatter32_ref);
    scalar_scatter_32_masked(vscatter32_ref);
    check_buffer(__func__, vtcm.vscatter32, vscatter32_ref,
                  SCATTER_BUFFER_SIZE * sizeof(unsigned int));
}

/* scatter the 16 bit elements with 32 bit offsets using C */
void scalar_scatter_16_32(unsigned short *vscatter16_32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vscatter16_32[word_offsets[i] / 2] = half_values[i];
    }
}

void check_scatter_16_32()
{
    memset(vscatter16_32_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned short));
    scalar_scatter_16_32(vscatter16_32_ref);
    check_buffer(__func__, vtcm.vscatter16_32, vscatter16_32_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}

/* scatter-accumulate the 16 bit elements with 32 bit offsets using C */
void scalar_scatter_16_32_acc(unsigned short *vscatter16_32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vscatter16_32[word_offsets[i] / 2] += half_values_acc[i];
    }
}

void check_scatter_16_32_acc()
{
    memset(vscatter16_32_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned short));
    scalar_scatter_16_32(vscatter16_32_ref);
    scalar_scatter_16_32_acc(vscatter16_32_ref);
    check_buffer(__func__, vtcm.vscatter16_32, vscatter16_32_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}

/* masked scatter the 16 bit elements with 32 bit offsets using C */
void scalar_scatter_16_32_masked(unsigned short *vscatter16_32)
{
    for (int i = 0; i < MATRIX_SIZE; i++) {
        if (half_predicates[i]) {
            vscatter16_32[word_offsets[i] / 2] = half_values_masked[i];
        }
    }
}

void check_scatter_16_32_masked()
{
    memset(vscatter16_32_ref, FILL_CHAR,
           SCATTER_BUFFER_SIZE * sizeof(unsigned short));
    scalar_scatter_16_32(vscatter16_32_ref);
    scalar_scatter_16_32_acc(vscatter16_32_ref);
    scalar_scatter_16_32_masked(vscatter16_32_ref);
    check_buffer(__func__, vtcm.vscatter16_32, vscatter16_32_ref,
                 SCATTER_BUFFER_SIZE * sizeof(unsigned short));
}

/* gather the elements from the scatter buffer using C */
void scalar_gather_16(unsigned short *vgather16)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vgather16[i] = vtcm.vscatter16[half_offsets[i] / 2];
    }
}

void check_gather_16()
{
      memset(vgather16_ref, 0, MATRIX_SIZE * sizeof(unsigned short));
      scalar_gather_16(vgather16_ref);
      check_buffer(__func__, vtcm.vgather16, vgather16_ref,
                   MATRIX_SIZE * sizeof(unsigned short));
}

/* masked gather the elements from the scatter buffer using C */
void scalar_gather_16_masked(unsigned short *vgather16)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        if (half_predicates[i]) {
            vgather16[i] = vtcm.vscatter16[half_offsets[i] / 2];
        }
    }
}

void check_gather_16_masked()
{
    memset(vgather16_ref, gather_16_masked_init(),
           MATRIX_SIZE * sizeof(unsigned short));
    scalar_gather_16_masked(vgather16_ref);
    check_buffer(__func__, vtcm.vgather16, vgather16_ref,
                 MATRIX_SIZE * sizeof(unsigned short));
}

/* gather the elements from the scatter32 buffer using C */
void scalar_gather_32(unsigned int *vgather32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vgather32[i] = vtcm.vscatter32[word_offsets[i] / 4];
    }
}

void check_gather_32(void)
{
    memset(vgather32_ref, 0, MATRIX_SIZE * sizeof(unsigned int));
    scalar_gather_32(vgather32_ref);
    check_buffer(__func__, vtcm.vgather32, vgather32_ref,
                 MATRIX_SIZE * sizeof(unsigned int));
}

/* masked gather the elements from the scatter32 buffer using C */
void scalar_gather_32_masked(unsigned int *vgather32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        if (word_predicates[i]) {
            vgather32[i] = vtcm.vscatter32[word_offsets[i] / 4];
        }
    }
}

void check_gather_32_masked(void)
{
    memset(vgather32_ref, gather_32_masked_init(),
           MATRIX_SIZE * sizeof(unsigned int));
    scalar_gather_32_masked(vgather32_ref);
    check_buffer(__func__, vtcm.vgather32,
                 vgather32_ref, MATRIX_SIZE * sizeof(unsigned int));
}

/* gather the elements from the scatter16_32 buffer using C */
void scalar_gather_16_32(unsigned short *vgather16_32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        vgather16_32[i] = vtcm.vscatter16_32[word_offsets[i] / 2];
    }
}

void check_gather_16_32(void)
{
    memset(vgather16_32_ref, 0, MATRIX_SIZE * sizeof(unsigned short));
    scalar_gather_16_32(vgather16_32_ref);
    check_buffer(__func__, vtcm.vgather16_32, vgather16_32_ref,
                 MATRIX_SIZE * sizeof(unsigned short));
}

/* masked gather the elements from the scatter16_32 buffer using C */
void scalar_gather_16_32_masked(unsigned short *vgather16_32)
{
    for (int i = 0; i < MATRIX_SIZE; ++i) {
        if (half_predicates[i]) {
            vgather16_32[i] = vtcm.vscatter16_32[word_offsets[i] / 2];
        }
    }

}

void check_gather_16_32_masked(void)
{
    memset(vgather16_32_ref, gather_16_masked_init(),
           MATRIX_SIZE * sizeof(unsigned short));
    scalar_gather_16_32_masked(vgather16_32_ref);
    check_buffer(__func__, vtcm.vgather16_32, vgather16_32_ref,
                 MATRIX_SIZE * sizeof(unsigned short));
}

/* print scatter16 buffer */
void print_scatter16_buffer(void)
{
    if (PRINT_DATA) {
        printf("\n\nPrinting the 16 bit scatter buffer");

        for (int i = 0; i < SCATTER_BUFFER_SIZE; i++) {
            if ((i % MATRIX_SIZE) == 0) {
                printf("\n");
            }
            for (int j = 0; j < 2; j++) {
                printf("%c", (char)((vtcm.vscatter16[i] >> j * 8) & 0xff));
            }
            printf(" ");
        }
        printf("\n");
    }
}

/* print the gather 16 buffer */
void print_gather_result_16(void)
{
    if (PRINT_DATA) {
        printf("\n\nPrinting the 16 bit gather result\n");

        for (int i = 0; i < MATRIX_SIZE; i++) {
            for (int j = 0; j < 2; j++) {
                printf("%c", (char)((vtcm.vgather16[i] >> j * 8) & 0xff));
            }
            printf(" ");
        }
        printf("\n");
    }
}

/* print the scatter32 buffer */
void print_scatter32_buffer(void)
{
    if (PRINT_DATA) {
        printf("\n\nPrinting the 32 bit scatter buffer");

        for (int i = 0; i < SCATTER_BUFFER_SIZE; i++) {
            if ((i % MATRIX_SIZE) == 0) {
                printf("\n");
            }
            for (int j = 0; j < 4; j++) {
                printf("%c", (char)((vtcm.vscatter32[i] >> j * 8) & 0xff));
            }
            printf(" ");
        }
        printf("\n");
    }
}

/* print the gather 32 buffer */
void print_gather_result_32(void)
{
    if (PRINT_DATA) {
        printf("\n\nPrinting the 32 bit gather result\n");

        for (int i = 0; i < MATRIX_SIZE; i++) {
            for (int j = 0; j < 4; j++) {
                printf("%c", (char)((vtcm.vgather32[i] >> j * 8) & 0xff));
            }
            printf(" ");
        }
        printf("\n");
    }
}

/* print the scatter16_32 buffer */
void print_scatter16_32_buffer(void)
{
    if (PRINT_DATA) {
        printf("\n\nPrinting the 16_32 bit scatter buffer");

        for (int i = 0; i < SCATTER_BUFFER_SIZE; i++) {
            if ((i % MATRIX_SIZE) == 0) {
                printf("\n");
            }
            for (int j = 0; j < 2; j++) {
                printf("%c",
                      (unsigned char)((vtcm.vscatter16_32[i] >> j * 8) & 0xff));
            }
            printf(" ");
        }
        printf("\n");
    }
}

/* print the gather 16_32 buffer */
void print_gather_result_16_32(void)
{
    if (PRINT_DATA) {
        printf("\n\nPrinting the 16_32 bit gather result\n");

        for (int i = 0; i < MATRIX_SIZE; i++) {
            for (int j = 0; j < 2; j++) {
                printf("%c",
                       (unsigned char)((vtcm.vgather16_32[i] >> j * 8) & 0xff));
            }
            printf(" ");
        }
        printf("\n");
    }
}

int main()
{
    prefill_vtcm_scratch();

    /* 16 bit elements with 16 bit offsets */
    create_offsets_values_preds_16();

    vector_scatter_16();
    print_scatter16_buffer();
    check_scatter_16();

    vector_gather_16();
    print_gather_result_16();
    check_gather_16();

    vector_gather_16_masked();
    print_gather_result_16();
    check_gather_16_masked();

    vector_scatter_16_acc();
    print_scatter16_buffer();
    check_scatter_16_acc();

    vector_scatter_16_masked();
    print_scatter16_buffer();
    check_scatter_16_masked();

    /* 32 bit elements with 32 bit offsets */
    create_offsets_values_preds_32();

    vector_scatter_32();
    print_scatter32_buffer();
    check_scatter_32();

    vector_gather_32();
    print_gather_result_32();
    check_gather_32();

    vector_gather_32_masked();
    print_gather_result_32();
    check_gather_32_masked();

    vector_scatter_32_acc();
    print_scatter32_buffer();
    check_scatter_32_acc();

    vector_scatter_32_masked();
    print_scatter32_buffer();
    check_scatter_32_masked();

    /* 16 bit elements with 32 bit offsets */
    create_offsets_values_preds_16_32();

    vector_scatter_16_32();
    print_scatter16_32_buffer();
    check_scatter_16_32();

    vector_gather_16_32();
    print_gather_result_16_32();
    check_gather_16_32();

    vector_gather_16_32_masked();
    print_gather_result_16_32();
    check_gather_16_32_masked();

    vector_scatter_16_32_acc();
    print_scatter16_32_buffer();
    check_scatter_16_32_acc();

    vector_scatter_16_32_masked();
    print_scatter16_32_buffer();
    check_scatter_16_32_masked();

    puts(err ? "FAIL" : "PASS");
    return err;
}