12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later
26c833275SMichal Ludvig /*
36c833275SMichal Ludvig * Cryptographic API.
46c833275SMichal Ludvig *
56c833275SMichal Ludvig * Support for VIA PadLock hardware crypto engine.
66c833275SMichal Ludvig *
76c833275SMichal Ludvig * Copyright (c) 2006 Michal Ludvig <michal@logix.cz>
86c833275SMichal Ludvig */
96c833275SMichal Ludvig
107d024608SHerbert Xu #include <crypto/internal/hash.h>
1121493088SHerbert Xu #include <crypto/padlock.h>
12*a24d22b2SEric Biggers #include <crypto/sha1.h>
13*a24d22b2SEric Biggers #include <crypto/sha2.h>
146010439fSHerbert Xu #include <linux/err.h>
156c833275SMichal Ludvig #include <linux/module.h>
166c833275SMichal Ludvig #include <linux/init.h>
176c833275SMichal Ludvig #include <linux/errno.h>
186c833275SMichal Ludvig #include <linux/interrupt.h>
196c833275SMichal Ludvig #include <linux/kernel.h>
206c833275SMichal Ludvig #include <linux/scatterlist.h>
213bd391f0SAndi Kleen #include <asm/cpu_device_id.h>
22df6b35f4SIngo Molnar #include <asm/fpu/api.h>
234c6ab3eeSHerbert Xu
24bbbee467SHerbert Xu struct padlock_sha_desc {
25bbbee467SHerbert Xu struct shash_desc fallback;
266c833275SMichal Ludvig };
276c833275SMichal Ludvig
28bbbee467SHerbert Xu struct padlock_sha_ctx {
29bbbee467SHerbert Xu struct crypto_shash *fallback;
30bbbee467SHerbert Xu };
31bbbee467SHerbert Xu
padlock_sha_init(struct shash_desc * desc)32bbbee467SHerbert Xu static int padlock_sha_init(struct shash_desc *desc)
336c833275SMichal Ludvig {
34bbbee467SHerbert Xu struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
35bbbee467SHerbert Xu struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
36bbbee467SHerbert Xu
37bbbee467SHerbert Xu dctx->fallback.tfm = ctx->fallback;
38bbbee467SHerbert Xu return crypto_shash_init(&dctx->fallback);
396c833275SMichal Ludvig }
406c833275SMichal Ludvig
padlock_sha_update(struct shash_desc * desc,const u8 * data,unsigned int length)41bbbee467SHerbert Xu static int padlock_sha_update(struct shash_desc *desc,
42bbbee467SHerbert Xu const u8 *data, unsigned int length)
436c833275SMichal Ludvig {
44bbbee467SHerbert Xu struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
457d024608SHerbert Xu
46bbbee467SHerbert Xu return crypto_shash_update(&dctx->fallback, data, length);
476c833275SMichal Ludvig }
486c833275SMichal Ludvig
padlock_sha_export(struct shash_desc * desc,void * out)49a8d7ac27SHerbert Xu static int padlock_sha_export(struct shash_desc *desc, void *out)
50a8d7ac27SHerbert Xu {
51a8d7ac27SHerbert Xu struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
52a8d7ac27SHerbert Xu
53a8d7ac27SHerbert Xu return crypto_shash_export(&dctx->fallback, out);
54a8d7ac27SHerbert Xu }
55a8d7ac27SHerbert Xu
padlock_sha_import(struct shash_desc * desc,const void * in)56a8d7ac27SHerbert Xu static int padlock_sha_import(struct shash_desc *desc, const void *in)
57a8d7ac27SHerbert Xu {
58a8d7ac27SHerbert Xu struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
59a8d7ac27SHerbert Xu struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm);
60a8d7ac27SHerbert Xu
61a8d7ac27SHerbert Xu dctx->fallback.tfm = ctx->fallback;
62a8d7ac27SHerbert Xu return crypto_shash_import(&dctx->fallback, in);
63a8d7ac27SHerbert Xu }
64a8d7ac27SHerbert Xu
padlock_output_block(uint32_t * src,uint32_t * dst,size_t count)656c833275SMichal Ludvig static inline void padlock_output_block(uint32_t *src,
666c833275SMichal Ludvig uint32_t *dst, size_t count)
676c833275SMichal Ludvig {
686c833275SMichal Ludvig while (count--)
696c833275SMichal Ludvig *dst++ = swab32(*src++);
706c833275SMichal Ludvig }
716c833275SMichal Ludvig
padlock_sha1_finup(struct shash_desc * desc,const u8 * in,unsigned int count,u8 * out)72bbbee467SHerbert Xu static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in,
73bbbee467SHerbert Xu unsigned int count, u8 *out)
746c833275SMichal Ludvig {
756c833275SMichal Ludvig /* We can't store directly to *out as it may be unaligned. */
766c833275SMichal Ludvig /* BTW Don't reduce the buffer size below 128 Bytes!
776c833275SMichal Ludvig * PadLock microcode needs it that big. */
784c6ab3eeSHerbert Xu char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
794c6ab3eeSHerbert Xu ((aligned(STACK_ALIGN)));
804c6ab3eeSHerbert Xu char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
81bbbee467SHerbert Xu struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
82bbbee467SHerbert Xu struct sha1_state state;
83bbbee467SHerbert Xu unsigned int space;
84bbbee467SHerbert Xu unsigned int leftover;
85bbbee467SHerbert Xu int err;
866c833275SMichal Ludvig
87bbbee467SHerbert Xu err = crypto_shash_export(&dctx->fallback, &state);
88bbbee467SHerbert Xu if (err)
89bbbee467SHerbert Xu goto out;
90bbbee467SHerbert Xu
91bbbee467SHerbert Xu if (state.count + count > ULONG_MAX)
92bbbee467SHerbert Xu return crypto_shash_finup(&dctx->fallback, in, count, out);
93bbbee467SHerbert Xu
94bbbee467SHerbert Xu leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1;
95bbbee467SHerbert Xu space = SHA1_BLOCK_SIZE - leftover;
96bbbee467SHerbert Xu if (space) {
97bbbee467SHerbert Xu if (count > space) {
98bbbee467SHerbert Xu err = crypto_shash_update(&dctx->fallback, in, space) ?:
99bbbee467SHerbert Xu crypto_shash_export(&dctx->fallback, &state);
100bbbee467SHerbert Xu if (err)
101bbbee467SHerbert Xu goto out;
102bbbee467SHerbert Xu count -= space;
103bbbee467SHerbert Xu in += space;
104bbbee467SHerbert Xu } else {
105bbbee467SHerbert Xu memcpy(state.buffer + leftover, in, count);
106bbbee467SHerbert Xu in = state.buffer;
107bbbee467SHerbert Xu count += leftover;
108e9b25f16SHerbert Xu state.count &= ~(SHA1_BLOCK_SIZE - 1);
109bbbee467SHerbert Xu }
110bbbee467SHerbert Xu }
111bbbee467SHerbert Xu
112bbbee467SHerbert Xu memcpy(result, &state.state, SHA1_DIGEST_SIZE);
1136c833275SMichal Ludvig
1146c833275SMichal Ludvig asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
115bbbee467SHerbert Xu : \
116faae8908SHerbert Xu : "c"((unsigned long)state.count + count), \
117faae8908SHerbert Xu "a"((unsigned long)state.count), \
118bbbee467SHerbert Xu "S"(in), "D"(result));
1196c833275SMichal Ludvig
1206c833275SMichal Ludvig padlock_output_block((uint32_t *)result, (uint32_t *)out, 5);
121bbbee467SHerbert Xu
122bbbee467SHerbert Xu out:
123bbbee467SHerbert Xu return err;
1246c833275SMichal Ludvig }
1256c833275SMichal Ludvig
padlock_sha1_final(struct shash_desc * desc,u8 * out)126bbbee467SHerbert Xu static int padlock_sha1_final(struct shash_desc *desc, u8 *out)
127bbbee467SHerbert Xu {
128bbbee467SHerbert Xu u8 buf[4];
129bbbee467SHerbert Xu
130bbbee467SHerbert Xu return padlock_sha1_finup(desc, buf, 0, out);
131bbbee467SHerbert Xu }
132bbbee467SHerbert Xu
padlock_sha256_finup(struct shash_desc * desc,const u8 * in,unsigned int count,u8 * out)133bbbee467SHerbert Xu static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in,
134bbbee467SHerbert Xu unsigned int count, u8 *out)
1356c833275SMichal Ludvig {
1366c833275SMichal Ludvig /* We can't store directly to *out as it may be unaligned. */
1376c833275SMichal Ludvig /* BTW Don't reduce the buffer size below 128 Bytes!
1386c833275SMichal Ludvig * PadLock microcode needs it that big. */
1394c6ab3eeSHerbert Xu char buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
1404c6ab3eeSHerbert Xu ((aligned(STACK_ALIGN)));
1414c6ab3eeSHerbert Xu char *result = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
142bbbee467SHerbert Xu struct padlock_sha_desc *dctx = shash_desc_ctx(desc);
143bbbee467SHerbert Xu struct sha256_state state;
144bbbee467SHerbert Xu unsigned int space;
145bbbee467SHerbert Xu unsigned int leftover;
146bbbee467SHerbert Xu int err;
1476c833275SMichal Ludvig
148bbbee467SHerbert Xu err = crypto_shash_export(&dctx->fallback, &state);
149bbbee467SHerbert Xu if (err)
150bbbee467SHerbert Xu goto out;
151bbbee467SHerbert Xu
152bbbee467SHerbert Xu if (state.count + count > ULONG_MAX)
153bbbee467SHerbert Xu return crypto_shash_finup(&dctx->fallback, in, count, out);
154bbbee467SHerbert Xu
155bbbee467SHerbert Xu leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1;
156bbbee467SHerbert Xu space = SHA256_BLOCK_SIZE - leftover;
157bbbee467SHerbert Xu if (space) {
158bbbee467SHerbert Xu if (count > space) {
159bbbee467SHerbert Xu err = crypto_shash_update(&dctx->fallback, in, space) ?:
160bbbee467SHerbert Xu crypto_shash_export(&dctx->fallback, &state);
161bbbee467SHerbert Xu if (err)
162bbbee467SHerbert Xu goto out;
163bbbee467SHerbert Xu count -= space;
164bbbee467SHerbert Xu in += space;
165bbbee467SHerbert Xu } else {
166bbbee467SHerbert Xu memcpy(state.buf + leftover, in, count);
167bbbee467SHerbert Xu in = state.buf;
168bbbee467SHerbert Xu count += leftover;
169e9b25f16SHerbert Xu state.count &= ~(SHA1_BLOCK_SIZE - 1);
170bbbee467SHerbert Xu }
171bbbee467SHerbert Xu }
172bbbee467SHerbert Xu
173bbbee467SHerbert Xu memcpy(result, &state.state, SHA256_DIGEST_SIZE);
1746c833275SMichal Ludvig
1756c833275SMichal Ludvig asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
176bbbee467SHerbert Xu : \
177faae8908SHerbert Xu : "c"((unsigned long)state.count + count), \
178faae8908SHerbert Xu "a"((unsigned long)state.count), \
179bbbee467SHerbert Xu "S"(in), "D"(result));
1806c833275SMichal Ludvig
1816c833275SMichal Ludvig padlock_output_block((uint32_t *)result, (uint32_t *)out, 8);
182bbbee467SHerbert Xu
183bbbee467SHerbert Xu out:
184bbbee467SHerbert Xu return err;
1856c833275SMichal Ludvig }
1866c833275SMichal Ludvig
padlock_sha256_final(struct shash_desc * desc,u8 * out)187bbbee467SHerbert Xu static int padlock_sha256_final(struct shash_desc *desc, u8 *out)
1886c833275SMichal Ludvig {
189bbbee467SHerbert Xu u8 buf[4];
1907d024608SHerbert Xu
191bbbee467SHerbert Xu return padlock_sha256_finup(desc, buf, 0, out);
1926c833275SMichal Ludvig }
1936c833275SMichal Ludvig
padlock_init_tfm(struct crypto_shash * hash)19496895693SHerbert Xu static int padlock_init_tfm(struct crypto_shash *hash)
1956c833275SMichal Ludvig {
19696895693SHerbert Xu const char *fallback_driver_name = crypto_shash_alg_name(hash);
19796895693SHerbert Xu struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash);
1987d024608SHerbert Xu struct crypto_shash *fallback_tfm;
1996010439fSHerbert Xu
2006c833275SMichal Ludvig /* Allocate a fallback and abort if it failed. */
2017d024608SHerbert Xu fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0,
2026010439fSHerbert Xu CRYPTO_ALG_NEED_FALLBACK);
2036010439fSHerbert Xu if (IS_ERR(fallback_tfm)) {
2046c833275SMichal Ludvig printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
2056c833275SMichal Ludvig fallback_driver_name);
20696895693SHerbert Xu return PTR_ERR(fallback_tfm);
2076c833275SMichal Ludvig }
2086c833275SMichal Ludvig
209bbbee467SHerbert Xu ctx->fallback = fallback_tfm;
210bbbee467SHerbert Xu hash->descsize += crypto_shash_descsize(fallback_tfm);
2116c833275SMichal Ludvig return 0;
2126c833275SMichal Ludvig }
2136c833275SMichal Ludvig
padlock_exit_tfm(struct crypto_shash * hash)21496895693SHerbert Xu static void padlock_exit_tfm(struct crypto_shash *hash)
2156c833275SMichal Ludvig {
21696895693SHerbert Xu struct padlock_sha_ctx *ctx = crypto_shash_ctx(hash);
217bbbee467SHerbert Xu
218bbbee467SHerbert Xu crypto_free_shash(ctx->fallback);
2196c833275SMichal Ludvig }
2206c833275SMichal Ludvig
221bbbee467SHerbert Xu static struct shash_alg sha1_alg = {
222bbbee467SHerbert Xu .digestsize = SHA1_DIGEST_SIZE,
223bbbee467SHerbert Xu .init = padlock_sha_init,
224bbbee467SHerbert Xu .update = padlock_sha_update,
225bbbee467SHerbert Xu .finup = padlock_sha1_finup,
226bbbee467SHerbert Xu .final = padlock_sha1_final,
227a8d7ac27SHerbert Xu .export = padlock_sha_export,
228a8d7ac27SHerbert Xu .import = padlock_sha_import,
22996895693SHerbert Xu .init_tfm = padlock_init_tfm,
23096895693SHerbert Xu .exit_tfm = padlock_exit_tfm,
231bbbee467SHerbert Xu .descsize = sizeof(struct padlock_sha_desc),
232a8d7ac27SHerbert Xu .statesize = sizeof(struct sha1_state),
233bbbee467SHerbert Xu .base = {
2346c833275SMichal Ludvig .cra_name = "sha1",
2356c833275SMichal Ludvig .cra_driver_name = "sha1-padlock",
2366c833275SMichal Ludvig .cra_priority = PADLOCK_CRA_PRIORITY,
237e50944e2SEric Biggers .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
2385265eeb2SJan Glauber .cra_blocksize = SHA1_BLOCK_SIZE,
2396c833275SMichal Ludvig .cra_ctxsize = sizeof(struct padlock_sha_ctx),
2406c833275SMichal Ludvig .cra_module = THIS_MODULE,
2416c833275SMichal Ludvig }
2426c833275SMichal Ludvig };
2436c833275SMichal Ludvig
244bbbee467SHerbert Xu static struct shash_alg sha256_alg = {
245bbbee467SHerbert Xu .digestsize = SHA256_DIGEST_SIZE,
246bbbee467SHerbert Xu .init = padlock_sha_init,
247bbbee467SHerbert Xu .update = padlock_sha_update,
248bbbee467SHerbert Xu .finup = padlock_sha256_finup,
249bbbee467SHerbert Xu .final = padlock_sha256_final,
250a8d7ac27SHerbert Xu .export = padlock_sha_export,
251a8d7ac27SHerbert Xu .import = padlock_sha_import,
25296895693SHerbert Xu .init_tfm = padlock_init_tfm,
25396895693SHerbert Xu .exit_tfm = padlock_exit_tfm,
254bbbee467SHerbert Xu .descsize = sizeof(struct padlock_sha_desc),
255a8d7ac27SHerbert Xu .statesize = sizeof(struct sha256_state),
256bbbee467SHerbert Xu .base = {
2576c833275SMichal Ludvig .cra_name = "sha256",
2586c833275SMichal Ludvig .cra_driver_name = "sha256-padlock",
2596c833275SMichal Ludvig .cra_priority = PADLOCK_CRA_PRIORITY,
260e50944e2SEric Biggers .cra_flags = CRYPTO_ALG_NEED_FALLBACK,
2615265eeb2SJan Glauber .cra_blocksize = SHA256_BLOCK_SIZE,
2626c833275SMichal Ludvig .cra_ctxsize = sizeof(struct padlock_sha_ctx),
2636c833275SMichal Ludvig .cra_module = THIS_MODULE,
2646c833275SMichal Ludvig }
2656c833275SMichal Ludvig };
2666c833275SMichal Ludvig
2670475add3SBrilly Wu /* Add two shash_alg instance for hardware-implemented *
2680475add3SBrilly Wu * multiple-parts hash supported by VIA Nano Processor.*/
padlock_sha1_init_nano(struct shash_desc * desc)2690475add3SBrilly Wu static int padlock_sha1_init_nano(struct shash_desc *desc)
2700475add3SBrilly Wu {
2710475add3SBrilly Wu struct sha1_state *sctx = shash_desc_ctx(desc);
2720475add3SBrilly Wu
2730475add3SBrilly Wu *sctx = (struct sha1_state){
2740475add3SBrilly Wu .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
2750475add3SBrilly Wu };
2760475add3SBrilly Wu
2770475add3SBrilly Wu return 0;
2780475add3SBrilly Wu }
2790475add3SBrilly Wu
padlock_sha1_update_nano(struct shash_desc * desc,const u8 * data,unsigned int len)2800475add3SBrilly Wu static int padlock_sha1_update_nano(struct shash_desc *desc,
2810475add3SBrilly Wu const u8 *data, unsigned int len)
2820475add3SBrilly Wu {
2830475add3SBrilly Wu struct sha1_state *sctx = shash_desc_ctx(desc);
2840475add3SBrilly Wu unsigned int partial, done;
2850475add3SBrilly Wu const u8 *src;
2860475add3SBrilly Wu /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
2870475add3SBrilly Wu u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
2880475add3SBrilly Wu ((aligned(STACK_ALIGN)));
2890475add3SBrilly Wu u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
2900475add3SBrilly Wu
2910475add3SBrilly Wu partial = sctx->count & 0x3f;
2920475add3SBrilly Wu sctx->count += len;
2930475add3SBrilly Wu done = 0;
2940475add3SBrilly Wu src = data;
2950475add3SBrilly Wu memcpy(dst, (u8 *)(sctx->state), SHA1_DIGEST_SIZE);
2960475add3SBrilly Wu
2970475add3SBrilly Wu if ((partial + len) >= SHA1_BLOCK_SIZE) {
2980475add3SBrilly Wu
2990475add3SBrilly Wu /* Append the bytes in state's buffer to a block to handle */
3000475add3SBrilly Wu if (partial) {
3010475add3SBrilly Wu done = -partial;
3020475add3SBrilly Wu memcpy(sctx->buffer + partial, data,
3030475add3SBrilly Wu done + SHA1_BLOCK_SIZE);
3040475add3SBrilly Wu src = sctx->buffer;
3050475add3SBrilly Wu asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
3060475add3SBrilly Wu : "+S"(src), "+D"(dst) \
3070475add3SBrilly Wu : "a"((long)-1), "c"((unsigned long)1));
3080475add3SBrilly Wu done += SHA1_BLOCK_SIZE;
3090475add3SBrilly Wu src = data + done;
3100475add3SBrilly Wu }
3110475add3SBrilly Wu
3120475add3SBrilly Wu /* Process the left bytes from the input data */
3130475add3SBrilly Wu if (len - done >= SHA1_BLOCK_SIZE) {
3140475add3SBrilly Wu asm volatile (".byte 0xf3,0x0f,0xa6,0xc8"
3150475add3SBrilly Wu : "+S"(src), "+D"(dst)
3160475add3SBrilly Wu : "a"((long)-1),
3170475add3SBrilly Wu "c"((unsigned long)((len - done) / SHA1_BLOCK_SIZE)));
3180475add3SBrilly Wu done += ((len - done) - (len - done) % SHA1_BLOCK_SIZE);
3190475add3SBrilly Wu src = data + done;
3200475add3SBrilly Wu }
3210475add3SBrilly Wu partial = 0;
3220475add3SBrilly Wu }
3230475add3SBrilly Wu memcpy((u8 *)(sctx->state), dst, SHA1_DIGEST_SIZE);
3240475add3SBrilly Wu memcpy(sctx->buffer + partial, src, len - done);
3250475add3SBrilly Wu
3260475add3SBrilly Wu return 0;
3270475add3SBrilly Wu }
3280475add3SBrilly Wu
padlock_sha1_final_nano(struct shash_desc * desc,u8 * out)3290475add3SBrilly Wu static int padlock_sha1_final_nano(struct shash_desc *desc, u8 *out)
3300475add3SBrilly Wu {
3310475add3SBrilly Wu struct sha1_state *state = (struct sha1_state *)shash_desc_ctx(desc);
3320475add3SBrilly Wu unsigned int partial, padlen;
3330475add3SBrilly Wu __be64 bits;
3340475add3SBrilly Wu static const u8 padding[64] = { 0x80, };
3350475add3SBrilly Wu
3360475add3SBrilly Wu bits = cpu_to_be64(state->count << 3);
3370475add3SBrilly Wu
3380475add3SBrilly Wu /* Pad out to 56 mod 64 */
3390475add3SBrilly Wu partial = state->count & 0x3f;
3400475add3SBrilly Wu padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
3410475add3SBrilly Wu padlock_sha1_update_nano(desc, padding, padlen);
3420475add3SBrilly Wu
3430475add3SBrilly Wu /* Append length field bytes */
3440475add3SBrilly Wu padlock_sha1_update_nano(desc, (const u8 *)&bits, sizeof(bits));
3450475add3SBrilly Wu
3460475add3SBrilly Wu /* Swap to output */
3470475add3SBrilly Wu padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 5);
3480475add3SBrilly Wu
3490475add3SBrilly Wu return 0;
3500475add3SBrilly Wu }
3510475add3SBrilly Wu
padlock_sha256_init_nano(struct shash_desc * desc)3520475add3SBrilly Wu static int padlock_sha256_init_nano(struct shash_desc *desc)
3530475add3SBrilly Wu {
3540475add3SBrilly Wu struct sha256_state *sctx = shash_desc_ctx(desc);
3550475add3SBrilly Wu
3560475add3SBrilly Wu *sctx = (struct sha256_state){
3570475add3SBrilly Wu .state = { SHA256_H0, SHA256_H1, SHA256_H2, SHA256_H3, \
3580475add3SBrilly Wu SHA256_H4, SHA256_H5, SHA256_H6, SHA256_H7},
3590475add3SBrilly Wu };
3600475add3SBrilly Wu
3610475add3SBrilly Wu return 0;
3620475add3SBrilly Wu }
3630475add3SBrilly Wu
padlock_sha256_update_nano(struct shash_desc * desc,const u8 * data,unsigned int len)3640475add3SBrilly Wu static int padlock_sha256_update_nano(struct shash_desc *desc, const u8 *data,
3650475add3SBrilly Wu unsigned int len)
3660475add3SBrilly Wu {
3670475add3SBrilly Wu struct sha256_state *sctx = shash_desc_ctx(desc);
3680475add3SBrilly Wu unsigned int partial, done;
3690475add3SBrilly Wu const u8 *src;
3700475add3SBrilly Wu /*The PHE require the out buffer must 128 bytes and 16-bytes aligned*/
3710475add3SBrilly Wu u8 buf[128 + PADLOCK_ALIGNMENT - STACK_ALIGN] __attribute__
3720475add3SBrilly Wu ((aligned(STACK_ALIGN)));
3730475add3SBrilly Wu u8 *dst = PTR_ALIGN(&buf[0], PADLOCK_ALIGNMENT);
3740475add3SBrilly Wu
3750475add3SBrilly Wu partial = sctx->count & 0x3f;
3760475add3SBrilly Wu sctx->count += len;
3770475add3SBrilly Wu done = 0;
3780475add3SBrilly Wu src = data;
3790475add3SBrilly Wu memcpy(dst, (u8 *)(sctx->state), SHA256_DIGEST_SIZE);
3800475add3SBrilly Wu
3810475add3SBrilly Wu if ((partial + len) >= SHA256_BLOCK_SIZE) {
3820475add3SBrilly Wu
3830475add3SBrilly Wu /* Append the bytes in state's buffer to a block to handle */
3840475add3SBrilly Wu if (partial) {
3850475add3SBrilly Wu done = -partial;
3860475add3SBrilly Wu memcpy(sctx->buf + partial, data,
3870475add3SBrilly Wu done + SHA256_BLOCK_SIZE);
3880475add3SBrilly Wu src = sctx->buf;
3890475add3SBrilly Wu asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
3900475add3SBrilly Wu : "+S"(src), "+D"(dst)
3910475add3SBrilly Wu : "a"((long)-1), "c"((unsigned long)1));
3920475add3SBrilly Wu done += SHA256_BLOCK_SIZE;
3930475add3SBrilly Wu src = data + done;
3940475add3SBrilly Wu }
3950475add3SBrilly Wu
3960475add3SBrilly Wu /* Process the left bytes from input data*/
3970475add3SBrilly Wu if (len - done >= SHA256_BLOCK_SIZE) {
3980475add3SBrilly Wu asm volatile (".byte 0xf3,0x0f,0xa6,0xd0"
3990475add3SBrilly Wu : "+S"(src), "+D"(dst)
4000475add3SBrilly Wu : "a"((long)-1),
4010475add3SBrilly Wu "c"((unsigned long)((len - done) / 64)));
4020475add3SBrilly Wu done += ((len - done) - (len - done) % 64);
4030475add3SBrilly Wu src = data + done;
4040475add3SBrilly Wu }
4050475add3SBrilly Wu partial = 0;
4060475add3SBrilly Wu }
4070475add3SBrilly Wu memcpy((u8 *)(sctx->state), dst, SHA256_DIGEST_SIZE);
4080475add3SBrilly Wu memcpy(sctx->buf + partial, src, len - done);
4090475add3SBrilly Wu
4100475add3SBrilly Wu return 0;
4110475add3SBrilly Wu }
4120475add3SBrilly Wu
padlock_sha256_final_nano(struct shash_desc * desc,u8 * out)4130475add3SBrilly Wu static int padlock_sha256_final_nano(struct shash_desc *desc, u8 *out)
4140475add3SBrilly Wu {
4150475add3SBrilly Wu struct sha256_state *state =
4160475add3SBrilly Wu (struct sha256_state *)shash_desc_ctx(desc);
4170475add3SBrilly Wu unsigned int partial, padlen;
4180475add3SBrilly Wu __be64 bits;
4190475add3SBrilly Wu static const u8 padding[64] = { 0x80, };
4200475add3SBrilly Wu
4210475add3SBrilly Wu bits = cpu_to_be64(state->count << 3);
4220475add3SBrilly Wu
4230475add3SBrilly Wu /* Pad out to 56 mod 64 */
4240475add3SBrilly Wu partial = state->count & 0x3f;
4250475add3SBrilly Wu padlen = (partial < 56) ? (56 - partial) : ((64+56) - partial);
4260475add3SBrilly Wu padlock_sha256_update_nano(desc, padding, padlen);
4270475add3SBrilly Wu
4280475add3SBrilly Wu /* Append length field bytes */
4290475add3SBrilly Wu padlock_sha256_update_nano(desc, (const u8 *)&bits, sizeof(bits));
4300475add3SBrilly Wu
4310475add3SBrilly Wu /* Swap to output */
4320475add3SBrilly Wu padlock_output_block((uint32_t *)(state->state), (uint32_t *)out, 8);
4330475add3SBrilly Wu
4340475add3SBrilly Wu return 0;
4350475add3SBrilly Wu }
4360475add3SBrilly Wu
padlock_sha_export_nano(struct shash_desc * desc,void * out)4370475add3SBrilly Wu static int padlock_sha_export_nano(struct shash_desc *desc,
4380475add3SBrilly Wu void *out)
4390475add3SBrilly Wu {
4400475add3SBrilly Wu int statesize = crypto_shash_statesize(desc->tfm);
4410475add3SBrilly Wu void *sctx = shash_desc_ctx(desc);
4420475add3SBrilly Wu
4430475add3SBrilly Wu memcpy(out, sctx, statesize);
4440475add3SBrilly Wu return 0;
4450475add3SBrilly Wu }
4460475add3SBrilly Wu
padlock_sha_import_nano(struct shash_desc * desc,const void * in)4470475add3SBrilly Wu static int padlock_sha_import_nano(struct shash_desc *desc,
4480475add3SBrilly Wu const void *in)
4490475add3SBrilly Wu {
4500475add3SBrilly Wu int statesize = crypto_shash_statesize(desc->tfm);
4510475add3SBrilly Wu void *sctx = shash_desc_ctx(desc);
4520475add3SBrilly Wu
4530475add3SBrilly Wu memcpy(sctx, in, statesize);
4540475add3SBrilly Wu return 0;
4550475add3SBrilly Wu }
4560475add3SBrilly Wu
4570475add3SBrilly Wu static struct shash_alg sha1_alg_nano = {
4580475add3SBrilly Wu .digestsize = SHA1_DIGEST_SIZE,
4590475add3SBrilly Wu .init = padlock_sha1_init_nano,
4600475add3SBrilly Wu .update = padlock_sha1_update_nano,
4610475add3SBrilly Wu .final = padlock_sha1_final_nano,
4620475add3SBrilly Wu .export = padlock_sha_export_nano,
4630475add3SBrilly Wu .import = padlock_sha_import_nano,
4640475add3SBrilly Wu .descsize = sizeof(struct sha1_state),
4650475add3SBrilly Wu .statesize = sizeof(struct sha1_state),
4660475add3SBrilly Wu .base = {
4670475add3SBrilly Wu .cra_name = "sha1",
4680475add3SBrilly Wu .cra_driver_name = "sha1-padlock-nano",
4690475add3SBrilly Wu .cra_priority = PADLOCK_CRA_PRIORITY,
4700475add3SBrilly Wu .cra_blocksize = SHA1_BLOCK_SIZE,
4710475add3SBrilly Wu .cra_module = THIS_MODULE,
4720475add3SBrilly Wu }
4730475add3SBrilly Wu };
4740475add3SBrilly Wu
4750475add3SBrilly Wu static struct shash_alg sha256_alg_nano = {
4760475add3SBrilly Wu .digestsize = SHA256_DIGEST_SIZE,
4770475add3SBrilly Wu .init = padlock_sha256_init_nano,
4780475add3SBrilly Wu .update = padlock_sha256_update_nano,
4790475add3SBrilly Wu .final = padlock_sha256_final_nano,
4800475add3SBrilly Wu .export = padlock_sha_export_nano,
4810475add3SBrilly Wu .import = padlock_sha_import_nano,
4820475add3SBrilly Wu .descsize = sizeof(struct sha256_state),
4830475add3SBrilly Wu .statesize = sizeof(struct sha256_state),
4840475add3SBrilly Wu .base = {
4850475add3SBrilly Wu .cra_name = "sha256",
4860475add3SBrilly Wu .cra_driver_name = "sha256-padlock-nano",
4870475add3SBrilly Wu .cra_priority = PADLOCK_CRA_PRIORITY,
4880475add3SBrilly Wu .cra_blocksize = SHA256_BLOCK_SIZE,
4890475add3SBrilly Wu .cra_module = THIS_MODULE,
4900475add3SBrilly Wu }
4910475add3SBrilly Wu };
4920475add3SBrilly Wu
49316d5cee5SArvind Yadav static const struct x86_cpu_id padlock_sha_ids[] = {
494f30cfacaSThomas Gleixner X86_MATCH_FEATURE(X86_FEATURE_PHE, NULL),
4953bd391f0SAndi Kleen {}
4963bd391f0SAndi Kleen };
4973bd391f0SAndi Kleen MODULE_DEVICE_TABLE(x86cpu, padlock_sha_ids);
4983bd391f0SAndi Kleen
padlock_init(void)4996c833275SMichal Ludvig static int __init padlock_init(void)
5006c833275SMichal Ludvig {
5016c833275SMichal Ludvig int rc = -ENODEV;
5020475add3SBrilly Wu struct cpuinfo_x86 *c = &cpu_data(0);
5030475add3SBrilly Wu struct shash_alg *sha1;
5040475add3SBrilly Wu struct shash_alg *sha256;
5056c833275SMichal Ludvig
506362f924bSBorislav Petkov if (!x86_match_cpu(padlock_sha_ids) || !boot_cpu_has(X86_FEATURE_PHE_EN))
5076c833275SMichal Ludvig return -ENODEV;
5086c833275SMichal Ludvig
5090475add3SBrilly Wu /* Register the newly added algorithm module if on *
5100475add3SBrilly Wu * VIA Nano processor, or else just do as before */
5110475add3SBrilly Wu if (c->x86_model < 0x0f) {
5120475add3SBrilly Wu sha1 = &sha1_alg;
5130475add3SBrilly Wu sha256 = &sha256_alg;
5140475add3SBrilly Wu } else {
5150475add3SBrilly Wu sha1 = &sha1_alg_nano;
5160475add3SBrilly Wu sha256 = &sha256_alg_nano;
5170475add3SBrilly Wu }
5180475add3SBrilly Wu
5190475add3SBrilly Wu rc = crypto_register_shash(sha1);
5206c833275SMichal Ludvig if (rc)
5216c833275SMichal Ludvig goto out;
5226c833275SMichal Ludvig
5230475add3SBrilly Wu rc = crypto_register_shash(sha256);
5246c833275SMichal Ludvig if (rc)
5256c833275SMichal Ludvig goto out_unreg1;
5266c833275SMichal Ludvig
5276c833275SMichal Ludvig printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");
5286c833275SMichal Ludvig
5296c833275SMichal Ludvig return 0;
5306c833275SMichal Ludvig
5316c833275SMichal Ludvig out_unreg1:
5320475add3SBrilly Wu crypto_unregister_shash(sha1);
5330475add3SBrilly Wu
5346c833275SMichal Ludvig out:
5356c833275SMichal Ludvig printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
5366c833275SMichal Ludvig return rc;
5376c833275SMichal Ludvig }
5386c833275SMichal Ludvig
padlock_fini(void)5396c833275SMichal Ludvig static void __exit padlock_fini(void)
5406c833275SMichal Ludvig {
5410475add3SBrilly Wu struct cpuinfo_x86 *c = &cpu_data(0);
5420475add3SBrilly Wu
5430475add3SBrilly Wu if (c->x86_model >= 0x0f) {
5440475add3SBrilly Wu crypto_unregister_shash(&sha1_alg_nano);
5450475add3SBrilly Wu crypto_unregister_shash(&sha256_alg_nano);
5460475add3SBrilly Wu } else {
547bbbee467SHerbert Xu crypto_unregister_shash(&sha1_alg);
548bbbee467SHerbert Xu crypto_unregister_shash(&sha256_alg);
5496c833275SMichal Ludvig }
5500475add3SBrilly Wu }
5516c833275SMichal Ludvig
5526c833275SMichal Ludvig module_init(padlock_init);
5536c833275SMichal Ludvig module_exit(padlock_fini);
5546c833275SMichal Ludvig
5556c833275SMichal Ludvig MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support.");
5566c833275SMichal Ludvig MODULE_LICENSE("GPL");
5576c833275SMichal Ludvig MODULE_AUTHOR("Michal Ludvig");
5586c833275SMichal Ludvig
5595d26a105SKees Cook MODULE_ALIAS_CRYPTO("sha1-all");
5605d26a105SKees Cook MODULE_ALIAS_CRYPTO("sha256-all");
5615d26a105SKees Cook MODULE_ALIAS_CRYPTO("sha1-padlock");
5625d26a105SKees Cook MODULE_ALIAS_CRYPTO("sha256-padlock");
563