GitHub-Mirror/linux-bl808
2
0
Fork 0
mirror of https://github.com/Fishwaldo/linux-bl808.git synced 2025-03-18 04:54:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6

Browse source 
Pull crypto updates from Herbert Xu:
 "Algorithms:

   - Drop alignment requirement for data in aesni

   - Use synchronous seeding from the /dev/random in DRBG

   - Reseed nopr DRBGs every 5 minutes from /dev/random

   - Add KDF algorithms currently used by security/DH

   - Fix lack of entropy on some AMD CPUs with jitter RNG

  Drivers:

   - Add support for the D1 variant in sun8i-ce

   - Add SEV_INIT_EX support in ccp

   - PFVF support for GEN4 host driver in qat

   - Compression support for GEN4 devices in qat

   - Add cn10k random number generator support"

* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (145 commits)
  crypto: af_alg - rewrite NULL pointer check
  lib/mpi: Add the return value check of kcalloc()
  crypto: qat - fix definition of ring reset results
  crypto: hisilicon - cleanup warning in qm_get_qos_value()
  crypto: kdf - select SHA-256 required for self-test
  crypto: x86/aesni - don't require alignment of data
  crypto: ccp - remove unneeded semicolon
  crypto: stm32/crc32 - Fix kernel BUG triggered in probe()
  crypto: s390/sha512 - Use macros instead of direct IV numbers
  crypto: sparc/sha - remove duplicate hash init function
  crypto: powerpc/sha - remove duplicate hash init function
  crypto: mips/sha - remove duplicate hash init function
  crypto: sha256 - remove duplicate generic hash init function
  crypto: jitter - add oversampling of noise source
  MAINTAINERS: update SEC2 driver maintainers list
  crypto: ux500 - Use platform_get_irq() to get the interrupt
  crypto: hisilicon/qm - disable qm clock-gating
  crypto: omap-aes - Fix broken pm_runtime_and_get() usage
  MAINTAINERS: update caam crypto driver maintainers list
  crypto: octeontx2 - prevent underflow in get_cores_bmap()
  ...
This commit is contained in:
Linus Torvalds 2022-01-11 10:21:35 -08:00
commit 5c947d0dba
142 changed files with 5492 additions and 2652 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
Documentation/virt/kvm/amd-memory-encryption.rst
View file

@ -85,6 +85,12 @@ guests, such as launching, running, snapshotting, migrating and decommissioning.
The KVM_SEV_INIT command is used by the hypervisor to initialize the SEV platform
context. In a typical workflow, this command should be the first command issued.
The firmware can be initialized either by using its own non-volatile storage or
the OS can manage the NV storage for the firmware using the module parameter
``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create
a new NV storage file allocate the file with 32KB bytes of 0xFF as required by
the SEV spec.
Returns: 0 on success, -negative on error
2. KVM_SEV_LAUNCH_START

3
MAINTAINERS
View file

@ -7557,6 +7557,7 @@ F: include/video/
FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER
M: Horia Geantă <horia.geanta@nxp.com>
M: Pankaj Gupta <pankaj.gupta@nxp.com>
M: Gaurav Jain <gaurav.jain@nxp.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: Documentation/devicetree/bindings/crypto/fsl-sec4.txt
@ -8676,6 +8677,7 @@ F: drivers/scsi/hisi_sas/
HISILICON SECURITY ENGINE V2 DRIVER (SEC2)
M: Zaibo Xu <xuzaibo@huawei.com>
M: Kai Ye <yekai13@huawei.com>
L: linux-crypto@vger.kernel.org
S: Maintained
F: Documentation/ABI/testing/debugfs-hisi-sec
@ -9721,7 +9723,6 @@ F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml
F: drivers/crypto/keembay/Kconfig
F: drivers/crypto/keembay/Makefile
F: drivers/crypto/keembay/keembay-ocs-ecc.c
F: drivers/crypto/keembay/ocs-ecc-curve-defs.h
INTEL KEEM BAY OCS HCU CRYPTO DRIVER
M: Daniele Alessandrelli <daniele.alessandrelli@intel.com>

17
arch/mips/cavium-octeon/crypto/octeon-sha1.c
View file

@ -15,6 +15,7 @@
#include <linux/mm.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/module.h>
@ -71,20 +72,6 @@ static void octeon_sha1_transform(const void *_block)
octeon_sha1_start(block[7]);
}
static int octeon_sha1_init(struct shash_desc *desc)
{
struct sha1_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA1_H0;
sctx->state[1] = SHA1_H1;
sctx->state[2] = SHA1_H2;
sctx->state[3] = SHA1_H3;
sctx->state[4] = SHA1_H4;
sctx->count = 0;
return 0;
}
static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data,
unsigned int len)
{
@ -200,7 +187,7 @@ static int octeon_sha1_import(struct shash_desc *desc, const void *in)
static struct shash_alg octeon_sha1_alg = {
.digestsize = SHA1_DIGEST_SIZE,
.init = octeon_sha1_init,
.init = sha1_base_init,
.update = octeon_sha1_update,
.final = octeon_sha1_final,
.export = octeon_sha1_export,

39
arch/mips/cavium-octeon/crypto/octeon-sha256.c
View file

@ -16,6 +16,7 @@
#include <linux/mm.h>
#include <crypto/sha2.h>
#include <crypto/sha256_base.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/module.h>
@ -63,40 +64,6 @@ static void octeon_sha256_transform(const void *_block)
octeon_sha256_start(block[7]);
}
static int octeon_sha224_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA224_H0;
sctx->state[1] = SHA224_H1;
sctx->state[2] = SHA224_H2;
sctx->state[3] = SHA224_H3;
sctx->state[4] = SHA224_H4;
sctx->state[5] = SHA224_H5;
sctx->state[6] = SHA224_H6;
sctx->state[7] = SHA224_H7;
sctx->count = 0;
return 0;
}
static int octeon_sha256_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA256_H0;
sctx->state[1] = SHA256_H1;
sctx->state[2] = SHA256_H2;
sctx->state[3] = SHA256_H3;
sctx->state[4] = SHA256_H4;
sctx->state[5] = SHA256_H5;
sctx->state[6] = SHA256_H6;
sctx->state[7] = SHA256_H7;
sctx->count = 0;
return 0;
}
static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data,
unsigned int len)
{
@ -224,7 +191,7 @@ static int octeon_sha256_import(struct shash_desc *desc, const void *in)
static struct shash_alg octeon_sha256_algs[2] = { {
.digestsize = SHA256_DIGEST_SIZE,
.init = octeon_sha256_init,
.init = sha256_base_init,
.update = octeon_sha256_update,
.final = octeon_sha256_final,
.export = octeon_sha256_export,
@ -240,7 +207,7 @@ static struct shash_alg octeon_sha256_algs[2] = { {
}
}, {
.digestsize = SHA224_DIGEST_SIZE,
.init = octeon_sha224_init,
.init = sha224_base_init,
.update = octeon_sha256_update,
.final = octeon_sha224_final,
.descsize = sizeof(struct sha256_state),

39
arch/mips/cavium-octeon/crypto/octeon-sha512.c
View file

@ -15,6 +15,7 @@
#include <linux/mm.h>
#include <crypto/sha2.h>
#include <crypto/sha512_base.h>
#include <linux/init.h>
#include <linux/types.h>
#include <linux/module.h>
@ -74,40 +75,6 @@ static void octeon_sha512_transform(const void *_block)
octeon_sha512_start(block[15]);
}
static int octeon_sha512_init(struct shash_desc *desc)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA512_H0;
sctx->state[1] = SHA512_H1;
sctx->state[2] = SHA512_H2;
sctx->state[3] = SHA512_H3;
sctx->state[4] = SHA512_H4;
sctx->state[5] = SHA512_H5;
sctx->state[6] = SHA512_H6;
sctx->state[7] = SHA512_H7;
sctx->count[0] = sctx->count[1] = 0;
return 0;
}
static int octeon_sha384_init(struct shash_desc *desc)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA384_H0;
sctx->state[1] = SHA384_H1;
sctx->state[2] = SHA384_H2;
sctx->state[3] = SHA384_H3;
sctx->state[4] = SHA384_H4;
sctx->state[5] = SHA384_H5;
sctx->state[6] = SHA384_H6;
sctx->state[7] = SHA384_H7;
sctx->count[0] = sctx->count[1] = 0;
return 0;
}
static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data,
unsigned int len)
{
@ -223,7 +190,7 @@ static int octeon_sha384_final(struct shash_desc *desc, u8 *hash)
static struct shash_alg octeon_sha512_algs[2] = { {
.digestsize = SHA512_DIGEST_SIZE,
.init = octeon_sha512_init,
.init = sha512_base_init,
.update = octeon_sha512_update,
.final = octeon_sha512_final,
.descsize = sizeof(struct sha512_state),
@ -236,7 +203,7 @@ static struct shash_alg octeon_sha512_algs[2] = { {
}
}, {
.digestsize = SHA384_DIGEST_SIZE,
.init = octeon_sha384_init,
.init = sha384_base_init,
.update = octeon_sha512_update,
.final = octeon_sha384_final,
.descsize = sizeof(struct sha512_state),

17
arch/powerpc/crypto/sha1-spe-glue.c
View file

@ -13,6 +13,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <asm/byteorder.h>
#include <asm/switch_to.h>
#include <linux/hardirq.h>
@ -55,20 +56,6 @@ static inline void ppc_sha1_clear_context(struct sha1_state *sctx)
do { *ptr++ = 0; } while (--count);
}
static int ppc_spe_sha1_init(struct shash_desc *desc)
{
struct sha1_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA1_H0;
sctx->state[1] = SHA1_H1;
sctx->state[2] = SHA1_H2;
sctx->state[3] = SHA1_H3;
sctx->state[4] = SHA1_H4;
sctx->count = 0;
return 0;
}
static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@ -168,7 +155,7 @@ static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in)
static struct shash_alg alg = {
.digestsize = SHA1_DIGEST_SIZE,
.init = ppc_spe_sha1_init,
.init = sha1_base_init,
.update = ppc_spe_sha1_update,
.final = ppc_spe_sha1_final,
.export = ppc_spe_sha1_export,

14
arch/powerpc/crypto/sha1.c
View file

@ -18,21 +18,11 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <asm/byteorder.h>
void powerpc_sha_transform(u32 *state, const u8 *src);
static int powerpc_sha1_init(struct shash_desc *desc)
{
struct sha1_state *sctx = shash_desc_ctx(desc);
*sctx = (struct sha1_state){
.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
};
return 0;
}
static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@ -114,7 +104,7 @@ static int powerpc_sha1_import(struct shash_desc *desc, const void *in)
static struct shash_alg alg = {
.digestsize = SHA1_DIGEST_SIZE,
.init = powerpc_sha1_init,
.init = sha1_base_init,
.update = powerpc_sha1_update,
.final = powerpc_sha1_final,
.export = powerpc_sha1_export,

39
arch/powerpc/crypto/sha256-spe-glue.c
View file

@ -14,6 +14,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha2.h>
#include <crypto/sha256_base.h>
#include <asm/byteorder.h>
#include <asm/switch_to.h>
#include <linux/hardirq.h>
@ -56,40 +57,6 @@ static inline void ppc_sha256_clear_context(struct sha256_state *sctx)
do { *ptr++ = 0; } while (--count);
}
static int ppc_spe_sha256_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA256_H0;
sctx->state[1] = SHA256_H1;
sctx->state[2] = SHA256_H2;
sctx->state[3] = SHA256_H3;
sctx->state[4] = SHA256_H4;
sctx->state[5] = SHA256_H5;
sctx->state[6] = SHA256_H6;
sctx->state[7] = SHA256_H7;
sctx->count = 0;
return 0;
}
static int ppc_spe_sha224_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA224_H0;
sctx->state[1] = SHA224_H1;
sctx->state[2] = SHA224_H2;
sctx->state[3] = SHA224_H3;
sctx->state[4] = SHA224_H4;
sctx->state[5] = SHA224_H5;
sctx->state[6] = SHA224_H6;
sctx->state[7] = SHA224_H7;
sctx->count = 0;
return 0;
}
static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@ -214,7 +181,7 @@ static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in)
static struct shash_alg algs[2] = { {
.digestsize = SHA256_DIGEST_SIZE,
.init = ppc_spe_sha256_init,
.init = sha256_base_init,
.update = ppc_spe_sha256_update,
.final = ppc_spe_sha256_final,
.export = ppc_spe_sha256_export,
@ -230,7 +197,7 @@ static struct shash_alg algs[2] = { {
}
}, {
.digestsize = SHA224_DIGEST_SIZE,
.init = ppc_spe_sha224_init,
.init = sha224_base_init,
.update = ppc_spe_sha256_update,
.final = ppc_spe_sha224_final,
.export = ppc_spe_sha256_export,

32
arch/s390/crypto/sha512_s390.c
View file

@ -22,14 +22,14 @@ static int sha512_init(struct shash_desc *desc)
{
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
*(__u64 *)&ctx->state[0] = 0x6a09e667f3bcc908ULL;
*(__u64 *)&ctx->state[2] = 0xbb67ae8584caa73bULL;
*(__u64 *)&ctx->state[4] = 0x3c6ef372fe94f82bULL;
*(__u64 *)&ctx->state[6] = 0xa54ff53a5f1d36f1ULL;
*(__u64 *)&ctx->state[8] = 0x510e527fade682d1ULL;
*(__u64 *)&ctx->state[10] = 0x9b05688c2b3e6c1fULL;
*(__u64 *)&ctx->state[12] = 0x1f83d9abfb41bd6bULL;
*(__u64 *)&ctx->state[14] = 0x5be0cd19137e2179ULL;
*(__u64 *)&ctx->state[0] = SHA512_H0;
*(__u64 *)&ctx->state[2] = SHA512_H1;
*(__u64 *)&ctx->state[4] = SHA512_H2;
*(__u64 *)&ctx->state[6] = SHA512_H3;
*(__u64 *)&ctx->state[8] = SHA512_H4;
*(__u64 *)&ctx->state[10] = SHA512_H5;
*(__u64 *)&ctx->state[12] = SHA512_H6;
*(__u64 *)&ctx->state[14] = SHA512_H7;
ctx->count = 0;
ctx->func = CPACF_KIMD_SHA_512;
@ -87,14 +87,14 @@ static int sha384_init(struct shash_desc *desc)
{
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
*(__u64 *)&ctx->state[0] = 0xcbbb9d5dc1059ed8ULL;
*(__u64 *)&ctx->state[2] = 0x629a292a367cd507ULL;
*(__u64 *)&ctx->state[4] = 0x9159015a3070dd17ULL;
*(__u64 *)&ctx->state[6] = 0x152fecd8f70e5939ULL;
*(__u64 *)&ctx->state[8] = 0x67332667ffc00b31ULL;
*(__u64 *)&ctx->state[10] = 0x8eb44a8768581511ULL;
*(__u64 *)&ctx->state[12] = 0xdb0c2e0d64f98fa7ULL;
*(__u64 *)&ctx->state[14] = 0x47b5481dbefa4fa4ULL;
*(__u64 *)&ctx->state[0] = SHA384_H0;
*(__u64 *)&ctx->state[2] = SHA384_H1;
*(__u64 *)&ctx->state[4] = SHA384_H2;
*(__u64 *)&ctx->state[6] = SHA384_H3;
*(__u64 *)&ctx->state[8] = SHA384_H4;
*(__u64 *)&ctx->state[10] = SHA384_H5;
*(__u64 *)&ctx->state[12] = SHA384_H6;
*(__u64 *)&ctx->state[14] = SHA384_H7;
ctx->count = 0;
ctx->func = CPACF_KIMD_SHA_512;

14
arch/sparc/crypto/sha1_glue.c
View file

@ -17,6 +17,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha1.h>
#include <crypto/sha1_base.h>
#include <asm/pstate.h>
#include <asm/elf.h>
@ -26,17 +27,6 @@
asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data,
unsigned int rounds);
static int sha1_sparc64_init(struct shash_desc *desc)
{
struct sha1_state *sctx = shash_desc_ctx(desc);
*sctx = (struct sha1_state){
.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
};
return 0;
}
static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data,
unsigned int len, unsigned int partial)
{
@ -128,7 +118,7 @@ static int sha1_sparc64_import(struct shash_desc *desc, const void *in)
static struct shash_alg alg = {
.digestsize = SHA1_DIGEST_SIZE,
.init = sha1_sparc64_init,
.init = sha1_base_init,
.update = sha1_sparc64_update,
.final = sha1_sparc64_final,
.export = sha1_sparc64_export,

37
arch/sparc/crypto/sha256_glue.c
View file

@ -17,6 +17,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha2.h>
#include <crypto/sha256_base.h>
#include <asm/pstate.h>
#include <asm/elf.h>
@ -26,38 +27,6 @@
asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data,
unsigned int rounds);
static int sha224_sparc64_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA224_H0;
sctx->state[1] = SHA224_H1;
sctx->state[2] = SHA224_H2;
sctx->state[3] = SHA224_H3;
sctx->state[4] = SHA224_H4;
sctx->state[5] = SHA224_H5;
sctx->state[6] = SHA224_H6;
sctx->state[7] = SHA224_H7;
sctx->count = 0;
return 0;
}
static int sha256_sparc64_init(struct shash_desc *desc)
{
struct sha256_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA256_H0;
sctx->state[1] = SHA256_H1;
sctx->state[2] = SHA256_H2;
sctx->state[3] = SHA256_H3;
sctx->state[4] = SHA256_H4;
sctx->state[5] = SHA256_H5;
sctx->state[6] = SHA256_H6;
sctx->state[7] = SHA256_H7;
sctx->count = 0;
return 0;
}
static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data,
unsigned int len, unsigned int partial)
{
@ -158,7 +127,7 @@ static int sha256_sparc64_import(struct shash_desc *desc, const void *in)
static struct shash_alg sha256_alg = {
.digestsize = SHA256_DIGEST_SIZE,
.init = sha256_sparc64_init,
.init = sha256_base_init,
.update = sha256_sparc64_update,
.final = sha256_sparc64_final,
.export = sha256_sparc64_export,
@ -176,7 +145,7 @@ static struct shash_alg sha256_alg = {
static struct shash_alg sha224_alg = {
.digestsize = SHA224_DIGEST_SIZE,
.init = sha224_sparc64_init,
.init = sha224_base_init,
.update = sha256_sparc64_update,
.final = sha224_sparc64_final,
.descsize = sizeof(struct sha256_state),

37
arch/sparc/crypto/sha512_glue.c
View file

@ -16,6 +16,7 @@
#include <linux/mm.h>
#include <linux/types.h>
#include <crypto/sha2.h>
#include <crypto/sha512_base.h>
#include <asm/pstate.h>
#include <asm/elf.h>
@ -25,38 +26,6 @@
asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data,
unsigned int rounds);
static int sha512_sparc64_init(struct shash_desc *desc)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA512_H0;
sctx->state[1] = SHA512_H1;
sctx->state[2] = SHA512_H2;
sctx->state[3] = SHA512_H3;
sctx->state[4] = SHA512_H4;
sctx->state[5] = SHA512_H5;
sctx->state[6] = SHA512_H6;
sctx->state[7] = SHA512_H7;
sctx->count[0] = sctx->count[1] = 0;
return 0;
}
static int sha384_sparc64_init(struct shash_desc *desc)
{
struct sha512_state *sctx = shash_desc_ctx(desc);
sctx->state[0] = SHA384_H0;
sctx->state[1] = SHA384_H1;
sctx->state[2] = SHA384_H2;
sctx->state[3] = SHA384_H3;
sctx->state[4] = SHA384_H4;
sctx->state[5] = SHA384_H5;
sctx->state[6] = SHA384_H6;
sctx->state[7] = SHA384_H7;
sctx->count[0] = sctx->count[1] = 0;
return 0;
}
static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data,
unsigned int len, unsigned int partial)
{
@ -146,7 +115,7 @@ static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash)
static struct shash_alg sha512 = {
.digestsize = SHA512_DIGEST_SIZE,
.init = sha512_sparc64_init,
.init = sha512_base_init,
.update = sha512_sparc64_update,
.final = sha512_sparc64_final,
.descsize = sizeof(struct sha512_state),
@ -161,7 +130,7 @@ static struct shash_alg sha512 = {
static struct shash_alg sha384 = {
.digestsize = SHA384_DIGEST_SIZE,
.init = sha384_sparc64_init,
.init = sha384_base_init,
.update = sha512_sparc64_update,
.final = sha384_sparc64_final,
.descsize = sizeof(struct sha512_state),

4
arch/x86/crypto/aesni-intel_glue.c
View file

@ -1107,7 +1107,7 @@ static struct aead_alg aesni_aeads[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
.cra_alignmask = AESNI_ALIGN - 1,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
}, {
@ -1124,7 +1124,7 @@ static struct aead_alg aesni_aeads[] = { {
.cra_flags = CRYPTO_ALG_INTERNAL,
.cra_blocksize = 1,
.cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
.cra_alignmask = AESNI_ALIGN - 1,
.cra_alignmask = 0,
.cra_module = THIS_MODULE,
},
} };

767
arch/x86/crypto/curve25519-x86_64.c
View file

@ -64,10 +64,9 @@ static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2)
/* Return the carry bit in a register */
" adcx %%r11, %1;"
: "+&r" (f2), "=&r" (carry_r)
: "r" (out), "r" (f1)
: "%r8", "%r9", "%r10", "%r11", "memory", "cc"
);
: "+&r"(f2), "=&r"(carry_r)
: "r"(out), "r"(f1)
: "%r8", "%r9", "%r10", "%r11", "memory", "cc");
return carry_r;
}
@ -108,10 +107,9 @@ static inline void fadd(u64 *out, const u64 *f1, const u64 *f2)
" cmovc %0, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%1);"
: "+&r" (f2)
: "r" (out), "r" (f1)
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"
);
: "+&r"(f2)
: "r"(out), "r"(f1)
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
}
/* Computes the field subtraction of two field elements */
@ -151,10 +149,9 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
" movq %%r9, 8(%0);"
" movq %%r10, 16(%0);"
" movq %%r11, 24(%0);"
:
: "r" (out), "r" (f1), "r" (f2)
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"
);
:
: "r"(out), "r"(f1), "r"(f2)
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
}
/* Computes a field multiplication: out <- f1 * f2
@ -162,239 +159,400 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
{
asm volatile(
/* Compute the raw multiplication: tmp <- src1 * src2 */
/* Compute src1[0] * src2 */
" movq 0(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;"
" movq 0(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" movq %%r8, 0(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" movq %%r10, 8(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
/* Compute src1[1] * src2 */
" movq 8(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
" movq 8(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 8(%2), %%r8;"
" movq %%r8, 8(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 16(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" mov $0, %%r8;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
/* Compute src1[2] * src2 */
" movq 16(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
" movq 16(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 16(%2), %%r8;"
" movq %%r8, 16(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 24(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" mov $0, %%r8;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
/* Compute src1[3] * src2 */
" movq 24(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);"
" movq 24(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 24(%2), %%r8;"
" movq %%r8, 24(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 32(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" movq %%rbx, 40(%2);"
" mov $0, %%r8;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" movq %%r14, 48(%2);"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
" movq %%rax, 56(%2);"
/* Line up pointers */
" mov %0, %1;"
" mov %2, %0;"
" mov %3, %2;"
/* Wrap the result back into the field */
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
" xor %k3, %k3;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" mulxq 32(%0), %%r8, %%r13;"
" xor %k1, %k1;"
" adoxq 0(%0), %%r8;"
" mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
" adoxq 8(%1), %%r9;"
" mulxq 48(%1), %%r10, %%r13;"
" adoxq 8(%0), %%r9;"
" mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
" adoxq 16(%1), %%r10;"
" mulxq 56(%1), %%r11, %%rax;"
" adoxq 16(%0), %%r10;"
" mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
" adoxq 24(%1), %%r11;"
" adcx %3, %%rax;"
" adox %3, %%rax;"
" adoxq 24(%0), %%r11;"
" adcx %1, %%rax;"
" adox %1, %%rax;"
" imul %%rdx, %%rax;"
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %3, %%r9;"
" movq %%r9, 8(%0);"
" adcx %3, %%r10;"
" movq %%r10, 16(%0);"
" adcx %3, %%r11;"
" movq %%r11, 24(%0);"
" adcx %1, %%r9;"
" movq %%r9, 8(%2);"
" adcx %1, %%r10;"
" movq %%r10, 16(%2);"
" adcx %1, %%r11;"
" movq %%r11, 24(%2);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%0);"
: "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2)
:
: "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc"
);
" movq %%r8, 0(%2);"
: "+&r"(f1), "+&r"(f2), "+&r"(tmp)
: "r"(out)
: "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
"%r14", "memory", "cc");
}
/* Computes two field multiplications:
* out[0] <- f1[0] * f2[0]
* out[1] <- f1[1] * f2[1]
* Uses the 16-element buffer tmp for intermediate results. */
* out[0] <- f1[0] * f2[0]
* out[1] <- f1[1] * f2[1]
* Uses the 16-element buffer tmp for intermediate results: */
static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
{
asm volatile(
/* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */
/* Compute src1[0] * src2 */
" movq 0(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;"
" movq 0(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" movq %%r8, 0(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" movq %%r10, 8(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
/* Compute src1[1] * src2 */
" movq 8(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
" movq 8(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 8(%2), %%r8;"
" movq %%r8, 8(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 16(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" mov $0, %%r8;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
/* Compute src1[2] * src2 */
" movq 16(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
" movq 16(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 16(%2), %%r8;"
" movq %%r8, 16(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 24(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" mov $0, %%r8;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
/* Compute src1[3] * src2 */
" movq 24(%1), %%rdx;"
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);"
" movq 24(%0), %%rdx;"
" mulxq 0(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 24(%2), %%r8;"
" movq %%r8, 24(%2);"
" mulxq 8(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 32(%2);"
" mulxq 16(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" movq %%rbx, 40(%2);"
" mov $0, %%r8;"
" mulxq 24(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" movq %%r14, 48(%2);"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
" movq %%rax, 56(%2);"
/* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */
/* Compute src1[0] * src2 */
" movq 32(%1), %%rdx;"
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);"
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;"
" movq 32(%0), %%rdx;"
" mulxq 32(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" movq %%r8, 64(%2);"
" mulxq 40(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" movq %%r10, 72(%2);"
" mulxq 48(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" mulxq 56(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
/* Compute src1[1] * src2 */
" movq 40(%1), %%rdx;"
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);"
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
" movq 40(%0), %%rdx;"
" mulxq 32(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 72(%2), %%r8;"
" movq %%r8, 72(%2);"
" mulxq 40(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 80(%2);"
" mulxq 48(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" mov $0, %%r8;"
" mulxq 56(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
/* Compute src1[2] * src2 */
" movq 48(%1), %%rdx;"
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);"
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
" movq 48(%0), %%rdx;"
" mulxq 32(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 80(%2), %%r8;"
" movq %%r8, 80(%2);"
" mulxq 40(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 88(%2);"
" mulxq 48(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" mov $0, %%r8;"
" mulxq 56(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
/* Compute src1[3] * src2 */
" movq 56(%1), %%rdx;"
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);"
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);"
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;"
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;"
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);"
" movq 56(%0), %%rdx;"
" mulxq 32(%1), %%r8, %%r9;"
" xor %%r10d, %%r10d;"
" adcxq 88(%2), %%r8;"
" movq %%r8, 88(%2);"
" mulxq 40(%1), %%r10, %%r11;"
" adox %%r9, %%r10;"
" adcx %%rbx, %%r10;"
" movq %%r10, 96(%2);"
" mulxq 48(%1), %%rbx, %%r13;"
" adox %%r11, %%rbx;"
" adcx %%r14, %%rbx;"
" movq %%rbx, 104(%2);"
" mov $0, %%r8;"
" mulxq 56(%1), %%r14, %%rdx;"
" adox %%r13, %%r14;"
" adcx %%rax, %%r14;"
" movq %%r14, 112(%2);"
" mov $0, %%rax;"
" adox %%rdx, %%rax;"
" adcx %%r8, %%rax;"
" movq %%rax, 120(%2);"
/* Line up pointers */
" mov %0, %1;"
" mov %2, %0;"
" mov %3, %2;"
/* Wrap the results back into the field */
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
" xor %k3, %k3;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" mulxq 32(%0), %%r8, %%r13;"
" xor %k1, %k1;"
" adoxq 0(%0), %%r8;"
" mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
" adoxq 8(%1), %%r9;"
" mulxq 48(%1), %%r10, %%r13;"
" adoxq 8(%0), %%r9;"
" mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
" adoxq 16(%1), %%r10;"
" mulxq 56(%1), %%r11, %%rax;"
" adoxq 16(%0), %%r10;"
" mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
" adoxq 24(%1), %%r11;"
" adcx %3, %%rax;"
" adox %3, %%rax;"
" adoxq 24(%0), %%r11;"
" adcx %1, %%rax;"
" adox %1, %%rax;"
" imul %%rdx, %%rax;"
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %3, %%r9;"
" movq %%r9, 8(%0);"
" adcx %3, %%r10;"
" movq %%r10, 16(%0);"
" adcx %3, %%r11;"
" movq %%r11, 24(%0);"
" adcx %1, %%r9;"
" movq %%r9, 8(%2);"
" adcx %1, %%r10;"
" movq %%r10, 16(%2);"
" adcx %1, %%r11;"
" movq %%r11, 24(%2);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%0);"
" movq %%r8, 0(%2);"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 96(%1), %%r8, %%r13;"
" xor %k3, %k3;"
" adoxq 64(%1), %%r8;"
" mulxq 104(%1), %%r9, %%rbx;"
" mulxq 96(%0), %%r8, %%r13;"
" xor %k1, %k1;"
" adoxq 64(%0), %%r8;"
" mulxq 104(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
" adoxq 72(%1), %%r9;"
" mulxq 112(%1), %%r10, %%r13;"
" adoxq 72(%0), %%r9;"
" mulxq 112(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
" adoxq 80(%1), %%r10;"
" mulxq 120(%1), %%r11, %%rax;"
" adoxq 80(%0), %%r10;"
" mulxq 120(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
" adoxq 88(%1), %%r11;"
" adcx %3, %%rax;"
" adox %3, %%rax;"
" adoxq 88(%0), %%r11;"
" adcx %1, %%rax;"
" adox %1, %%rax;"
" imul %%rdx, %%rax;"
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %3, %%r9;"
" movq %%r9, 40(%0);"
" adcx %3, %%r10;"
" movq %%r10, 48(%0);"
" adcx %3, %%r11;"
" movq %%r11, 56(%0);"
" adcx %1, %%r9;"
" movq %%r9, 40(%2);"
" adcx %1, %%r10;"
" movq %%r10, 48(%2);"
" adcx %1, %%r11;"
" movq %%r11, 56(%2);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 32(%0);"
: "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2)
:
: "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc"
);
" movq %%r8, 32(%2);"
: "+&r"(f1), "+&r"(f2), "+&r"(tmp)
: "r"(out)
: "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
"%r14", "memory", "cc");
}
/* Computes the field multiplication of four-element f1 with value in f2 */
/* Computes the field multiplication of four-element f1 with value in f2
* Requires f2 to be smaller than 2^17 */
static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
{
register u64 f2_r asm("rdx") = f2;
asm volatile(
/* Compute the raw multiplication of f1*f2 */
" mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
" mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
" mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
" mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
" add %%rcx, %%r9;"
" mov $0, %%rcx;"
" mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
" mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
" adcx %%rbx, %%r10;"
" mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
" mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
" adcx %%r13, %%r11;"
" adcx %%rcx, %%rax;"
@ -418,17 +576,17 @@ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%1);"
: "+&r" (f2_r)
: "r" (out), "r" (f1)
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc"
);
: "+&r"(f2_r)
: "r"(out), "r"(f1)
: "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13",
"memory", "cc");
}
/* Computes p1 <- bit ? p2 : p1 in constant time */
static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
{
asm volatile(
/* Invert the polarity of bit to match cmov expectations */
/* Transfer bit into CF flag */
" add $18446744073709551615, %0;"
/* cswap p1[0], p2[0] */
@ -502,10 +660,9 @@ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
" cmovc %%r10, %%r9;"
" movq %%r8, 56(%1);"
" movq %%r9, 56(%2);"
: "+&r" (bit)
: "r" (p1), "r" (p2)
: "%r8", "%r9", "%r10", "memory", "cc"
);
: "+&r"(bit)
: "r"(p1), "r"(p2)
: "%r8", "%r9", "%r10", "memory", "cc");
}
/* Computes the square of a field element: out <- f * f
@ -516,15 +673,22 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
/* Compute the raw multiplication: tmp <- f * f */
/* Step 1: Compute all partial products */
" movq 0(%1), %%rdx;" /* f[0] */
" mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 24(%1), %%rdx;" /* f[3] */
" mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
" mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
" movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
" mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
" movq 0(%0), %%rdx;" /* f[0] */
" mulxq 8(%0), %%r8, %%r14;"
" xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 16(%0), %%r9, %%r10;"
" adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 24(%0), %%rax, %%rcx;"
" adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 24(%0), %%rdx;" /* f[3] */
" mulxq 8(%0), %%r11, %%rbx;"
" adcx %%rcx, %%r11;" /* f[1]*f[3] */
" mulxq 16(%0), %%rax, %%r13;"
" adcx %%rax, %%rbx;" /* f[2]*f[3] */
" movq 8(%0), %%rdx;"
" adcx %%r15, %%r13;" /* f1 */
" mulxq 16(%0), %%rax, %%rcx;"
" mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
" xor %%r15d, %%r15d;"
@ -542,39 +706,50 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
" adcx %%r14, %%r14;"
/* Step 3: Compute intermediate squares */
" movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
" movq %%rax, 0(%0);"
" add %%rcx, %%r8;" " movq %%r8, 8(%0);"
" movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
" adcx %%rax, %%r9;" " movq %%r9, 16(%0);"
" adcx %%rcx, %%r10;" " movq %%r10, 24(%0);"
" movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
" adcx %%rax, %%r11;" " movq %%r11, 32(%0);"
" adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);"
" movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
" adcx %%rax, %%r13;" " movq %%r13, 48(%0);"
" adcx %%rcx, %%r14;" " movq %%r14, 56(%0);"
" movq 0(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
" movq %%rax, 0(%1);"
" add %%rcx, %%r8;"
" movq %%r8, 8(%1);"
" movq 8(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
" adcx %%rax, %%r9;"
" movq %%r9, 16(%1);"
" adcx %%rcx, %%r10;"
" movq %%r10, 24(%1);"
" movq 16(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
" adcx %%rax, %%r11;"
" movq %%r11, 32(%1);"
" adcx %%rcx, %%rbx;"
" movq %%rbx, 40(%1);"
" movq 24(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
" adcx %%rax, %%r13;"
" movq %%r13, 48(%1);"
" adcx %%rcx, %%r14;"
" movq %%r14, 56(%1);"
/* Line up pointers */
" mov %0, %1;"
" mov %2, %0;"
" mov %1, %0;"
" mov %2, %1;"
/* Wrap the result back into the field */
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
" mulxq 32(%0), %%r8, %%r13;"
" xor %%ecx, %%ecx;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" adoxq 0(%0), %%r8;"
" mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
" adoxq 8(%1), %%r9;"
" mulxq 48(%1), %%r10, %%r13;"
" adoxq 8(%0), %%r9;"
" mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
" adoxq 16(%1), %%r10;"
" mulxq 56(%1), %%r11, %%rax;"
" adoxq 16(%0), %%r10;"
" mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
" adoxq 24(%1), %%r11;"
" adoxq 24(%0), %%r11;"
" adcx %%rcx, %%rax;"
" adox %%rcx, %%rax;"
" imul %%rdx, %%rax;"
@ -582,40 +757,47 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
" movq %%r9, 8(%0);"
" movq %%r9, 8(%1);"
" adcx %%rcx, %%r10;"
" movq %%r10, 16(%0);"
" movq %%r10, 16(%1);"
" adcx %%rcx, %%r11;"
" movq %%r11, 24(%0);"
" movq %%r11, 24(%1);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%0);"
: "+&r" (tmp), "+&r" (f), "+&r" (out)
:
: "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc"
);
" movq %%r8, 0(%1);"
: "+&r"(f), "+&r"(tmp)
: "r"(out)
: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
"%r13", "%r14", "%r15", "memory", "cc");
}
/* Computes two field squarings:
* out[0] <- f[0] * f[0]
* out[1] <- f[1] * f[1]
* out[0] <- f[0] * f[0]
* out[1] <- f[1] * f[1]
* Uses the 16-element buffer tmp for intermediate results */
static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
{
asm volatile(
/* Step 1: Compute all partial products */
" movq 0(%1), %%rdx;" /* f[0] */
" mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 24(%1), %%rdx;" /* f[3] */
" mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
" mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
" movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
" mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
" movq 0(%0), %%rdx;" /* f[0] */
" mulxq 8(%0), %%r8, %%r14;"
" xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 16(%0), %%r9, %%r10;"
" adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 24(%0), %%rax, %%rcx;"
" adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 24(%0), %%rdx;" /* f[3] */
" mulxq 8(%0), %%r11, %%rbx;"
" adcx %%rcx, %%r11;" /* f[1]*f[3] */
" mulxq 16(%0), %%rax, %%r13;"
" adcx %%rax, %%rbx;" /* f[2]*f[3] */
" movq 8(%0), %%rdx;"
" adcx %%r15, %%r13;" /* f1 */
" mulxq 16(%0), %%rax, %%rcx;"
" mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
" xor %%r15d, %%r15d;"
@ -633,29 +815,47 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
" adcx %%r14, %%r14;"
/* Step 3: Compute intermediate squares */
" movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
" movq %%rax, 0(%0);"
" add %%rcx, %%r8;" " movq %%r8, 8(%0);"
" movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
" adcx %%rax, %%r9;" " movq %%r9, 16(%0);"
" adcx %%rcx, %%r10;" " movq %%r10, 24(%0);"
" movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
" adcx %%rax, %%r11;" " movq %%r11, 32(%0);"
" adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);"
" movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
" adcx %%rax, %%r13;" " movq %%r13, 48(%0);"
" adcx %%rcx, %%r14;" " movq %%r14, 56(%0);"
" movq 0(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
" movq %%rax, 0(%1);"
" add %%rcx, %%r8;"
" movq %%r8, 8(%1);"
" movq 8(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
" adcx %%rax, %%r9;"
" movq %%r9, 16(%1);"
" adcx %%rcx, %%r10;"
" movq %%r10, 24(%1);"
" movq 16(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
" adcx %%rax, %%r11;"
" movq %%r11, 32(%1);"
" adcx %%rcx, %%rbx;"
" movq %%rbx, 40(%1);"
" movq 24(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
" adcx %%rax, %%r13;"
" movq %%r13, 48(%1);"
" adcx %%rcx, %%r14;"
" movq %%r14, 56(%1);"
/* Step 1: Compute all partial products */
" movq 32(%1), %%rdx;" /* f[0] */
" mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 56(%1), %%rdx;" /* f[3] */
" mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
" mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
" movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
" mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
" movq 32(%0), %%rdx;" /* f[0] */
" mulxq 40(%0), %%r8, %%r14;"
" xor %%r15d, %%r15d;" /* f[1]*f[0] */
" mulxq 48(%0), %%r9, %%r10;"
" adcx %%r14, %%r9;" /* f[2]*f[0] */
" mulxq 56(%0), %%rax, %%rcx;"
" adcx %%rax, %%r10;" /* f[3]*f[0] */
" movq 56(%0), %%rdx;" /* f[3] */
" mulxq 40(%0), %%r11, %%rbx;"
" adcx %%rcx, %%r11;" /* f[1]*f[3] */
" mulxq 48(%0), %%rax, %%r13;"
" adcx %%rax, %%rbx;" /* f[2]*f[3] */
" movq 40(%0), %%rdx;"
" adcx %%r15, %%r13;" /* f1 */
" mulxq 48(%0), %%rax, %%rcx;"
" mov $0, %%r14;" /* f[2]*f[1] */
/* Step 2: Compute two parallel carry chains */
" xor %%r15d, %%r15d;"
@ -673,37 +873,48 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
" adcx %%r14, %%r14;"
/* Step 3: Compute intermediate squares */
" movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
" movq %%rax, 64(%0);"
" add %%rcx, %%r8;" " movq %%r8, 72(%0);"
" movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
" adcx %%rax, %%r9;" " movq %%r9, 80(%0);"
" adcx %%rcx, %%r10;" " movq %%r10, 88(%0);"
" movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
" adcx %%rax, %%r11;" " movq %%r11, 96(%0);"
" adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);"
" movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
" adcx %%rax, %%r13;" " movq %%r13, 112(%0);"
" adcx %%rcx, %%r14;" " movq %%r14, 120(%0);"
" movq 32(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
" movq %%rax, 64(%1);"
" add %%rcx, %%r8;"
" movq %%r8, 72(%1);"
" movq 40(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
" adcx %%rax, %%r9;"
" movq %%r9, 80(%1);"
" adcx %%rcx, %%r10;"
" movq %%r10, 88(%1);"
" movq 48(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
" adcx %%rax, %%r11;"
" movq %%r11, 96(%1);"
" adcx %%rcx, %%rbx;"
" movq %%rbx, 104(%1);"
" movq 56(%0), %%rdx;"
" mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
" adcx %%rax, %%r13;"
" movq %%r13, 112(%1);"
" adcx %%rcx, %%r14;"
" movq %%r14, 120(%1);"
/* Line up pointers */
" mov %0, %1;"
" mov %2, %0;"
" mov %1, %0;"
" mov %2, %1;"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 32(%1), %%r8, %%r13;"
" mulxq 32(%0), %%r8, %%r13;"
" xor %%ecx, %%ecx;"
" adoxq 0(%1), %%r8;"
" mulxq 40(%1), %%r9, %%rbx;"
" adoxq 0(%0), %%r8;"
" mulxq 40(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
" adoxq 8(%1), %%r9;"
" mulxq 48(%1), %%r10, %%r13;"
" adoxq 8(%0), %%r9;"
" mulxq 48(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
" adoxq 16(%1), %%r10;"
" mulxq 56(%1), %%r11, %%rax;"
" adoxq 16(%0), %%r10;"
" mulxq 56(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
" adoxq 24(%1), %%r11;"
" adoxq 24(%0), %%r11;"
" adcx %%rcx, %%rax;"
" adox %%rcx, %%rax;"
" imul %%rdx, %%rax;"
@ -711,32 +922,32 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
" movq %%r9, 8(%0);"
" movq %%r9, 8(%1);"
" adcx %%rcx, %%r10;"
" movq %%r10, 16(%0);"
" movq %%r10, 16(%1);"
" adcx %%rcx, %%r11;"
" movq %%r11, 24(%0);"
" movq %%r11, 24(%1);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 0(%0);"
" movq %%r8, 0(%1);"
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
" mov $38, %%rdx;"
" mulxq 96(%1), %%r8, %%r13;"
" mulxq 96(%0), %%r8, %%r13;"
" xor %%ecx, %%ecx;"
" adoxq 64(%1), %%r8;"
" mulxq 104(%1), %%r9, %%rbx;"
" adoxq 64(%0), %%r8;"
" mulxq 104(%0), %%r9, %%rbx;"
" adcx %%r13, %%r9;"
" adoxq 72(%1), %%r9;"
" mulxq 112(%1), %%r10, %%r13;"
" adoxq 72(%0), %%r9;"
" mulxq 112(%0), %%r10, %%r13;"
" adcx %%rbx, %%r10;"
" adoxq 80(%1), %%r10;"
" mulxq 120(%1), %%r11, %%rax;"
" adoxq 80(%0), %%r10;"
" mulxq 120(%0), %%r11, %%rax;"
" adcx %%r13, %%r11;"
" adoxq 88(%1), %%r11;"
" adoxq 88(%0), %%r11;"
" adcx %%rcx, %%rax;"
" adox %%rcx, %%rax;"
" imul %%rdx, %%rax;"
@ -744,21 +955,21 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
/* Step 2: Fold the carry back into dst */
" add %%rax, %%r8;"
" adcx %%rcx, %%r9;"
" movq %%r9, 40(%0);"
" movq %%r9, 40(%1);"
" adcx %%rcx, %%r10;"
" movq %%r10, 48(%0);"
" movq %%r10, 48(%1);"
" adcx %%rcx, %%r11;"
" movq %%r11, 56(%0);"
" movq %%r11, 56(%1);"
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
" mov $0, %%rax;"
" cmovc %%rdx, %%rax;"
" add %%rax, %%r8;"
" movq %%r8, 32(%0);"
: "+&r" (tmp), "+&r" (f), "+&r" (out)
:
: "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc"
);
" movq %%r8, 32(%1);"
: "+&r"(f), "+&r"(tmp)
: "r"(out)
: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
"%r13", "%r14", "%r15", "memory", "cc");
}
static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2)

4
arch/x86/crypto/des3_ede_glue.c
View file

@ -164,7 +164,7 @@ static int cbc_encrypt(struct skcipher_request *req)
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes)) {
while (walk.nbytes) {
nbytes = __cbc_encrypt(ctx, &walk);
err = skcipher_walk_done(&walk, nbytes);
}
@ -243,7 +243,7 @@ static int cbc_decrypt(struct skcipher_request *req)
err = skcipher_walk_virt(&walk, req, false);
while ((nbytes = walk.nbytes)) {
while (walk.nbytes) {
nbytes = __cbc_decrypt(ctx, &walk);
err = skcipher_walk_done(&walk, nbytes);
}

4
crypto/Kconfig
View file

@ -1845,6 +1845,10 @@ config CRYPTO_JITTERENTROPY
random numbers. This Jitterentropy RNG registers with
the kernel crypto API and can be used by any caller.
config CRYPTO_KDF800108_CTR
tristate
select CRYPTO_SHA256
config CRYPTO_USER_API
tristate

5
crypto/Makefile
View file

@ -200,3 +200,8 @@ obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
crypto_simd-y := simd.o
obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
#
# Key derivation function
#
obj-$(CONFIG_CRYPTO_KDF800108_CTR) += kdf_sp800108.o

7
crypto/af_alg.c
View file

@ -931,16 +931,19 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
sg_unmark_end(sg + sgl->cur - 1);
do {
struct page *pg;
unsigned int i = sgl->cur;
plen = min_t(size_t, len, PAGE_SIZE);
sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
if (!sg_page(sg + i)) {
pg = alloc_page(GFP_KERNEL);
if (!pg) {
err = -ENOMEM;
goto unlock;
}
sg_assign_page(sg + i, pg);
err = memcpy_from_msg(page_address(sg_page(sg + i)),
msg, plen);
if (err) {

5
crypto/dh.c
View file

@ -5,11 +5,11 @@
* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
*/
#include <linux/fips.h>
#include <linux/module.h>
#include <crypto/internal/kpp.h>
#include <crypto/kpp.h>
#include <crypto/dh.h>
#include <linux/fips.h>
#include <linux/mpi.h>
struct dh_ctx {
@ -47,6 +47,9 @@ static inline struct dh_ctx *dh_get_ctx(struct crypto_kpp *tfm)
static int dh_check_params_length(unsigned int p_len)
{
if (fips_enabled)
return (p_len < 2048) ? -EINVAL : 0;
return (p_len < 1536) ? -EINVAL : 0;
}

154
crypto/drbg.c
View file

@ -100,6 +100,7 @@
#include <crypto/drbg.h>
#include <crypto/internal/cipher.h>
#include <linux/kernel.h>
#include <linux/jiffies.h>
/***************************************************************
* Backend cipher definitions available to DRBG
@ -1036,17 +1037,39 @@ static const struct drbg_state_ops drbg_hash_ops = {
******************************************************************/
static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
int reseed)
int reseed, enum drbg_seed_state new_seed_state)
{
int ret = drbg->d_ops->update(drbg, seed, reseed);
if (ret)
return ret;
drbg->seeded = true;
drbg->seeded = new_seed_state;
drbg->last_seed_time = jiffies;
/* 10.1.1.2 / 10.1.1.3 step 5 */
drbg->reseed_ctr = 1;
switch (drbg->seeded) {
case DRBG_SEED_STATE_UNSEEDED:
/* Impossible, but handle it to silence compiler warnings. */
fallthrough;
case DRBG_SEED_STATE_PARTIAL:
/*
* Require frequent reseeds until the seed source is
* fully initialized.
*/
drbg->reseed_threshold = 50;
break;
case DRBG_SEED_STATE_FULL:
/*
* Seed source has become fully initialized, frequent
* reseeds no longer required.
*/
drbg->reseed_threshold = drbg_max_requests(drbg);
break;
}
return ret;
}
@ -1066,12 +1089,10 @@ static inline int drbg_get_random_bytes(struct drbg_state *drbg,
return 0;
}
static void drbg_async_seed(struct work_struct *work)
static int drbg_seed_from_random(struct drbg_state *drbg)
{
struct drbg_string data;
LIST_HEAD(seedlist);
struct drbg_state *drbg = container_of(work, struct drbg_state,
seed_work);
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
unsigned char entropy[32];
int ret;
@ -1082,26 +1103,35 @@ static void drbg_async_seed(struct work_struct *work)
drbg_string_fill(&data, entropy, entropylen);
list_add_tail(&data.list, &seedlist);
mutex_lock(&drbg->drbg_mutex);
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
if (ret)
goto unlock;
goto out;
/* Set seeded to false so that if __drbg_seed fails the
* next generate call will trigger a reseed.
*/
drbg->seeded = false;
__drbg_seed(drbg, &seedlist, true);
if (drbg->seeded)
drbg->reseed_threshold = drbg_max_requests(drbg);
unlock:
mutex_unlock(&drbg->drbg_mutex);
ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL);
out:
memzero_explicit(entropy, entropylen);
return ret;
}
static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg)
{
unsigned long next_reseed;
/* Don't ever reseed from get_random_bytes() in test mode. */
if (list_empty(&drbg->test_data.list))
return false;
/*
* Obtain fresh entropy for the nopr DRBGs after 300s have
* elapsed in order to still achieve sort of partial
* prediction resistance over the time domain at least. Note
* that the period of 300s has been chosen to match the
* CRNG_RESEED_INTERVAL of the get_random_bytes()' chacha
* rngs.
*/
next_reseed = drbg->last_seed_time + 300 * HZ;
return time_after(jiffies, next_reseed);
}
/*
@ -1123,6 +1153,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
struct drbg_string data1;
LIST_HEAD(seedlist);
enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL;
/* 9.1 / 9.2 / 9.3.1 step 3 */
if (pers && pers->len > (drbg_max_addtl(drbg))) {
@ -1150,6 +1181,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
BUG_ON((entropylen * 2) > sizeof(entropy));
/* Get seed from in-kernel /dev/urandom */
if (!rng_is_initialized())
new_seed_state = DRBG_SEED_STATE_PARTIAL;
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
if (ret)
goto out;
@ -1159,11 +1193,14 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
pr_devel("DRBG: (re)seeding with %u bytes of entropy\n",
entropylen);
} else {
/* Get seed from Jitter RNG */
/*
* Get seed from Jitter RNG, failures are
* fatal only in FIPS mode.
*/
ret = crypto_rng_get_bytes(drbg->jent,
entropy + entropylen,
entropylen);
if (ret) {
if (fips_enabled && ret) {
pr_devel("DRBG: jent failed with %d\n", ret);
/*
@ -1206,7 +1243,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
memset(drbg->C, 0, drbg_statelen(drbg));
}
ret = __drbg_seed(drbg, &seedlist, reseed);
ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state);
out:
memzero_explicit(entropy, entropylen * 2);
@ -1386,19 +1423,26 @@ static int drbg_generate(struct drbg_state *drbg,
* here. The spec is a bit convoluted here, we make it simpler.
*/
if (drbg->reseed_threshold < drbg->reseed_ctr)
drbg->seeded = false;
drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
if (drbg->pr || !drbg->seeded) {
if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) {
pr_devel("DRBG: reseeding before generation (prediction "
"resistance: %s, state %s)\n",
drbg->pr ? "true" : "false",
drbg->seeded ? "seeded" : "unseeded");
(drbg->seeded == DRBG_SEED_STATE_FULL ?
"seeded" : "unseeded"));
/* 9.3.1 steps 7.1 through 7.3 */
len = drbg_seed(drbg, addtl, true);
if (len)
goto err;
/* 9.3.1 step 7.4 */
addtl = NULL;
} else if (rng_is_initialized() &&
(drbg->seeded == DRBG_SEED_STATE_PARTIAL ||
drbg_nopr_reseed_interval_elapsed(drbg))) {
len = drbg_seed_from_random(drbg);
if (len)
goto err;
}
if (addtl && 0 < addtl->len)
@ -1491,51 +1535,23 @@ static int drbg_generate_long(struct drbg_state *drbg,
return 0;
}
static void drbg_schedule_async_seed(struct random_ready_callback *rdy)
{
struct drbg_state *drbg = container_of(rdy, struct drbg_state,
random_ready);
schedule_work(&drbg->seed_work);
}
static int drbg_prepare_hrng(struct drbg_state *drbg)
{
int err;
/* We do not need an HRNG in test mode. */
if (list_empty(&drbg->test_data.list))
return 0;
drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0);
if (IS_ERR(drbg->jent)) {
const int err = PTR_ERR(drbg->jent);
INIT_WORK(&drbg->seed_work, drbg_async_seed);
drbg->random_ready.owner = THIS_MODULE;
drbg->random_ready.func = drbg_schedule_async_seed;
err = add_random_ready_callback(&drbg->random_ready);
switch (err) {
case 0:
break;
case -EALREADY:
err = 0;
fallthrough;
default:
drbg->random_ready.func = NULL;
return err;
drbg->jent = NULL;
if (fips_enabled || err != -ENOENT)
return err;
pr_info("DRBG: Continuing without Jitter RNG\n");
}
/*
* Require frequent reseeds until the seed source is fully
* initialized.
*/
drbg->reseed_threshold = 50;
return err;
return 0;
}
/*
@ -1578,7 +1594,8 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
if (!drbg->core) {
drbg->core = &drbg_cores[coreref];
drbg->pr = pr;
drbg->seeded = false;
drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
drbg->last_seed_time = 0;
drbg->reseed_threshold = drbg_max_requests(drbg);
ret = drbg_alloc_state(drbg);
@ -1589,14 +1606,6 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
if (ret)
goto free_everything;
if (IS_ERR(drbg->jent)) {
ret = PTR_ERR(drbg->jent);
drbg->jent = NULL;
if (fips_enabled || ret != -ENOENT)
goto free_everything;
pr_info("DRBG: Continuing without Jitter RNG\n");
}
reseed = false;
}
@ -1629,11 +1638,6 @@ free_everything:
*/
static int drbg_uninstantiate(struct drbg_state *drbg)
{
if (drbg->random_ready.func) {
del_random_ready_callback(&drbg->random_ready);
cancel_work_sync(&drbg->seed_work);
}
if (!IS_ERR_OR_NULL(drbg->jent))
crypto_free_rng(drbg->jent);
drbg->jent = NULL;

6
crypto/jitterentropy-kcapi.c
View file

@ -40,7 +40,6 @@
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/fips.h>
#include <linux/time.h>
#include <crypto/internal/rng.h>
@ -60,11 +59,6 @@ void jent_zfree(void *ptr)
kfree_sensitive(ptr);
}
int jent_fips_enabled(void)
{
return fips_enabled;
}
void jent_panic(char *s)
{
panic("%s", s);

32
crypto/jitterentropy.c
View file

@ -117,6 +117,22 @@ struct rand_data {
#define JENT_EHEALTH 9 /* Health test failed during initialization */
#define JENT_ERCT 10 /* RCT failed during initialization */
/*
* The output n bits can receive more than n bits of min entropy, of course,
* but the fixed output of the conditioning function can only asymptotically
* approach the output size bits of min entropy, not attain that bound. Random
* maps will tend to have output collisions, which reduces the creditable
* output entropy (that is what SP 800-90B Section 3.1.5.1.2 attempts to bound).
*
* The value "64" is justified in Appendix A.4 of the current 90C draft,
* and aligns with NIST's in "epsilon" definition in this document, which is
* that a string can be considered "full entropy" if you can bound the min
* entropy in each bit of output to at least 1-epsilon, where epsilon is
* required to be <= 2^(-32).
*/
#define JENT_ENTROPY_SAFETY_FACTOR 64
#include <linux/fips.h>
#include "jitterentropy.h"
/***************************************************************************
@ -265,7 +281,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
{
__u64 delta2 = jent_delta(ec->last_delta, current_delta);
__u64 delta3 = jent_delta(ec->last_delta2, delta2);
unsigned int delta_masked = current_delta & JENT_APT_WORD_MASK;
ec->last_delta = current_delta;
ec->last_delta2 = delta2;
@ -274,7 +289,7 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
* Insert the result of the comparison of two back-to-back time
* deltas.
*/
jent_apt_insert(ec, delta_masked);
jent_apt_insert(ec, current_delta);
if (!current_delta || !delta2 || !delta3) {
/* RCT with a stuck bit */
@ -299,10 +314,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
*/
static int jent_health_failure(struct rand_data *ec)
{
/* Test is only enabled in FIPS mode */
if (!jent_fips_enabled())
return 0;
return ec->health_failure;
}
@ -547,12 +558,15 @@ static int jent_measure_jitter(struct rand_data *ec)
*/
static void jent_gen_entropy(struct rand_data *ec)
{
unsigned int k = 0;
unsigned int k = 0, safety_factor = 0;
if (fips_enabled)
safety_factor = JENT_ENTROPY_SAFETY_FACTOR;
/* priming of the ->prev_time value */
jent_measure_jitter(ec);
while (1) {
while (!jent_health_failure(ec)) {
/* If a stuck measurement is received, repeat measurement */
if (jent_measure_jitter(ec))
continue;
@ -561,7 +575,7 @@ static void jent_gen_entropy(struct rand_data *ec)
* We multiply the loop value with ->osr to obtain the
* oversampling rate requested by the caller
*/
if (++k >= (DATA_SIZE_BITS * ec->osr))
if (++k >= ((DATA_SIZE_BITS + safety_factor) * ec->osr))
break;
}
}

1
crypto/jitterentropy.h
View file

@ -2,7 +2,6 @@
extern void *jent_zalloc(unsigned int len);
extern void jent_zfree(void *ptr);
extern int jent_fips_enabled(void);
extern void jent_panic(char *s);
extern void jent_memcpy(void *dest, const void *src, unsigned int n);
extern void jent_get_nstime(__u64 *out);

153
crypto/kdf_sp800108.c Normal file
View file

@ -0,0 +1,153 @@
// SPDX-License-Identifier: GPL-2.0
/*
* SP800-108 Key-derivation function
*
* Copyright (C) 2021, Stephan Mueller <smueller@chronox.de>
*/
#include <linux/fips.h>
#include <linux/module.h>
#include <crypto/kdf_sp800108.h>
#include <crypto/internal/kdf_selftest.h>
/*
* SP800-108 CTR KDF implementation
*/
int crypto_kdf108_ctr_generate(struct crypto_shash *kmd,
const struct kvec *info, unsigned int info_nvec,
u8 *dst, unsigned int dlen)
{
SHASH_DESC_ON_STACK(desc, kmd);
__be32 counter = cpu_to_be32(1);
const unsigned int h = crypto_shash_digestsize(kmd), dlen_orig = dlen;
unsigned int i;
int err = 0;
u8 *dst_orig = dst;
desc->tfm = kmd;
while (dlen) {
err = crypto_shash_init(desc);
if (err)
goto out;
err = crypto_shash_update(desc, (u8 *)&counter, sizeof(__be32));
if (err)
goto out;
for (i = 0; i < info_nvec; i++) {
err = crypto_shash_update(desc, info[i].iov_base,
info[i].iov_len);
if (err)
goto out;
}
if (dlen < h) {
u8 tmpbuffer[HASH_MAX_DIGESTSIZE];
err = crypto_shash_final(desc, tmpbuffer);
if (err)
goto out;
memcpy(dst, tmpbuffer, dlen);
memzero_explicit(tmpbuffer, h);
goto out;
}
err = crypto_shash_final(desc, dst);
if (err)
goto out;
dlen -= h;
dst += h;
counter = cpu_to_be32(be32_to_cpu(counter) + 1);
}
out:
if (err)
memzero_explicit(dst_orig, dlen_orig);
shash_desc_zero(desc);
return err;
}
EXPORT_SYMBOL(crypto_kdf108_ctr_generate);
/*
* The seeding of the KDF
*/
int crypto_kdf108_setkey(struct crypto_shash *kmd,
const u8 *key, size_t keylen,
const u8 *ikm, size_t ikmlen)
{
unsigned int ds = crypto_shash_digestsize(kmd);
/* SP800-108 does not support IKM */
if (ikm || ikmlen)
return -EINVAL;
/* Check according to SP800-108 section 7.2 */
if (ds > keylen)
return -EINVAL;
/* Set the key for the MAC used for the KDF. */
return crypto_shash_setkey(kmd, key, keylen);
}
EXPORT_SYMBOL(crypto_kdf108_setkey);
/*
* Test vector obtained from
* http://csrc.nist.gov/groups/STM/cavp/documents/KBKDF800-108/CounterMode.zip
*/
static const struct kdf_testvec kdf_ctr_hmac_sha256_tv_template[] = {
{
.key = "\xdd\x1d\x91\xb7\xd9\x0b\x2b\xd3"
"\x13\x85\x33\xce\x92\xb2\x72\xfb"
"\xf8\xa3\x69\x31\x6a\xef\xe2\x42"
"\xe6\x59\xcc\x0a\xe2\x38\xaf\xe0",
.keylen = 32,
.ikm = NULL,
.ikmlen = 0,
.info = {
.iov_base = "\x01\x32\x2b\x96\xb3\x0a\xcd\x19"
"\x79\x79\x44\x4e\x46\x8e\x1c\x5c"
"\x68\x59\xbf\x1b\x1c\xf9\x51\xb7"
"\xe7\x25\x30\x3e\x23\x7e\x46\xb8"
"\x64\xa1\x45\xfa\xb2\x5e\x51\x7b"
"\x08\xf8\x68\x3d\x03\x15\xbb\x29"
"\x11\xd8\x0a\x0e\x8a\xba\x17\xf3"
"\xb4\x13\xfa\xac",
.iov_len = 60
},
.expected = "\x10\x62\x13\x42\xbf\xb0\xfd\x40"
"\x04\x6c\x0e\x29\xf2\xcf\xdb\xf0",
.expectedlen = 16
}
};
static int __init crypto_kdf108_init(void)
{
int ret = kdf_test(&kdf_ctr_hmac_sha256_tv_template[0], "hmac(sha256)",
crypto_kdf108_setkey, crypto_kdf108_ctr_generate);
if (ret) {
if (fips_enabled)
panic("alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
ret);
WARN(1,
"alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
ret);
} else {
pr_info("alg: self-tests for CTR-KDF (hmac(sha256)) passed\n");
}
return ret;
}
static void __exit crypto_kdf108_exit(void) { }
module_init(crypto_kdf108_init);
module_exit(crypto_kdf108_exit);
MODULE_LICENSE("GPL v2");
MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
MODULE_DESCRIPTION("Key Derivation Function conformant to SP800-108");

4
crypto/rsa.c
View file

@ -5,6 +5,7 @@
* Authors: Tadeusz Struk <tadeusz.struk@intel.com>
*/
#include <linux/fips.h>
#include <linux/module.h>
#include <linux/mpi.h>
#include <crypto/internal/rsa.h>
@ -144,6 +145,9 @@ static int rsa_check_key_length(unsigned int len)
case 512:
case 1024:
case 1536:
if (fips_enabled)
return -EINVAL;
fallthrough;
case 2048:
case 3072:
case 4096:

16
crypto/sha256_generic.c
View file

@ -33,18 +33,6 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
};
EXPORT_SYMBOL_GPL(sha256_zero_message_hash);
static int crypto_sha256_init(struct shash_desc *desc)
{
sha256_init(shash_desc_ctx(desc));
return 0;
}
static int crypto_sha224_init(struct shash_desc *desc)
{
sha224_init(shash_desc_ctx(desc));
return 0;
}
int crypto_sha256_update(struct shash_desc *desc, const u8 *data,
unsigned int len)
{
@ -72,7 +60,7 @@ EXPORT_SYMBOL(crypto_sha256_finup);
static struct shash_alg sha256_algs[2] = { {
.digestsize = SHA256_DIGEST_SIZE,
.init = crypto_sha256_init,
.init = sha256_base_init,
.update = crypto_sha256_update,
.final = crypto_sha256_final,
.finup = crypto_sha256_finup,
@ -86,7 +74,7 @@ static struct shash_alg sha256_algs[2] = { {
}
}, {
.digestsize = SHA224_DIGEST_SIZE,
.init = crypto_sha224_init,
.init = sha224_base_init,
.update = crypto_sha256_update,
.final = crypto_sha256_final,
.finup = crypto_sha256_finup,

9
crypto/testmgr.c
View file

@ -4193,7 +4193,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha1),cbc(des3_ede))",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
}
@ -4220,7 +4219,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha224),cbc(des3_ede))",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
}
@ -4240,7 +4238,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha256),cbc(des3_ede))",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
}
@ -4261,7 +4258,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha384),cbc(des3_ede))",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
}
@ -4289,7 +4285,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "authenc(hmac(sha512),cbc(des3_ede))",
.test = alg_test_aead,
.fips_allowed = 1,
.suite = {
.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
}
@ -4399,7 +4394,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "cbc(des3_ede)",
.test = alg_test_skcipher,
.fips_allowed = 1,
.suite = {
.cipher = __VECS(des3_ede_cbc_tv_template)
},
@ -4505,7 +4499,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}
}, {
.alg = "cmac(des3_ede)",
.fips_allowed = 1,
.test = alg_test_hash,
.suite = {
.hash = __VECS(des3_ede_cmac64_tv_template)
@ -4580,7 +4573,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "ctr(des3_ede)",
.test = alg_test_skcipher,
.fips_allowed = 1,
.suite = {
.cipher = __VECS(des3_ede_ctr_tv_template)
}
@ -4846,7 +4838,6 @@ static const struct alg_test_desc alg_test_descs[] = {
}, {
.alg = "ecb(des3_ede)",
.test = alg_test_skcipher,
.fips_allowed = 1,
.suite = {
.cipher = __VECS(des3_ede_tv_template)
}

6
crypto/testmgr.h
View file

@ -257,9 +257,9 @@ static const struct akcipher_testvec rsa_tv_template[] = {
}, {
#endif
.key =
"\x30\x82\x02\x1F" /* sequence of 543 bytes */
"\x30\x82\x02\x20" /* sequence of 544 bytes */
"\x02\x01\x01" /* version - integer of 1 byte */
"\x02\x82\x01\x00" /* modulus - integer of 256 bytes */
"\x02\x82\x01\x01\x00" /* modulus - integer of 256 bytes */
"\xDB\x10\x1A\xC2\xA3\xF1\xDC\xFF\x13\x6B\xED\x44\xDF\xF0\x02\x6D"
"\x13\xC7\x88\xDA\x70\x6B\x54\xF1\xE8\x27\xDC\xC3\x0F\x99\x6A\xFA"
"\xC6\x67\xFF\x1D\x1E\x3C\x1D\xC1\xB5\x5F\x6C\xC0\xB2\x07\x3A\x6D"
@ -299,7 +299,7 @@ static const struct akcipher_testvec rsa_tv_template[] = {
"\x02\x01\x00" /* exponent1 - integer of 1 byte */
"\x02\x01\x00" /* exponent2 - integer of 1 byte */
"\x02\x01\x00", /* coefficient - integer of 1 byte */
.key_len = 547,
.key_len = 548,
.m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a",
.c =
"\xb2\x97\x76\xb4\xae\x3e\x38\x3c\x7e\x64\x1f\xcc\xa2\x7f\xf6\xbe"

13
drivers/char/hw_random/Kconfig
View file

@ -414,7 +414,7 @@ config HW_RANDOM_MESON
config HW_RANDOM_CAVIUM
tristate "Cavium ThunderX Random Number Generator support"
depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT))
depends on HW_RANDOM && PCI && ARM64
default HW_RANDOM
help
This driver provides kernel-side support for the Random Number
@ -538,6 +538,17 @@ config HW_RANDOM_ARM_SMCCC_TRNG
To compile this driver as a module, choose M here: the
module will be called arm_smccc_trng.
config HW_RANDOM_CN10K
tristate "Marvell CN10K Random Number Generator support"
depends on HW_RANDOM && PCI && ARM64
default HW_RANDOM
help
This driver provides support for the True Random Number
generator available in Marvell CN10K SoCs.
To compile this driver as a module, choose M here.
The module will be called cn10k_rng. If unsure, say Y.
endif # HW_RANDOM
config UML_RANDOM

1
drivers/char/hw_random/Makefile
View file

@ -46,3 +46,4 @@ obj-$(CONFIG_HW_RANDOM_NPCM) += npcm-rng.o
obj-$(CONFIG_HW_RANDOM_CCTRNG) += cctrng.o
obj-$(CONFIG_HW_RANDOM_XIPHERA) += xiphera-trng.o
obj-$(CONFIG_HW_RANDOM_ARM_SMCCC_TRNG) += arm_smccc_trng.o
obj-$(CONFIG_HW_RANDOM_CN10K) += cn10k-rng.o

194
drivers/char/hw_random/cavium-rng-vf.c
View file

@ -1,10 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Hardware Random Number Generator support for Cavium, Inc.
* Thunder processor family.
*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
* for more details.
* Hardware Random Number Generator support.
* Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
*
* Copyright (C) 2016 Cavium, Inc.
*/
@ -15,16 +12,146 @@
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <asm/arch_timer.h>
/* PCI device IDs */
#define PCI_DEVID_CAVIUM_RNG_PF 0xA018
#define PCI_DEVID_CAVIUM_RNG_VF 0xA033
#define HEALTH_STATUS_REG 0x38
/* RST device info */
#define PCI_DEVICE_ID_RST_OTX2 0xA085
#define RST_BOOT_REG 0x1600ULL
#define CLOCK_BASE_RATE 50000000ULL
#define MSEC_TO_NSEC(x) (x * 1000000)
struct cavium_rng {
struct hwrng ops;
void __iomem *result;
void __iomem *pf_regbase;
struct pci_dev *pdev;
u64 clock_rate;
u64 prev_error;
u64 prev_time;
};
static inline bool is_octeontx(struct pci_dev *pdev)
{
if (midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_83XX,
MIDR_CPU_VAR_REV(0, 0),
MIDR_CPU_VAR_REV(3, 0)) ||
midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_81XX,
MIDR_CPU_VAR_REV(0, 0),
MIDR_CPU_VAR_REV(3, 0)) ||
midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX,
MIDR_CPU_VAR_REV(0, 0),
MIDR_CPU_VAR_REV(3, 0)))
return true;
return false;
}
static u64 rng_get_coprocessor_clkrate(void)
{
u64 ret = CLOCK_BASE_RATE * 16; /* Assume 800Mhz as default */
struct pci_dev *pdev;
void __iomem *base;
pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
PCI_DEVICE_ID_RST_OTX2, NULL);
if (!pdev)
goto error;
base = pci_ioremap_bar(pdev, 0);
if (!base)
goto error_put_pdev;
/* RST: PNR_MUL * 50Mhz gives clockrate */
ret = CLOCK_BASE_RATE * ((readq(base + RST_BOOT_REG) >> 33) & 0x3F);
iounmap(base);
error_put_pdev:
pci_dev_put(pdev);
error:
return ret;
}
static int check_rng_health(struct cavium_rng *rng)
{
u64 cur_err, cur_time;
u64 status, cycles;
u64 time_elapsed;
/* Skip checking health for OcteonTx */
if (!rng->pf_regbase)
return 0;
status = readq(rng->pf_regbase + HEALTH_STATUS_REG);
if (status & BIT_ULL(0)) {
dev_err(&rng->pdev->dev, "HWRNG: Startup health test failed\n");
return -EIO;
}
cycles = status >> 1;
if (!cycles)
return 0;
cur_time = arch_timer_read_counter();
/* RNM_HEALTH_STATUS[CYCLES_SINCE_HEALTH_FAILURE]
* Number of coprocessor cycles times 2 since the last failure.
* This field doesn't get cleared/updated until another failure.
*/
cycles = cycles / 2;
cur_err = (cycles * 1000000000) / rng->clock_rate; /* In nanosec */
/* Ignore errors that happenned a long time ago, these
* are most likely false positive errors.
*/
if (cur_err > MSEC_TO_NSEC(10)) {
rng->prev_error = 0;
rng->prev_time = 0;
return 0;
}
if (rng->prev_error) {
/* Calculate time elapsed since last error
* '1' tick of CNTVCT is 10ns, since it runs at 100Mhz.
*/
time_elapsed = (cur_time - rng->prev_time) * 10;
time_elapsed += rng->prev_error;
/* Check if current error is a new one or the old one itself.
* If error is a new one then consider there is a persistent
* issue with entropy, declare hardware failure.
*/
if (cur_err < time_elapsed) {
dev_err(&rng->pdev->dev, "HWRNG failure detected\n");
rng->prev_error = cur_err;
rng->prev_time = cur_time;
return -EIO;
}
}
rng->prev_error = cur_err;
rng->prev_time = cur_time;
return 0;
}
/* Read data from the RNG unit */
static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
{
struct cavium_rng *p = container_of(rng, struct cavium_rng, ops);
unsigned int size = max;
int err = 0;
err = check_rng_health(p);
if (err)
return err;
while (size >= 8) {
*((u64 *)dat) = readq(p->result);
@ -39,6 +166,39 @@ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
return max;
}
static int cavium_map_pf_regs(struct cavium_rng *rng)
{
struct pci_dev *pdev;
/* Health status is not supported on 83xx, skip mapping PF CSRs */
if (is_octeontx(rng->pdev)) {
rng->pf_regbase = NULL;
return 0;
}
pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
PCI_DEVID_CAVIUM_RNG_PF, NULL);
if (!pdev) {
dev_err(&pdev->dev, "Cannot find RNG PF device\n");
return -EIO;
}
rng->pf_regbase = ioremap(pci_resource_start(pdev, 0),
pci_resource_len(pdev, 0));
if (!rng->pf_regbase) {
dev_err(&pdev->dev, "Failed to map PF CSR region\n");
pci_dev_put(pdev);
return -ENOMEM;
}
pci_dev_put(pdev);
/* Get co-processor clock rate */
rng->clock_rate = rng_get_coprocessor_clkrate();
return 0;
}
/* Map Cavium RNG to an HWRNG object */
static int cavium_rng_probe_vf(struct pci_dev *pdev,
const struct pci_device_id *id)
@ -50,6 +210,8 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
if (!rng)
return -ENOMEM;
rng->pdev = pdev;
/* Map the RNG result */
rng->result = pcim_iomap(pdev, 0, 0);
if (!rng->result) {
@ -67,6 +229,11 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
pci_set_drvdata(pdev, rng);
/* Health status is available only at PF, hence map PF registers. */
ret = cavium_map_pf_regs(rng);
if (ret)
return ret;
ret = devm_hwrng_register(&pdev->dev, &rng->ops);
if (ret) {
dev_err(&pdev->dev, "Error registering device as HWRNG.\n");
@ -76,10 +243,18 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
return 0;
}
/* Remove the VF */
static void cavium_rng_remove_vf(struct pci_dev *pdev)
{
struct cavium_rng *rng;
rng = pci_get_drvdata(pdev);
iounmap(rng->pf_regbase);
}
static const struct pci_device_id cavium_rng_vf_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0},
{0,},
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CAVIUM_RNG_VF) },
{ 0, }
};
MODULE_DEVICE_TABLE(pci, cavium_rng_vf_id_table);
@ -87,8 +262,9 @@ static struct pci_driver cavium_rng_vf_driver = {
.name = "cavium_rng_vf",
.id_table = cavium_rng_vf_id_table,
.probe = cavium_rng_probe_vf,
.remove = cavium_rng_remove_vf,
};
module_pci_driver(cavium_rng_vf_driver);
MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
MODULE_LICENSE("GPL");
MODULE_LICENSE("GPL v2");

11
drivers/char/hw_random/cavium-rng.c
View file

@ -1,10 +1,7 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Hardware Random Number Generator support for Cavium Inc.
* Thunder processor family.
*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
* for more details.
* Hardware Random Number Generator support.
* Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
*
* Copyright (C) 2016 Cavium, Inc.
*/
@ -91,4 +88,4 @@ static struct pci_driver cavium_rng_pf_driver = {
module_pci_driver(cavium_rng_pf_driver);
MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
MODULE_LICENSE("GPL");
MODULE_LICENSE("GPL v2");

181
drivers/char/hw_random/cn10k-rng.c Normal file
View file

@ -0,0 +1,181 @@
// SPDX-License-Identifier: GPL-2.0
/* Marvell CN10K RVU Hardware Random Number Generator.
*
* Copyright (C) 2021 Marvell.
*
*/
#include <linux/hw_random.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/pci.h>
#include <linux/pci_ids.h>
#include <linux/delay.h>
#include <linux/arm-smccc.h>
/* CSRs */
#define RNM_CTL_STATUS 0x000
#define RNM_ENTROPY_STATUS 0x008
#define RNM_CONST 0x030
#define RNM_EBG_ENT 0x048
#define RNM_PF_EBG_HEALTH 0x050
#define RNM_PF_RANDOM 0x400
#define RNM_TRNG_RESULT 0x408
struct cn10k_rng {
void __iomem *reg_base;
struct hwrng ops;
struct pci_dev *pdev;
};
#define PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE 0xc2000b0f
static int reset_rng_health_state(struct cn10k_rng *rng)
{
struct arm_smccc_res res;
/* Send SMC service call to reset EBG health state */
arm_smccc_smc(PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE, 0, 0, 0, 0, 0, 0, 0, &res);
if (res.a0 != 0UL)
return -EIO;
return 0;
}
static int check_rng_health(struct cn10k_rng *rng)
{
u64 status;
int err;
/* Skip checking health */
if (!rng->reg_base)
return 0;
status = readq(rng->reg_base + RNM_PF_EBG_HEALTH);
if (status & BIT_ULL(20)) {
err = reset_rng_health_state(rng);
if (err) {
dev_err(&rng->pdev->dev, "HWRNG: Health test failed (status=%llx)\n",
status);
dev_err(&rng->pdev->dev, "HWRNG: error during reset\n");
}
}
return 0;
}
static void cn10k_read_trng(struct cn10k_rng *rng, u64 *value)
{
u64 upper, lower;
*value = readq(rng->reg_base + RNM_PF_RANDOM);
/* HW can run out of entropy if large amount random data is read in
* quick succession. Zeros may not be real random data from HW.
*/
if (!*value) {
upper = readq(rng->reg_base + RNM_PF_RANDOM);
lower = readq(rng->reg_base + RNM_PF_RANDOM);
while (!(upper & 0x00000000FFFFFFFFULL))
upper = readq(rng->reg_base + RNM_PF_RANDOM);
while (!(lower & 0xFFFFFFFF00000000ULL))
lower = readq(rng->reg_base + RNM_PF_RANDOM);
*value = (upper & 0xFFFFFFFF00000000) | (lower & 0xFFFFFFFF);
}
}
static int cn10k_rng_read(struct hwrng *hwrng, void *data,
size_t max, bool wait)
{
struct cn10k_rng *rng = (struct cn10k_rng *)hwrng->priv;
unsigned int size;
int err = 0;
u64 value;
err = check_rng_health(rng);
if (err)
return err;
size = max;
while (size >= 8) {
cn10k_read_trng(rng, &value);
*((u64 *)data) = (u64)value;
size -= 8;
data += 8;
}
while (size > 0) {
cn10k_read_trng(rng, &value);
*((u8 *)data) = (u8)value;
size--;
data++;
}
return max - size;
}
static int cn10k_rng_probe(struct pci_dev *pdev, const struct pci_device_id *id)
{
struct cn10k_rng *rng;
int err;
rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
if (!rng)
return -ENOMEM;
rng->pdev = pdev;
pci_set_drvdata(pdev, rng);
rng->reg_base = pcim_iomap(pdev, 0, 0);
if (!rng->reg_base) {
dev_err(&pdev->dev, "Error while mapping CSRs, exiting\n");
return -ENOMEM;
}
rng->ops.name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
"cn10k-rng-%s", dev_name(&pdev->dev));
if (!rng->ops.name)
return -ENOMEM;
rng->ops.read = cn10k_rng_read;
rng->ops.quality = 1000;
rng->ops.priv = (unsigned long)rng;
reset_rng_health_state(rng);
err = devm_hwrng_register(&pdev->dev, &rng->ops);
if (err) {
dev_err(&pdev->dev, "Could not register hwrng device.\n");
return err;
}
return 0;
}
static void cn10k_rng_remove(struct pci_dev *pdev)
{
/* Nothing to do */
}
static const struct pci_device_id cn10k_rng_id_table[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xA098) }, /* RNG PF */
{0,},
};
MODULE_DEVICE_TABLE(pci, cn10k_rng_id_table);
static struct pci_driver cn10k_rng_driver = {
.name = "cn10k_rng",
.id_table = cn10k_rng_id_table,
.probe = cn10k_rng_probe,
.remove = cn10k_rng_remove,
};
module_pci_driver(cn10k_rng_driver);
MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
MODULE_DESCRIPTION("Marvell CN10K HW RNG Driver");
MODULE_LICENSE("GPL v2");

21
drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c
View file

@ -106,6 +106,24 @@ static const struct ce_variant ce_a64_variant = {
.trng = CE_ID_NOTSUPP,
};
static const struct ce_variant ce_d1_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
.alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
CE_ALG_SHA384, CE_ALG_SHA512
},
.op_mode = { CE_OP_ECB, CE_OP_CBC
},
.ce_clks = {
{ "bus", 0, 200000000 },
{ "mod", 300000000, 0 },
{ "ram", 0, 400000000 },
},
.esr = ESR_D1,
.prng = CE_ALG_PRNG,
.trng = CE_ALG_TRNG,
};
static const struct ce_variant ce_r40_variant = {
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
},
@ -192,6 +210,7 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name)
dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
break;
case ESR_A64:
case ESR_D1:
case ESR_H5:
case ESR_R40:
v >>= (flow * 4);
@ -990,6 +1009,8 @@ static const struct of_device_id sun8i_ce_crypto_of_match_table[] = {
.data = &ce_h3_variant },
{ .compatible = "allwinner,sun8i-r40-crypto",
.data = &ce_r40_variant },
{ .compatible = "allwinner,sun20i-d1-crypto",
.data = &ce_d1_variant },
{ .compatible = "allwinner,sun50i-a64-crypto",
.data = &ce_a64_variant },
{ .compatible = "allwinner,sun50i-h5-crypto",

1
drivers/crypto/allwinner/sun8i-ce/sun8i-ce.h
View file

@ -94,6 +94,7 @@
#define ESR_R40 2
#define ESR_H5 3
#define ESR_H6 4
#define ESR_D1 5
#define PRNG_DATA_SIZE (160 / 8)
#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)

6
drivers/crypto/atmel-aes.c
View file

@ -960,6 +960,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
ctx = crypto_tfm_ctx(areq->tfm);
dd->areq = areq;
dd->ctx = ctx;
start_async = (areq != new_areq);
dd->is_async = start_async;
@ -1274,7 +1275,6 @@ static int atmel_aes_init_tfm(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
ctx->base.dd = dd;
ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_start;
return 0;
@ -1291,7 +1291,6 @@ static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
ctx->base.dd = dd;
ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_ctr_start;
return 0;
@ -1783,7 +1782,6 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm)
crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
ctx->base.dd = dd;
ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_gcm_start;
return 0;
@ -1927,7 +1925,6 @@ static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm)
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) +
crypto_skcipher_reqsize(ctx->fallback_tfm));
ctx->base.dd = dd;
ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_xts_start;
return 0;
@ -2154,7 +2151,6 @@ static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
auth_reqsize));
ctx->base.dd = dd;
ctx->base.dd->ctx = &ctx->base;
ctx->base.start = atmel_aes_authenc_start;
return 0;

6
drivers/crypto/caam/caamalg.c
View file

@ -1533,6 +1533,9 @@ static int aead_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
if (ret == -ENOSPC && engine->retry_support)
return ret;
if (ret != -EINPROGRESS) {
aead_unmap(ctx->jrdev, rctx->edesc, req);
kfree(rctx->edesc);
@ -1762,6 +1765,9 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
if (ret == -ENOSPC && engine->retry_support)
return ret;
if (ret != -EINPROGRESS) {
skcipher_unmap(ctx->jrdev, rctx->edesc, req);
kfree(rctx->edesc);

2
drivers/crypto/caam/caamalg_qi2.c
View file

@ -5470,7 +5470,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
dpaa2_fd_set_flc(&fd, req->flc_dma);
ppriv = this_cpu_ptr(priv->ppriv);
ppriv = raw_cpu_ptr(priv->ppriv);
for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
&fd);

3
drivers/crypto/caam/caamhash.c
View file

@ -765,6 +765,9 @@ static int ahash_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);
if (ret == -ENOSPC && engine->retry_support)
return ret;
if (ret != -EINPROGRESS) {
ahash_unmap(jrdev, state->edesc, req, 0);
kfree(state->edesc);

3
drivers/crypto/caam/caampkc.c
View file

@ -380,6 +380,9 @@ static int akcipher_do_one_req(struct crypto_engine *engine, void *areq)
ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req);
if (ret == -ENOSPC && engine->retry_support)
return ret;
if (ret != -EINPROGRESS) {
rsa_pub_unmap(jrdev, req_ctx->edesc, req);
rsa_io_unmap(jrdev, req_ctx->edesc, req);

5
drivers/crypto/cavium/cpt/cptvf_main.c
View file

@ -104,17 +104,14 @@ static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
u32 nr_queues)
{
u32 i;
size_t size;
int ret;
struct pending_queue *queue = NULL;
pqinfo->nr_queues = nr_queues;
pqinfo->qlen = qlen;
size = (qlen * sizeof(struct pending_entry));
for_each_pending_queue(pqinfo, queue, i) {
queue->head = kzalloc((size), GFP_KERNEL);
queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
if (!queue->head) {
ret = -ENOMEM;
goto pending_qfail;

2
drivers/crypto/ccp/ccp-dev.c
View file

@ -31,7 +31,7 @@
#define MAX_CCPS 32
/* Limit CCP use to a specifed number of queues per device */
static unsigned int nqueues = 0;
static unsigned int nqueues;
module_param(nqueues, uint, 0444);
MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");

259
drivers/crypto/ccp/sev-dev.c
View file

@ -22,6 +22,7 @@
#include <linux/firmware.h>
#include <linux/gfp.h>
#include <linux/cpufeature.h>
#include <linux/fs.h>
#include <asm/smp.h>
@ -43,6 +44,14 @@ static int psp_probe_timeout = 5;
module_param(psp_probe_timeout, int, 0644);
MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
static char *init_ex_path;
module_param(init_ex_path, charp, 0444);
MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
static bool psp_init_on_probe = true;
module_param(psp_init_on_probe, bool, 0444);
MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
@ -58,6 +67,14 @@ static int psp_timeout;
#define SEV_ES_TMR_SIZE (1024 * 1024)
static void *sev_es_tmr;
/* INIT_EX NV Storage:
* The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page
* allocator to allocate the memory, which will return aligned memory for the
* specified allocation order.
*/
#define NV_LENGTH (32 * 1024)
static void *sev_init_ex_buffer;
static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
{
struct sev_device *sev = psp_master->sev_data;
@ -107,6 +124,7 @@ static int sev_cmd_buffer_len(int cmd)
{
switch (cmd) {
case SEV_CMD_INIT: return sizeof(struct sev_data_init);
case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex);
case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status);
case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr);
case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import);
@ -141,6 +159,112 @@ static int sev_cmd_buffer_len(int cmd)
return 0;
}
static void *sev_fw_alloc(unsigned long len)
{
struct page *page;
page = alloc_pages(GFP_KERNEL, get_order(len));
if (!page)
return NULL;
return page_address(page);
}
static int sev_read_init_ex_file(void)
{
struct sev_device *sev = psp_master->sev_data;
struct file *fp;
ssize_t nread;
lockdep_assert_held(&sev_cmd_mutex);
if (!sev_init_ex_buffer)
return -EOPNOTSUPP;
fp = filp_open(init_ex_path, O_RDONLY, 0);
if (IS_ERR(fp)) {
int ret = PTR_ERR(fp);
dev_err(sev->dev,
"SEV: could not open %s for read, error %d\n",
init_ex_path, ret);
return ret;
}
nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
if (nread != NV_LENGTH) {
dev_err(sev->dev,
"SEV: failed to read %u bytes to non volatile memory area, ret %ld\n",
NV_LENGTH, nread);
return -EIO;
}
dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
filp_close(fp, NULL);
return 0;
}
static void sev_write_init_ex_file(void)
{
struct sev_device *sev = psp_master->sev_data;
struct file *fp;
loff_t offset = 0;
ssize_t nwrite;
lockdep_assert_held(&sev_cmd_mutex);
if (!sev_init_ex_buffer)
return;
fp = filp_open(init_ex_path, O_CREAT | O_WRONLY, 0600);
if (IS_ERR(fp)) {
dev_err(sev->dev,
"SEV: could not open file for write, error %ld\n",
PTR_ERR(fp));
return;
}
nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
vfs_fsync(fp, 0);
filp_close(fp, NULL);
if (nwrite != NV_LENGTH) {
dev_err(sev->dev,
"SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
NV_LENGTH, nwrite);
return;
}
dev_dbg(sev->dev, "SEV: write successful to NV file\n");
}
static void sev_write_init_ex_file_if_required(int cmd_id)
{
lockdep_assert_held(&sev_cmd_mutex);
if (!sev_init_ex_buffer)
return;
/*
* Only a few platform commands modify the SPI/NV area, but none of the
* non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
* PEK_CERT_IMPORT, and PDH_GEN do.
*/
switch (cmd_id) {
case SEV_CMD_FACTORY_RESET:
case SEV_CMD_INIT_EX:
case SEV_CMD_PDH_GEN:
case SEV_CMD_PEK_CERT_IMPORT:
case SEV_CMD_PEK_GEN:
break;
default:
return;
}
sev_write_init_ex_file();
}
static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
{
struct psp_device *psp = psp_master;
@ -210,6 +334,8 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n",
cmd, reg & PSP_CMDRESP_ERR_MASK);
ret = -EIO;
} else {
sev_write_init_ex_file_if_required(cmd);
}
print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
@ -236,12 +362,59 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret)
return rc;
}
static int __sev_init_locked(int *error)
{
struct sev_data_init data;
memset(&data, 0, sizeof(data));
if (sev_es_tmr) {
/*
* Do not include the encryption mask on the physical
* address of the TMR (firmware should clear it anyway).
*/
data.tmr_address = __pa(sev_es_tmr);
data.flags |= SEV_INIT_FLAGS_SEV_ES;
data.tmr_len = SEV_ES_TMR_SIZE;
}
return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
}
static int __sev_init_ex_locked(int *error)
{
struct sev_data_init_ex data;
int ret;
memset(&data, 0, sizeof(data));
data.length = sizeof(data);
data.nv_address = __psp_pa(sev_init_ex_buffer);
data.nv_len = NV_LENGTH;
ret = sev_read_init_ex_file();
if (ret)
return ret;
if (sev_es_tmr) {
/*
* Do not include the encryption mask on the physical
* address of the TMR (firmware should clear it anyway).
*/
data.tmr_address = __pa(sev_es_tmr);
data.flags |= SEV_INIT_FLAGS_SEV_ES;
data.tmr_len = SEV_ES_TMR_SIZE;
}
return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
}
static int __sev_platform_init_locked(int *error)
{
struct psp_device *psp = psp_master;
struct sev_data_init data;
struct sev_device *sev;
int rc = 0;
int rc, psp_ret;
int (*init_function)(int *error);
if (!psp || !psp->sev_data)
return -ENODEV;
@ -251,22 +424,23 @@ static int __sev_platform_init_locked(int *error)
if (sev->state == SEV_STATE_INIT)
return 0;
memset(&data, 0, sizeof(data));
if (sev_es_tmr) {
u64 tmr_pa;
init_function = sev_init_ex_buffer ? __sev_init_ex_locked :
__sev_init_locked;
rc = init_function(&psp_ret);
if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
/*
* Do not include the encryption mask on the physical
* address of the TMR (firmware should clear it anyway).
* Initialization command returned an integrity check failure
* status code, meaning that firmware load and validation of SEV
* related persistent data has failed. Retrying the
* initialization function should succeed by replacing the state
* with a reset state.
*/
tmr_pa = __pa(sev_es_tmr);
data.flags |= SEV_INIT_FLAGS_SEV_ES;
data.tmr_address = tmr_pa;
data.tmr_len = SEV_ES_TMR_SIZE;
dev_dbg(sev->dev, "SEV: retrying INIT command");
rc = init_function(&psp_ret);
}
if (error)
*error = psp_ret;
rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
if (rc)
return rc;
@ -280,7 +454,10 @@ static int __sev_platform_init_locked(int *error)
dev_dbg(sev->dev, "SEV firmware initialized\n");
return rc;
dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
sev->api_minor, sev->build);
return 0;
}
int sev_platform_init(int *error)
@ -1034,6 +1211,12 @@ static void sev_firmware_shutdown(struct sev_device *sev)
get_order(SEV_ES_TMR_SIZE));
sev_es_tmr = NULL;
}
if (sev_init_ex_buffer) {
free_pages((unsigned long)sev_init_ex_buffer,
get_order(NV_LENGTH));
sev_init_ex_buffer = NULL;
}
}
void sev_dev_destroy(struct psp_device *psp)
@ -1064,7 +1247,6 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
void sev_pci_init(void)
{
struct sev_device *sev = psp_master->sev_data;
struct page *tmr_page;
int error, rc;
if (!sev)
@ -1079,37 +1261,32 @@ void sev_pci_init(void)
sev_update_firmware(sev->dev) == 0)
sev_get_api_version();
/* If an init_ex_path is provided rely on INIT_EX for PSP initialization
* instead of INIT.
*/
if (init_ex_path) {
sev_init_ex_buffer = sev_fw_alloc(NV_LENGTH);
if (!sev_init_ex_buffer) {
dev_err(sev->dev,
"SEV: INIT_EX NV memory allocation failed\n");
goto err;
}
}
/* Obtain the TMR memory area for SEV-ES use */
tmr_page = alloc_pages(GFP_KERNEL, get_order(SEV_ES_TMR_SIZE));
if (tmr_page) {
sev_es_tmr = page_address(tmr_page);
} else {
sev_es_tmr = NULL;
sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE);
if (!sev_es_tmr)
dev_warn(sev->dev,
"SEV: TMR allocation failed, SEV-ES support unavailable\n");
}
if (!psp_init_on_probe)
return;
/* Initialize the platform */
rc = sev_platform_init(&error);
if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) {
/*
* INIT command returned an integrity check failure
* status code, meaning that firmware load and
* validation of SEV related persistent data has
* failed and persistent state has been erased.
* Retrying INIT command here should succeed.
*/
dev_dbg(sev->dev, "SEV: retrying INIT command");
rc = sev_platform_init(&error);
}
if (rc) {
dev_err(sev->dev, "SEV: failed to INIT error %#x\n", error);
return;
}
dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
sev->api_minor, sev->build);
if (rc)
dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n",
error, rc);
return;

1
drivers/crypto/ccree/cc_request_mgr.c
View file

@ -101,7 +101,6 @@ void cc_req_mgr_fini(struct cc_drvdata *drvdata)
dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
#ifdef COMP_IN_WQ
flush_workqueue(req_mgr_h->workq);
destroy_workqueue(req_mgr_h->workq);
#else
/* Kill tasklet */

7
drivers/crypto/hisilicon/hpre/hpre_crypto.c
View file

@ -1177,13 +1177,10 @@ static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
static void hpre_key_to_big_end(u8 *data, int len)
{
int i, j;
u8 tmp;
for (i = 0; i < len / 2; i++) {
j = len - i - 1;
tmp = data[j];
data[j] = data[i];
data[i] = tmp;
swap(data[j], data[i]);
}
}
@ -1865,7 +1862,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
*/
if (memcmp(ptr, p, ctx->key_sz) == 0) {
dev_err(dev, "gx is p!\n");
return -EINVAL;
goto err;
} else if (memcmp(ptr, p, ctx->key_sz) > 0) {
hpre_curve25519_src_modulo_p(ptr);
}

2
drivers/crypto/hisilicon/hpre/hpre_main.c
View file

@ -103,7 +103,7 @@
#define HPRE_QM_PM_FLR BIT(11)
#define HPRE_QM_SRIOV_FLR BIT(12)
#define HPRE_SHAPER_TYPE_RATE 128
#define HPRE_SHAPER_TYPE_RATE 640
#define HPRE_VIA_MSI_DSM 1
#define HPRE_SQE_MASK_OFFSET 8
#define HPRE_SQE_MASK_LEN 24

487
drivers/crypto/hisilicon/qm.c
View file

@ -89,6 +89,10 @@
#define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
#define QM_AEQE_TYPE_SHIFT 17
#define QM_AEQE_CQN_MASK GENMASK(15, 0)
#define QM_CQ_OVERFLOW 0
#define QM_EQ_OVERFLOW 1
#define QM_CQE_ERROR 2
#define QM_DOORBELL_CMD_SQ 0
#define QM_DOORBELL_CMD_CQ 1
@ -122,6 +126,8 @@
#define QM_CQC_VFT 0x1
#define QM_VFT_CFG 0x100060
#define QM_VFT_CFG_OP_ENABLE 0x100054
#define QM_PM_CTRL 0x100148
#define QM_IDLE_DISABLE BIT(9)
#define QM_VFT_CFG_DATA_L 0x100064
#define QM_VFT_CFG_DATA_H 0x100068
@ -501,10 +507,30 @@ static const char * const qp_s[] = {
"none", "init", "start", "stop", "close",
};
static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000,
10000, 25000, 50000, 100000};
static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16,
17, 18, 19};
struct qm_typical_qos_table {
u32 start;
u32 end;
u32 val;
};
/* the qos step is 100 */
static struct qm_typical_qos_table shaper_cir_s[] = {
{100, 100, 4},
{200, 200, 3},
{300, 500, 2},
{600, 1000, 1},
{1100, 100000, 0},
};
static struct qm_typical_qos_table shaper_cbs_s[] = {
{100, 200, 9},
{300, 500, 11},
{600, 1000, 12},
{1100, 10000, 16},
{10100, 25000, 17},
{25100, 50000, 18},
{50100, 100000, 19}
};
static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
{
@ -585,6 +611,75 @@ static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
return avail;
}
static u32 qm_get_hw_error_status(struct hisi_qm *qm)
{
return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
}
static u32 qm_get_dev_err_status(struct hisi_qm *qm)
{
return qm->err_ini->get_dev_hw_err_status(qm);
}
/* Check if the error causes the master ooo block */
static int qm_check_dev_error(struct hisi_qm *qm)
{
u32 val, dev_val;
if (qm->fun_type == QM_HW_VF)
return 0;
val = qm_get_hw_error_status(qm);
dev_val = qm_get_dev_err_status(qm);
if (qm->ver < QM_HW_V3)
return (val & QM_ECC_MBIT) ||
(dev_val & qm->err_info.ecc_2bits_mask);
return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
(dev_val & (~qm->err_info.dev_ce_mask));
}
static int qm_wait_reset_finish(struct hisi_qm *qm)
{
int delay = 0;
/* All reset requests need to be queued for processing */
while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
msleep(++delay);
if (delay > QM_RESET_WAIT_TIMEOUT)
return -EBUSY;
}
return 0;
}
static int qm_reset_prepare_ready(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
/*
* PF and VF on host doesnot support resetting at the
* same time on Kunpeng920.
*/
if (qm->ver < QM_HW_V3)
return qm_wait_reset_finish(pf_qm);
return qm_wait_reset_finish(qm);
}
static void qm_reset_bit_clear(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
if (qm->ver < QM_HW_V3)
clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
clear_bit(QM_RESETTING, &qm->misc_ctl);
}
static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
u64 base, u16 queue, bool op)
{
@ -707,6 +802,19 @@ static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
qm->ops->qm_db(qm, qn, cmd, index, priority);
}
static void qm_disable_clock_gate(struct hisi_qm *qm)
{
u32 val;
/* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */
if (qm->ver < QM_HW_V3)
return;
val = readl(qm->io_base + QM_PM_CTRL);
val |= QM_IDLE_DISABLE;
writel(val, qm->io_base + QM_PM_CTRL);
}
static int qm_dev_mem_reset(struct hisi_qm *qm)
{
u32 val;
@ -899,24 +1007,71 @@ static void qm_set_qp_disable(struct hisi_qp *qp, int offset)
mb();
}
static irqreturn_t qm_aeq_irq(int irq, void *data)
static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)
{
struct hisi_qp *qp = &qm->qp_array[qp_id];
qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);
hisi_qm_stop_qp(qp);
qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);
}
static void qm_reset_function(struct hisi_qm *qm)
{
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
struct device *dev = &qm->pdev->dev;
int ret;
if (qm_check_dev_error(pf_qm))
return;
ret = qm_reset_prepare_ready(qm);
if (ret) {
dev_err(dev, "reset function not ready\n");
return;
}
ret = hisi_qm_stop(qm, QM_FLR);
if (ret) {
dev_err(dev, "failed to stop qm when reset function\n");
goto clear_bit;
}
ret = hisi_qm_start(qm);
if (ret)
dev_err(dev, "failed to start qm when reset function\n");
clear_bit:
qm_reset_bit_clear(qm);
}
static irqreturn_t qm_aeq_thread(int irq, void *data)
{
struct hisi_qm *qm = data;
struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
u32 type;
atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
return IRQ_NONE;
u32 type, qp_id;
while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
if (type < ARRAY_SIZE(qm_fifo_overflow))
dev_err(&qm->pdev->dev, "%s overflow\n",
qm_fifo_overflow[type]);
else
qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;
switch (type) {
case QM_EQ_OVERFLOW:
dev_err(&qm->pdev->dev, "eq overflow, reset function\n");
qm_reset_function(qm);
return IRQ_HANDLED;
case QM_CQ_OVERFLOW:
dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",
qp_id);
fallthrough;
case QM_CQE_ERROR:
qm_disable_qp(qm, qp_id);
break;
default:
dev_err(&qm->pdev->dev, "unknown error type %u\n",
type);
break;
}
if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
qm->status.aeqc_phase = !qm->status.aeqc_phase;
@ -926,13 +1081,24 @@ static irqreturn_t qm_aeq_irq(int irq, void *data)
aeqe++;
qm->status.aeq_head++;
}
qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
}
qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
return IRQ_HANDLED;
}
static irqreturn_t qm_aeq_irq(int irq, void *data)
{
struct hisi_qm *qm = data;
atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
return IRQ_NONE;
return IRQ_WAKE_THREAD;
}
static void qm_irq_unregister(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
@ -988,12 +1154,14 @@ static void qm_init_prefetch(struct hisi_qm *qm)
}
/*
* acc_shaper_para_calc() Get the IR value by the qos formula, the return value
* is the expected qos calculated.
* the formula:
* IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
*
* IR_b * (2 ^ IR_u) * 8
* IR(Mbps) * 10 ^ -3 = -------------------------
* Tick * (2 ^ IR_s)
* IR_b * (2 ^ IR_u) * 8000
* IR(Mbps) = -------------------------
* Tick * (2 ^ IR_s)
*/
static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
{
@ -1003,17 +1171,28 @@ static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
static u32 acc_shaper_calc_cbs_s(u32 ir)
{
int table_size = ARRAY_SIZE(shaper_cbs_s);
int i;
if (ir < typical_qos_val[0])
return QM_SHAPER_MIN_CBS_S;
for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) {
if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i])
return typical_qos_cbs_s[i - 1];
for (i = 0; i < table_size; i++) {
if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)
return shaper_cbs_s[i].val;
}
return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1];
return QM_SHAPER_MIN_CBS_S;
}
static u32 acc_shaper_calc_cir_s(u32 ir)
{
int table_size = ARRAY_SIZE(shaper_cir_s);
int i;
for (i = 0; i < table_size; i++) {
if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)
return shaper_cir_s[i].val;
}
return 0;
}
static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
@ -1022,25 +1201,18 @@ static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
u32 error_rate;
factor->cbs_s = acc_shaper_calc_cbs_s(ir);
cir_s = acc_shaper_calc_cir_s(ir);
for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) {
/** the formula is changed to:
* IR_b * (2 ^ IR_u) * DIVISOR_CLK
* IR(Mbps) = -------------------------
* 768 * (2 ^ IR_s)
*/
ir_calc = acc_shaper_para_calc(cir_b, cir_u,
cir_s);
error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
factor->cir_b = cir_b;
factor->cir_u = cir_u;
factor->cir_s = cir_s;
ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
return 0;
}
error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
factor->cir_b = cir_b;
factor->cir_u = cir_u;
factor->cir_s = cir_s;
return 0;
}
}
}
@ -1126,10 +1298,10 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
{
u32 qos = qm->factor[fun_num].func_qos;
int ret, i;
qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL;
ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]);
ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);
if (ret) {
dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
return ret;
@ -2082,35 +2254,6 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
return ACC_ERR_RECOVERED;
}
static u32 qm_get_hw_error_status(struct hisi_qm *qm)
{
return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
}
static u32 qm_get_dev_err_status(struct hisi_qm *qm)
{
return qm->err_ini->get_dev_hw_err_status(qm);
}
/* Check if the error causes the master ooo block */
static int qm_check_dev_error(struct hisi_qm *qm)
{
u32 val, dev_val;
if (qm->fun_type == QM_HW_VF)
return 0;
val = qm_get_hw_error_status(qm);
dev_val = qm_get_dev_err_status(qm);
if (qm->ver < QM_HW_V3)
return (val & QM_ECC_MBIT) ||
(dev_val & qm->err_info.ecc_2bits_mask);
return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
(dev_val & (~qm->err_info.dev_ce_mask));
}
static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
{
struct qm_mailbox mailbox;
@ -3399,6 +3542,7 @@ void hisi_qm_uninit(struct hisi_qm *qm)
dma_free_coherent(dev, qm->qdma.size,
qm->qdma.va, qm->qdma.dma);
}
up_write(&qm->qps_lock);
qm_irq_unregister(qm);
hisi_qm_pci_uninit(qm);
@ -3406,8 +3550,6 @@ void hisi_qm_uninit(struct hisi_qm *qm)
uacce_remove(qm->uacce);
qm->uacce = NULL;
}
up_write(&qm->qps_lock);
}
EXPORT_SYMBOL_GPL(hisi_qm_uninit);
@ -3473,6 +3615,22 @@ static void qm_init_eq_aeq_status(struct hisi_qm *qm)
status->aeqc_phase = true;
}
static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)
{
/* Clear eq/aeq interrupt source */
qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
}
static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)
{
writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
}
static int qm_eq_ctx_cfg(struct hisi_qm *qm)
{
struct device *dev = &qm->pdev->dev;
@ -3556,10 +3714,6 @@ static int __hisi_qm_start(struct hisi_qm *qm)
WARN_ON(!qm->qdma.va);
if (qm->fun_type == QM_HW_PF) {
ret = qm_dev_mem_reset(qm);
if (ret)
return ret;
ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
if (ret)
return ret;
@ -3578,9 +3732,7 @@ static int __hisi_qm_start(struct hisi_qm *qm)
return ret;
qm_init_prefetch(qm);
writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
qm_enable_eq_aeq_interrupts(qm);
return 0;
}
@ -3728,10 +3880,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
}
/* Mask eq and aeq irq */
writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
qm_disable_eq_aeq_interrupts(qm);
if (qm->fun_type == QM_HW_PF) {
ret = hisi_qm_set_vft(qm, 0, 0, 0);
if (ret < 0) {
@ -4231,66 +4380,69 @@ static ssize_t qm_qos_value_init(const char *buf, unsigned long *val)
return 0;
}
static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
unsigned long *val,
unsigned int *fun_index)
{
char tbuf_bdf[QM_DBG_READ_LEN] = {0};
char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
u32 tmp1, device, function;
int ret, bus;
ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);
if (ret != QM_QOS_PARAM_NUM)
return -EINVAL;
ret = qm_qos_value_init(val_buf, val);
if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {
pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
return -EINVAL;
}
ret = sscanf(tbuf_bdf, "%u:%x:%u.%u", &tmp1, &bus, &device, &function);
if (ret != QM_QOS_BDF_PARAM_NUM) {
pci_err(qm->pdev, "input pci bdf value is error!\n");
return -EINVAL;
}
*fun_index = PCI_DEVFN(device, function);
return 0;
}
static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
size_t count, loff_t *pos)
{
struct hisi_qm *qm = filp->private_data;
char tbuf[QM_DBG_READ_LEN];
int tmp1, bus, device, function;
char tbuf_bdf[QM_DBG_READ_LEN] = {0};
char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
unsigned int fun_index;
unsigned long val = 0;
unsigned long val;
int len, ret;
if (qm->fun_type == QM_HW_VF)
return -EINVAL;
if (*pos != 0)
return 0;
if (count >= QM_DBG_READ_LEN)
return -ENOSPC;
len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
if (len < 0)
return len;
tbuf[len] = '\0';
ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);
if (ret)
return ret;
/* Mailbox and reset cannot be operated at the same time */
if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
return -EAGAIN;
}
if (*pos != 0) {
ret = 0;
goto err_get_status;
}
if (count >= QM_DBG_READ_LEN) {
ret = -ENOSPC;
goto err_get_status;
}
len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
if (len < 0) {
ret = len;
goto err_get_status;
}
tbuf[len] = '\0';
ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf);
if (ret != QM_QOS_PARAM_NUM) {
ret = -EINVAL;
goto err_get_status;
}
ret = qm_qos_value_init(val_buf, &val);
if (val == 0 || val > QM_QOS_MAX_VAL || ret) {
pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
ret = -EINVAL;
goto err_get_status;
}
ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function);
if (ret != QM_QOS_BDF_PARAM_NUM) {
pci_err(qm->pdev, "input pci bdf value is error!\n");
ret = -EINVAL;
goto err_get_status;
}
fun_index = device * 8 + function;
ret = qm_pm_get_sync(qm);
if (ret) {
ret = -EINVAL;
@ -4304,6 +4456,8 @@ static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
goto err_put_sync;
}
pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",
fun_index, val);
ret = count;
err_put_sync:
@ -4728,46 +4882,6 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd,
return ret;
}
static int qm_wait_reset_finish(struct hisi_qm *qm)
{
int delay = 0;
/* All reset requests need to be queued for processing */
while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
msleep(++delay);
if (delay > QM_RESET_WAIT_TIMEOUT)
return -EBUSY;
}
return 0;
}
static int qm_reset_prepare_ready(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
/*
* PF and VF on host doesnot support resetting at the
* same time on Kunpeng920.
*/
if (qm->ver < QM_HW_V3)
return qm_wait_reset_finish(pf_qm);
return qm_wait_reset_finish(qm);
}
static void qm_reset_bit_clear(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
if (qm->ver < QM_HW_V3)
clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
clear_bit(QM_RESETTING, &qm->misc_ctl);
}
static int qm_controller_reset_prepare(struct hisi_qm *qm)
{
struct pci_dev *pdev = qm->pdev;
@ -5053,6 +5167,12 @@ static int qm_controller_reset_done(struct hisi_qm *qm)
if (qm->err_ini->open_axi_master_ooo)
qm->err_ini->open_axi_master_ooo(qm);
ret = qm_dev_mem_reset(qm);
if (ret) {
pci_err(pdev, "failed to reset device memory\n");
return ret;
}
ret = qm_restart(qm);
if (ret) {
pci_err(pdev, "Failed to start QM!\n");
@ -5267,8 +5387,10 @@ static int qm_irq_register(struct hisi_qm *qm)
return ret;
if (qm->ver > QM_HW_V1) {
ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
qm_aeq_irq, 0, qm->dev_name, qm);
ret = request_threaded_irq(pci_irq_vector(pdev,
QM_AEQ_EVENT_IRQ_VECTOR),
qm_aeq_irq, qm_aeq_thread,
0, qm->dev_name, qm);
if (ret)
goto err_aeq_irq;
@ -5750,13 +5872,15 @@ err_init_qp_mem:
static int hisi_qm_memory_init(struct hisi_qm *qm)
{
struct device *dev = &qm->pdev->dev;
int ret, total_vfs;
int ret, total_func, i;
size_t off = 0;
total_vfs = pci_sriov_get_totalvfs(qm->pdev);
qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL);
total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;
qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);
if (!qm->factor)
return -ENOMEM;
for (i = 0; i < total_func; i++)
qm->factor[i].func_qos = QM_QOS_MAX_VAL;
#define QM_INIT_BUF(qm, type, num) do { \
(qm)->type = ((qm)->qdma.va + (off)); \
@ -5825,6 +5949,15 @@ int hisi_qm_init(struct hisi_qm *qm)
goto err_irq_register;
}
if (qm->fun_type == QM_HW_PF) {
qm_disable_clock_gate(qm);
ret = qm_dev_mem_reset(qm);
if (ret) {
dev_err(dev, "failed to reset device memory\n");
goto err_irq_register;
}
}
if (qm->mode == UACCE_MODE_SVA) {
ret = qm_alloc_uacce(qm);
if (ret < 0)
@ -5982,8 +6115,12 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm)
qm_cmd_init(qm);
hisi_qm_dev_err_init(qm);
qm_disable_clock_gate(qm);
ret = qm_dev_mem_reset(qm);
if (ret)
pci_err(pdev, "failed to reset device memory\n");
return 0;
return ret;
}
/**
@ -6038,7 +6175,7 @@ int hisi_qm_resume(struct device *dev)
if (ret)
pci_err(pdev, "failed to start qm(%d)\n", ret);
return 0;
return ret;
}
EXPORT_SYMBOL_GPL(hisi_qm_resume);

2
drivers/crypto/hisilicon/sec2/sec_main.c
View file

@ -105,7 +105,7 @@
#define SEC_SQE_MASK_OFFSET 64
#define SEC_SQE_MASK_LEN 48
#define SEC_SHAPER_TYPE_RATE 128
#define SEC_SHAPER_TYPE_RATE 400
struct sec_hw_error {
u32 int_msk;

12
drivers/crypto/hisilicon/zip/zip_main.c
View file

@ -103,8 +103,8 @@
#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0)))
#define HZIP_SVA_PREFETCH_DISABLE BIT(26)
#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30))
#define HZIP_SHAPER_RATE_COMPRESS 252
#define HZIP_SHAPER_RATE_DECOMPRESS 229
#define HZIP_SHAPER_RATE_COMPRESS 750
#define HZIP_SHAPER_RATE_DECOMPRESS 140
#define HZIP_DELAY_1_US 1
#define HZIP_POLL_TIMEOUT_US 1000
@ -364,15 +364,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
/* user domain configurations */
writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
if (qm->use_sva && qm->ver == QM_HW_V2) {
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);
} else {
writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
}
/* let's open all compression/decompression cores */
@ -829,7 +830,10 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
qm->pdev = pdev;
qm->ver = pdev->revision;
qm->algs = "zlib\ngzip";
if (pdev->revision >= QM_HW_V3)
qm->algs = "zlib\ngzip\ndeflate\nlz77_zstd";
else
qm->algs = "zlib\ngzip";
qm->mode = uacce_mode;
qm->sqe_size = HZIP_SQE_SIZE;
qm->dev_name = hisi_zip_name;

1
drivers/crypto/keembay/keembay-ocs-ecc.c
View file

@ -930,6 +930,7 @@ static int kmb_ocs_ecc_probe(struct platform_device *pdev)
ecc_dev->engine = crypto_engine_alloc_init(dev, 1);
if (!ecc_dev->engine) {
dev_err(dev, "Could not allocate crypto engine\n");
rc = -ENOMEM;
goto list_del;
}

4
drivers/crypto/marvell/octeontx/otx_cptvf_main.c
View file

@ -94,15 +94,13 @@ static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen,
u32 num_queues)
{
struct otx_cpt_pending_queue *queue = NULL;
size_t size;
int ret;
u32 i;
pqinfo->num_queues = num_queues;
size = (qlen * sizeof(struct otx_cpt_pending_entry));
for_each_pending_queue(pqinfo, queue, i) {
queue->head = kzalloc((size), GFP_KERNEL);
queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
if (!queue->head) {
ret = -ENOMEM;
goto pending_qfail;

2
drivers/crypto/marvell/octeontx2/Makefile
View file

@ -3,7 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptpf.o rvu_cptvf.o
rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \
otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o \
cn10k_cpt.o
cn10k_cpt.o otx2_cpt_devlink.o
rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \
otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \
otx2_cptvf_algs.o cn10k_cpt.o

1
drivers/crypto/marvell/octeontx2/otx2_cpt_common.h
View file

@ -10,6 +10,7 @@
#include <linux/module.h>
#include <linux/delay.h>
#include <linux/crypto.h>
#include <net/devlink.h>
#include "otx2_cpt_hw_types.h"
#include "rvu.h"
#include "mbox.h"

108
drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c Normal file
View file

@ -0,0 +1,108 @@
// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2021 Marvell. */
#include "otx2_cpt_devlink.h"
static int otx2_cpt_dl_egrp_create(struct devlink *dl, u32 id,
struct devlink_param_gset_ctx *ctx)
{
struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
return otx2_cpt_dl_custom_egrp_create(cptpf, ctx);
}
static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id,
struct devlink_param_gset_ctx *ctx)
{
struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
return otx2_cpt_dl_custom_egrp_delete(cptpf, ctx);
}
static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id,
struct devlink_param_gset_ctx *ctx)
{
struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
otx2_cpt_print_uc_dbg_info(cptpf);
return 0;
}
enum otx2_cpt_dl_param_id {
OTX2_CPT_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE,
OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE,
};
static const struct devlink_param otx2_cpt_dl_params[] = {
DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE,
"egrp_create", DEVLINK_PARAM_TYPE_STRING,
BIT(DEVLINK_PARAM_CMODE_RUNTIME),
otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_create,
NULL),
DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE,
"egrp_delete", DEVLINK_PARAM_TYPE_STRING,
BIT(DEVLINK_PARAM_CMODE_RUNTIME),
otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_delete,
NULL),
};
static int otx2_cpt_devlink_info_get(struct devlink *devlink,
struct devlink_info_req *req,
struct netlink_ext_ack *extack)
{
return devlink_info_driver_name_put(req, "rvu_cptpf");
}
static const struct devlink_ops otx2_cpt_devlink_ops = {
.info_get = otx2_cpt_devlink_info_get,
};
int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf)
{
struct device *dev = &cptpf->pdev->dev;
struct otx2_cpt_devlink *cpt_dl;
struct devlink *dl;
int ret;
dl = devlink_alloc(&otx2_cpt_devlink_ops,
sizeof(struct otx2_cpt_devlink), dev);
if (!dl) {
dev_warn(dev, "devlink_alloc failed\n");
return -ENOMEM;
}
cpt_dl = devlink_priv(dl);
cpt_dl->dl = dl;
cpt_dl->cptpf = cptpf;
cptpf->dl = dl;
ret = devlink_params_register(dl, otx2_cpt_dl_params,
ARRAY_SIZE(otx2_cpt_dl_params));
if (ret) {
dev_err(dev, "devlink params register failed with error %d",
ret);
devlink_free(dl);
return ret;
}
devlink_register(dl);
return 0;
}
void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf)
{
struct devlink *dl = cptpf->dl;
if (!dl)
return;
devlink_unregister(dl);
devlink_params_unregister(dl, otx2_cpt_dl_params,
ARRAY_SIZE(otx2_cpt_dl_params));
devlink_free(dl);
}

20
drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h Normal file
View file

@ -0,0 +1,20 @@
/* SPDX-License-Identifier: GPL-2.0-only
* Copyright (C) 2021 Marvell.
*/
#ifndef __OTX2_CPT_DEVLINK_H
#define __OTX2_CPT_DEVLINK_H
#include "otx2_cpt_common.h"
#include "otx2_cptpf.h"
struct otx2_cpt_devlink {
struct devlink *dl;
struct otx2_cptpf_dev *cptpf;
};
/* Devlink APIs */
int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf);
void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf);
#endif /* __OTX2_CPT_DEVLINK_H */

3
drivers/crypto/marvell/octeontx2/otx2_cptpf.h
View file

@ -53,6 +53,9 @@ struct otx2_cptpf_dev {
u8 enabled_vfs; /* Number of enabled VFs */
u8 kvf_limits; /* Kernel crypto limits */
bool has_cpt1;
/* Devlink */
struct devlink *dl;
};
irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg);

18
drivers/crypto/marvell/octeontx2/otx2_cptpf_main.c
View file

@ -4,6 +4,7 @@
#include <linux/firmware.h>
#include "otx2_cpt_hw_types.h"
#include "otx2_cpt_common.h"
#include "otx2_cpt_devlink.h"
#include "otx2_cptpf_ucode.h"
#include "otx2_cptpf.h"
#include "cn10k_cpt.h"
@ -494,12 +495,11 @@ static ssize_t kvf_limits_store(struct device *dev,
{
struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);
int lfs_num;
int ret;
if (kstrtoint(buf, 0, &lfs_num)) {
dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
lfs_num, num_online_cpus());
return -EINVAL;
}
ret = kstrtoint(buf, 0, &lfs_num);
if (ret)
return ret;
if (lfs_num < 1 || lfs_num > num_online_cpus()) {
dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
lfs_num, num_online_cpus());
@ -767,8 +767,15 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group);
if (err)
goto cleanup_eng_grps;
err = otx2_cpt_register_dl(cptpf);
if (err)
goto sysfs_grp_del;
return 0;
sysfs_grp_del:
sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group);
cleanup_eng_grps:
otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps);
unregister_intr:
@ -788,6 +795,7 @@ static void otx2_cptpf_remove(struct pci_dev *pdev)
return;
cptpf_sriov_disable(pdev);
otx2_cpt_unregister_dl(cptpf);
/* Delete sysfs entry created for kernel VF limits */
sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group);
/* Cleanup engine groups */

315
drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c
View file

@ -29,7 +29,8 @@ static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev,
bool found = false;
int i;
if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
if (eng_grp->g->engs_num < 0 ||
eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
dev_err(dev, "unsupported number of engines %d on octeontx2\n",
eng_grp->g->engs_num);
return bmap;
@ -1110,18 +1111,19 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} };
struct pci_dev *pdev = cptpf->pdev;
struct fw_info_t fw_info;
int ret;
int ret = 0;
mutex_lock(&eng_grps->lock);
/*
* We don't create engine groups if it was already
* made (when user enabled VFs for the first time)
*/
if (eng_grps->is_grps_created)
return 0;
goto unlock;
ret = cpt_ucode_load_fw(pdev, &fw_info);
if (ret)
return ret;
goto unlock;
/*
* Create engine group with SE engines for kernel
@ -1186,7 +1188,7 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
cpt_ucode_release_fw(&fw_info);
if (is_dev_otx2(pdev))
return 0;
goto unlock;
/*
* Configure engine group mask to allow context prefetching
* for the groups.
@ -1201,12 +1203,15 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
*/
otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER,
CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0);
mutex_unlock(&eng_grps->lock);
return 0;
delete_eng_grp:
delete_engine_grps(pdev, eng_grps);
release_fw:
cpt_ucode_release_fw(&fw_info);
unlock:
mutex_unlock(&eng_grps->lock);
return ret;
}
@ -1286,6 +1291,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
struct otx2_cpt_eng_grp_info *grp;
int i, j;
mutex_lock(&eng_grps->lock);
delete_engine_grps(pdev, eng_grps);
/* Release memory */
for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
@ -1295,6 +1301,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
grp->engs[j].bmap = NULL;
}
}
mutex_unlock(&eng_grps->lock);
}
int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
@ -1303,6 +1310,7 @@ int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
struct otx2_cpt_eng_grp_info *grp;
int i, j, ret;
mutex_init(&eng_grps->lock);
eng_grps->obj = pci_get_drvdata(pdev);
eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt;
eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt;
@ -1349,11 +1357,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
struct fw_info_t fw_info;
int ret;
mutex_lock(&eng_grps->lock);
ret = cpt_ucode_load_fw(pdev, &fw_info);
if (ret)
if (ret) {
mutex_unlock(&eng_grps->lock);
return ret;
}
uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES);
uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES);
if (uc_info[0] == NULL) {
dev_err(&pdev->dev, "Unable to find firmware for AE\n");
ret = -EINVAL;
@ -1396,12 +1407,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
goto delete_eng_grp;
cpt_ucode_release_fw(&fw_info);
mutex_unlock(&eng_grps->lock);
return 0;
delete_eng_grp:
delete_engine_grps(pdev, eng_grps);
release_fw:
cpt_ucode_release_fw(&fw_info);
mutex_unlock(&eng_grps->lock);
return ret;
}
@ -1501,3 +1514,291 @@ delete_grps:
return ret;
}
int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
struct devlink_param_gset_ctx *ctx)
{
struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { { 0 } };
struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = {};
struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
char *ucode_filename[OTX2_CPT_MAX_ETYPES_PER_GRP];
char tmp_buf[OTX2_CPT_NAME_LENGTH] = { 0 };
struct device *dev = &cptpf->pdev->dev;
char *start, *val, *err_msg, *tmp;
int grp_idx = 0, ret = -EINVAL;
bool has_se, has_ie, has_ae;
struct fw_info_t fw_info;
int ucode_idx = 0;
if (!eng_grps->is_grps_created) {
dev_err(dev, "Not allowed before creating the default groups\n");
return -EINVAL;
}
err_msg = "Invalid engine group format";
strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1);
start = tmp_buf;
has_se = has_ie = has_ae = false;
for (;;) {
val = strsep(&start, ";");
if (!val)
break;
val = strim(val);
if (!*val)
continue;
if (!strncasecmp(val, "se", 2) && strchr(val, ':')) {
if (has_se || ucode_idx)
goto err_print;
tmp = strim(strsep(&val, ":"));
if (!val)
goto err_print;
if (strlen(tmp) != 2)
goto err_print;
if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
goto err_print;
engs[grp_idx++].type = OTX2_CPT_SE_TYPES;
has_se = true;
} else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) {
if (has_ae || ucode_idx)
goto err_print;
tmp = strim(strsep(&val, ":"));
if (!val)
goto err_print;
if (strlen(tmp) != 2)
goto err_print;
if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
goto err_print;
engs[grp_idx++].type = OTX2_CPT_AE_TYPES;
has_ae = true;
} else if (!strncasecmp(val, "ie", 2) && strchr(val, ':')) {
if (has_ie || ucode_idx)
goto err_print;
tmp = strim(strsep(&val, ":"));
if (!val)
goto err_print;
if (strlen(tmp) != 2)
goto err_print;
if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
goto err_print;
engs[grp_idx++].type = OTX2_CPT_IE_TYPES;
has_ie = true;
} else {
if (ucode_idx > 1)
goto err_print;
if (!strlen(val))
goto err_print;
if (strnstr(val, " ", strlen(val)))
goto err_print;
ucode_filename[ucode_idx++] = val;
}
}
/* Validate input parameters */
if (!(grp_idx && ucode_idx))
goto err_print;
if (ucode_idx > 1 && grp_idx < 2)
goto err_print;
if (grp_idx > OTX2_CPT_MAX_ETYPES_PER_GRP) {
err_msg = "Error max 2 engine types can be attached";
goto err_print;
}
if (grp_idx > 1) {
if ((engs[0].type + engs[1].type) !=
(OTX2_CPT_SE_TYPES + OTX2_CPT_IE_TYPES)) {
err_msg = "Only combination of SE+IE engines is allowed";
goto err_print;
}
/* Keep SE engines at zero index */
if (engs[1].type == OTX2_CPT_SE_TYPES)
swap(engs[0], engs[1]);
}
mutex_lock(&eng_grps->lock);
if (cptpf->enabled_vfs) {
dev_err(dev, "Disable VFs before modifying engine groups\n");
ret = -EACCES;
goto err_unlock;
}
INIT_LIST_HEAD(&fw_info.ucodes);
ret = load_fw(dev, &fw_info, ucode_filename[0]);
if (ret) {
dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]);
goto err_unlock;
}
if (ucode_idx > 1) {
ret = load_fw(dev, &fw_info, ucode_filename[1]);
if (ret) {
dev_err(dev, "Unable to load firmware %s\n",
ucode_filename[1]);
goto release_fw;
}
}
uc_info[0] = get_ucode(&fw_info, engs[0].type);
if (uc_info[0] == NULL) {
dev_err(dev, "Unable to find firmware for %s\n",
get_eng_type_str(engs[0].type));
ret = -EINVAL;
goto release_fw;
}
if (ucode_idx > 1) {
uc_info[1] = get_ucode(&fw_info, engs[1].type);
if (uc_info[1] == NULL) {
dev_err(dev, "Unable to find firmware for %s\n",
get_eng_type_str(engs[1].type));
ret = -EINVAL;
goto release_fw;
}
}
ret = create_engine_group(dev, eng_grps, engs, grp_idx,
(void **)uc_info, 1);
release_fw:
cpt_ucode_release_fw(&fw_info);
err_unlock:
mutex_unlock(&eng_grps->lock);
return ret;
err_print:
dev_err(dev, "%s\n", err_msg);
return ret;
}
int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
struct devlink_param_gset_ctx *ctx)
{
struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
struct device *dev = &cptpf->pdev->dev;
char *tmp, *err_msg;
int egrp;
int ret;
err_msg = "Invalid input string format(ex: egrp:0)";
if (strncasecmp(ctx->val.vstr, "egrp", 4))
goto err_print;
tmp = ctx->val.vstr;
strsep(&tmp, ":");
if (!tmp)
goto err_print;
if (kstrtoint(tmp, 10, &egrp))
goto err_print;
if (egrp < 0 || egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) {
dev_err(dev, "Invalid engine group %d", egrp);
return -EINVAL;
}
if (!eng_grps->grp[egrp].is_enabled) {
dev_err(dev, "Error engine_group%d is not configured", egrp);
return -EINVAL;
}
mutex_lock(&eng_grps->lock);
ret = delete_engine_group(dev, &eng_grps->grp[egrp]);
mutex_unlock(&eng_grps->lock);
return ret;
err_print:
dev_err(dev, "%s\n", err_msg);
return -EINVAL;
}
static void get_engs_info(struct otx2_cpt_eng_grp_info *eng_grp, char *buf,
int size, int idx)
{
struct otx2_cpt_engs_rsvd *mirrored_engs = NULL;
struct otx2_cpt_engs_rsvd *engs;
int len, i;
buf[0] = '\0';
for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
engs = &eng_grp->engs[i];
if (!engs->type)
continue;
if (idx != -1 && idx != i)
continue;
if (eng_grp->mirror.is_ena)
mirrored_engs = find_engines_by_type(
&eng_grp->g->grp[eng_grp->mirror.idx],
engs->type);
if (i > 0 && idx == -1) {
len = strlen(buf);
scnprintf(buf + len, size - len, ", ");
}
len = strlen(buf);
scnprintf(buf + len, size - len, "%d %s ",
mirrored_engs ? engs->count + mirrored_engs->count :
engs->count,
get_eng_type_str(engs->type));
if (mirrored_engs) {
len = strlen(buf);
scnprintf(buf + len, size - len,
"(%d shared with engine_group%d) ",
engs->count <= 0 ?
engs->count + mirrored_engs->count :
mirrored_engs->count,
eng_grp->mirror.idx);
}
}
}
void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf)
{
struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
struct otx2_cpt_eng_grp_info *mirrored_grp;
char engs_info[2 * OTX2_CPT_NAME_LENGTH];
struct otx2_cpt_eng_grp_info *grp;
struct otx2_cpt_engs_rsvd *engs;
u32 mask[4];
int i, j;
pr_debug("Engine groups global info");
pr_debug("max SE %d, max IE %d, max AE %d", eng_grps->avail.max_se_cnt,
eng_grps->avail.max_ie_cnt, eng_grps->avail.max_ae_cnt);
pr_debug("free SE %d", eng_grps->avail.se_cnt);
pr_debug("free IE %d", eng_grps->avail.ie_cnt);
pr_debug("free AE %d", eng_grps->avail.ae_cnt);
for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
grp = &eng_grps->grp[i];
pr_debug("engine_group%d, state %s", i,
grp->is_enabled ? "enabled" : "disabled");
if (grp->is_enabled) {
mirrored_grp = &eng_grps->grp[grp->mirror.idx];
pr_debug("Ucode0 filename %s, version %s",
grp->mirror.is_ena ?
mirrored_grp->ucode[0].filename :
grp->ucode[0].filename,
grp->mirror.is_ena ?
mirrored_grp->ucode[0].ver_str :
grp->ucode[0].ver_str);
if (is_2nd_ucode_used(grp))
pr_debug("Ucode1 filename %s, version %s",
grp->ucode[1].filename,
grp->ucode[1].ver_str);
}
for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) {
engs = &grp->engs[j];
if (engs->type) {
get_engs_info(grp, engs_info,
2 * OTX2_CPT_NAME_LENGTH, j);
pr_debug("Slot%d: %s", j, engs_info);
bitmap_to_arr32(mask, engs->bmap,
eng_grps->engs_num);
if (is_dev_otx2(cptpf->pdev))
pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x",
mask[3], mask[2], mask[1],
mask[0]);
else
pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x %8.8x",
mask[4], mask[3], mask[2], mask[1],
mask[0]);
}
}
}
}

7
drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.h
View file

@ -143,6 +143,7 @@ struct otx2_cpt_eng_grp_info {
};
struct otx2_cpt_eng_grps {
struct mutex lock;
struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS];
struct otx2_cpt_engs_available avail;
void *obj; /* device specific data */
@ -160,5 +161,9 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf);
int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type);
int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf);
int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
struct devlink_param_gset_ctx *ctx);
int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
struct devlink_param_gset_ctx *ctx);
void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf);
#endif /* __OTX2_CPTPF_UCODE_H */

5
drivers/crypto/marvell/octeontx2/otx2_cptvf_algs.c
View file

@ -1682,11 +1682,8 @@ static void swap_func(void *lptr, void *rptr, int size)
{
struct cpt_device_desc *ldesc = lptr;
struct cpt_device_desc *rdesc = rptr;
struct cpt_device_desc desc;
desc = *ldesc;
*ldesc = *rdesc;
*rdesc = desc;
swap(*ldesc, *rdesc);
}
int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,

2
drivers/crypto/omap-aes.c
View file

@ -1302,7 +1302,7 @@ static int omap_aes_suspend(struct device *dev)
static int omap_aes_resume(struct device *dev)
{
pm_runtime_resume_and_get(dev);
pm_runtime_get_sync(dev);
return 0;
}
#endif

8
drivers/crypto/omap-des.c
View file

@ -735,7 +735,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "ecb(des)",
.base.cra_driver_name = "ecb-des-omap",
.base.cra_priority = 100,
.base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
@ -752,7 +752,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "cbc(des)",
.base.cra_driver_name = "cbc-des-omap",
.base.cra_priority = 100,
.base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES_BLOCK_SIZE,
@ -770,7 +770,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "ecb(des3_ede)",
.base.cra_driver_name = "ecb-des3-omap",
.base.cra_priority = 100,
.base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
@ -787,7 +787,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
{
.base.cra_name = "cbc(des3_ede)",
.base.cra_driver_name = "cbc-des3-omap",
.base.cra_priority = 100,
.base.cra_priority = 300,
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
CRYPTO_ALG_ASYNC,
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,

1
drivers/crypto/qat/Kconfig
View file

@ -13,6 +13,7 @@ config CRYPTO_DEV_QAT
select CRYPTO_SHA512
select CRYPTO_LIB_AES
select FW_LOADER
select CRC8
config CRYPTO_DEV_QAT_DH895xCC
tristate "Support for Intel(R) DH895xCC"

154
drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.c
View file

@ -1,10 +1,11 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2020 Intel Corporation */
/* Copyright(c) 2020 - 2021 Intel Corporation */
#include <linux/iopoll.h>
#include <adf_accel_devices.h>
#include <adf_cfg.h>
#include <adf_common_drv.h>
#include <adf_pf2vf_msg.h>
#include <adf_gen4_hw_data.h>
#include <adf_gen4_pfvf.h>
#include "adf_4xxx_hw_data.h"
#include "icp_qat_hw.h"
@ -13,12 +14,18 @@ struct adf_fw_config {
char *obj_name;
};
static struct adf_fw_config adf_4xxx_fw_config[] = {
static struct adf_fw_config adf_4xxx_fw_cy_config[] = {
{0xF0, ADF_4XXX_SYM_OBJ},
{0xF, ADF_4XXX_ASYM_OBJ},
{0x100, ADF_4XXX_ADMIN_OBJ},
};
static struct adf_fw_config adf_4xxx_fw_dc_config[] = {
{0xF0, ADF_4XXX_DC_OBJ},
{0xF, ADF_4XXX_DC_OBJ},
{0x100, ADF_4XXX_ADMIN_OBJ},
};
/* Worker thread to service arbiter mappings */
static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = {
0x5555555, 0x5555555, 0x5555555, 0x5555555,
@ -32,6 +39,39 @@ static struct adf_hw_device_class adf_4xxx_class = {
.instances = 0,
};
enum dev_services {
SVC_CY = 0,
SVC_DC,
};
static const char *const dev_cfg_services[] = {
[SVC_CY] = ADF_CFG_CY,
[SVC_DC] = ADF_CFG_DC,
};
static int get_service_enabled(struct adf_accel_dev *accel_dev)
{
char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
u32 ret;
ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
ADF_SERVICES_ENABLED, services);
if (ret) {
dev_err(&GET_DEV(accel_dev),
ADF_SERVICES_ENABLED " param not found\n");
return ret;
}
ret = match_string(dev_cfg_services, ARRAY_SIZE(dev_cfg_services),
services);
if (ret < 0)
dev_err(&GET_DEV(accel_dev),
"Invalid value of " ADF_SERVICES_ENABLED " param: %s\n",
services);
return ret;
}
static u32 get_accel_mask(struct adf_hw_device_data *self)
{
return ADF_4XXX_ACCELERATORS_MASK;
@ -96,23 +136,67 @@ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev)
static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
{
struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev;
u32 capabilities_cy, capabilities_dc;
u32 fusectl1;
u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
ICP_ACCEL_CAPABILITIES_AES_V2;
/* Read accelerator capabilities mask */
pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1);
if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
capabilities_cy = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
ICP_ACCEL_CAPABILITIES_CIPHER |
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
ICP_ACCEL_CAPABILITIES_SHA3 |
ICP_ACCEL_CAPABILITIES_SHA3_EXT |
ICP_ACCEL_CAPABILITIES_HKDF |
ICP_ACCEL_CAPABILITIES_ECEDMONT |
ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
ICP_ACCEL_CAPABILITIES_AES_V2;
return capabilities;
/* A set bit in fusectl1 means the feature is OFF in this SKU */
if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) {
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_HKDF;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
}
if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) {
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
}
if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) {
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
}
if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) {
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT;
}
capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) {
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
}
switch (get_service_enabled(accel_dev)) {
case SVC_CY:
return capabilities_cy;
case SVC_DC:
return capabilities_dc;
}
return 0;
}
static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
@ -191,29 +275,36 @@ static int adf_init_device(struct adf_accel_dev *accel_dev)
return ret;
}
static int pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
{
return 0;
}
static u32 uof_get_num_objs(void)
{
return ARRAY_SIZE(adf_4xxx_fw_config);
BUILD_BUG_ON_MSG(ARRAY_SIZE(adf_4xxx_fw_cy_config) !=
ARRAY_SIZE(adf_4xxx_fw_dc_config),
"Size mismatch between adf_4xxx_fw_*_config arrays");
return ARRAY_SIZE(adf_4xxx_fw_cy_config);
}
static char *uof_get_name(u32 obj_num)
static char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
{
return adf_4xxx_fw_config[obj_num].obj_name;
switch (get_service_enabled(accel_dev)) {
case SVC_CY:
return adf_4xxx_fw_cy_config[obj_num].obj_name;
case SVC_DC:
return adf_4xxx_fw_dc_config[obj_num].obj_name;
}
return NULL;
}
static u32 uof_get_ae_mask(u32 obj_num)
static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
{
return adf_4xxx_fw_config[obj_num].ae_mask;
}
switch (get_service_enabled(accel_dev)) {
case SVC_CY:
return adf_4xxx_fw_cy_config[obj_num].ae_mask;
case SVC_DC:
return adf_4xxx_fw_dc_config[obj_num].ae_mask;
}
static u32 get_vf2pf_sources(void __iomem *pmisc_addr)
{
/* For the moment do not report vf2pf sources */
return 0;
}
@ -222,12 +313,14 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
hw_data->dev_class = &adf_4xxx_class;
hw_data->instance_id = adf_4xxx_class.instances++;
hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS;
hw_data->num_banks_per_vf = ADF_4XXX_NUM_BANKS_PER_VF;
hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK;
hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS;
hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES;
hw_data->num_logical_accel = 1;
hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK;
hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_enable_error_correction;
@ -259,12 +352,11 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
hw_data->uof_get_ae_mask = uof_get_ae_mask;
hw_data->set_msix_rttable = set_msix_default_rttable;
hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
hw_data->enable_pfvf_comms = pfvf_comms_disabled;
hw_data->get_vf2pf_sources = get_vf2pf_sources;
hw_data->disable_iov = adf_disable_sriov;
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
hw_data->ring_pair_reset = adf_gen4_ring_pair_reset;
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);
adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops);
}
void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data)

2
drivers/crypto/qat/qat_4xxx/adf_4xxx_hw_data.h
View file

@ -37,6 +37,7 @@
/* Bank and ring configuration */
#define ADF_4XXX_NUM_RINGS_PER_BANK 2
#define ADF_4XXX_NUM_BANKS_PER_VF 4
/* Error source registers */
#define ADF_4XXX_ERRSOU0 (0x41A200)
@ -76,6 +77,7 @@
#define ADF_4XXX_FW "qat_4xxx.bin"
#define ADF_4XXX_MMP "qat_4xxx_mmp.bin"
#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin"
#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin"
#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin"
#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin"

33
drivers/crypto/qat/qat_4xxx/adf_drv.c
View file

@ -29,6 +29,29 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
adf_devmgr_rm_dev(accel_dev, NULL);
}
static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev)
{
const char *config;
int ret;
config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY;
ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC);
if (ret)
return ret;
/* Default configuration is crypto only for even devices
* and compression for odd devices
*/
ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
ADF_SERVICES_ENABLED, config,
ADF_STR);
if (ret)
return ret;
return 0;
}
static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev)
{
char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
@ -227,8 +250,18 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
goto out_err;
}
ret = adf_cfg_dev_init(accel_dev);
if (ret) {
dev_err(&pdev->dev, "Failed to initialize configuration.\n");
goto out_err;
}
/* Get accelerator capabilities mask */
hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev);
if (!hw_data->accel_capabilities_mask) {
dev_err(&pdev->dev, "Failed to get capabilities mask.\n");
goto out_err;
}
/* Find and map all the device's BARS */
bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK;

12
drivers/crypto/qat/qat_c3xxx/adf_c3xxx_hw_data.c
View file

@ -1,9 +1,9 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2014 - 2020 Intel Corporation */
/* Copyright(c) 2014 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
#include <adf_pf2vf_msg.h>
#include <adf_gen2_hw_data.h>
#include <adf_gen2_pfvf.h>
#include "adf_c3xxx_hw_data.h"
#include "icp_qat_hw.h"
@ -109,6 +109,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
@ -135,14 +136,9 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data)
hw_data->enable_ints = adf_enable_ints;
hw_data->reset_device = adf_reset_flr;
hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer;
hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
hw_data->disable_iov = adf_disable_sriov;
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}

15
drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.c
View file

@ -1,9 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2015 - 2020 Intel Corporation */
/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_gen2_hw_data.h>
#include <adf_gen2_pfvf.h>
#include <adf_pfvf_vf_msg.h>
#include "adf_c3xxxvf_hw_data.h"
static struct adf_hw_device_class c3xxxiov_class = {
@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
return DEV_SKU_VF;
}
static u32 get_pf2vf_offset(u32 i)
{
return ADF_C3XXXIOV_PF2VF_OFFSET;
}
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
{
return 0;
@ -71,6 +67,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK;
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
hw_data->free_irq = adf_vf_isr_resource_free;
hw_data->enable_error_correction = adf_vf_void_noop;
@ -86,13 +83,11 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data)
hw_data->get_num_aes = get_num_aes;
hw_data->get_etr_bar_id = get_etr_bar_id;
hw_data->get_misc_bar_id = get_misc_bar_id;
hw_data->get_pf2vf_offset = get_pf2vf_offset;
hw_data->get_sku = get_sku;
hw_data->enable_ints = adf_vf_void_noop;
hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
hw_data->dev_class->instances++;
adf_devmgr_update_class_index(hw_data);
adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}

1
drivers/crypto/qat/qat_c3xxxvf/adf_c3xxxvf_hw_data.h
View file

@ -12,7 +12,6 @@
#define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF
#define ADF_C3XXXIOV_ETR_BAR 0
#define ADF_C3XXXIOV_ETR_MAX_BANKS 1
#define ADF_C3XXXIOV_PF2VF_OFFSET 0x200
void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data);
void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data);

6
drivers/crypto/qat/qat_c3xxxvf/adf_drv.c
View file

@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
pci_set_master(pdev);
/* Completion for VF2PF request/response message exchange */
init_completion(&accel_dev->vf.iov_msg_completion);
ret = qat_crypto_dev_config(accel_dev);
if (ret)
goto out_err_free_reg;
init_completion(&accel_dev->vf.msg_received);
ret = adf_dev_init(accel_dev);
if (ret)

12
drivers/crypto/qat/qat_c62x/adf_c62x_hw_data.c
View file

@ -1,9 +1,9 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2014 - 2020 Intel Corporation */
/* Copyright(c) 2014 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
#include <adf_common_drv.h>
#include <adf_pf2vf_msg.h>
#include <adf_gen2_hw_data.h>
#include <adf_gen2_pfvf.h>
#include "adf_c62x_hw_data.h"
#include "icp_qat_hw.h"
@ -111,6 +111,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_isr_resource_alloc;
hw_data->free_irq = adf_isr_resource_free;
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
@ -137,14 +138,9 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data)
hw_data->enable_ints = adf_enable_ints;
hw_data->reset_device = adf_reset_flr;
hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer;
hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
hw_data->disable_iov = adf_disable_sriov;
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}

15
drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.c
View file

@ -1,9 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2015 - 2020 Intel Corporation */
/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <adf_accel_devices.h>
#include <adf_pf2vf_msg.h>
#include <adf_common_drv.h>
#include <adf_gen2_hw_data.h>
#include <adf_gen2_pfvf.h>
#include <adf_pfvf_vf_msg.h>
#include "adf_c62xvf_hw_data.h"
static struct adf_hw_device_class c62xiov_class = {
@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
return DEV_SKU_VF;
}
static u32 get_pf2vf_offset(u32 i)
{
return ADF_C62XIOV_PF2VF_OFFSET;
}
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
{
return 0;
@ -71,6 +67,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data)
hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES;
hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET;
hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK;
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
hw_data->free_irq = adf_vf_isr_resource_free;
hw_data->enable_error_correction = adf_vf_void_noop;
@ -86,13 +83,11 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data)
hw_data->get_num_aes = get_num_aes;
hw_data->get_etr_bar_id = get_etr_bar_id;
hw_data->get_misc_bar_id = get_misc_bar_id;
hw_data->get_pf2vf_offset = get_pf2vf_offset;
hw_data->get_sku = get_sku;
hw_data->enable_ints = adf_vf_void_noop;
hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
hw_data->dev_class->instances++;
adf_devmgr_update_class_index(hw_data);
adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
}

1
drivers/crypto/qat/qat_c62xvf/adf_c62xvf_hw_data.h
View file

@ -12,7 +12,6 @@
#define ADF_C62XIOV_TX_RINGS_MASK 0xFF
#define ADF_C62XIOV_ETR_BAR 0
#define ADF_C62XIOV_ETR_MAX_BANKS 1
#define ADF_C62XIOV_PF2VF_OFFSET 0x200
void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data);
void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data);

6
drivers/crypto/qat/qat_c62xvf/adf_drv.c
View file

@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
}
pci_set_master(pdev);
/* Completion for VF2PF request/response message exchange */
init_completion(&accel_dev->vf.iov_msg_completion);
ret = qat_crypto_dev_config(accel_dev);
if (ret)
goto out_err_free_reg;
init_completion(&accel_dev->vf.msg_received);
ret = adf_dev_init(accel_dev);
if (ret)

6
drivers/crypto/qat/qat_common/Makefile
View file

@ -19,5 +19,7 @@ intel_qat-objs := adf_cfg.o \
qat_hal.o
intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o
intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \
adf_vf2pf_msg.o adf_vf_isr.o
intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \
adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \
adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \
adf_gen2_pfvf.o adf_gen4_pfvf.o

47
drivers/crypto/qat/qat_common/adf_accel_devices.h
View file

@ -8,6 +8,7 @@
#include <linux/io.h>
#include <linux/ratelimit.h>
#include "adf_cfg_common.h"
#include "adf_pfvf_msg.h"
#define ADF_DH895XCC_DEVICE_NAME "dh895xcc"
#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf"
@ -147,6 +148,19 @@ struct adf_accel_dev;
struct adf_etr_data;
struct adf_etr_ring_data;
struct adf_pfvf_ops {
int (*enable_comms)(struct adf_accel_dev *accel_dev);
u32 (*get_pf2vf_offset)(u32 i);
u32 (*get_vf2pf_offset)(u32 i);
u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr);
void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);
void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);
int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
u32 pfvf_offset, struct mutex *csr_lock);
struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev,
u32 pfvf_offset, u8 compat_ver);
};
struct adf_hw_device_data {
struct adf_hw_device_class *dev_class;
u32 (*get_accel_mask)(struct adf_hw_device_data *self);
@ -157,7 +171,6 @@ struct adf_hw_device_data {
u32 (*get_etr_bar_id)(struct adf_hw_device_data *self);
u32 (*get_num_aes)(struct adf_hw_device_data *self);
u32 (*get_num_accels)(struct adf_hw_device_data *self);
u32 (*get_pf2vf_offset)(u32 i);
void (*get_arb_info)(struct arb_info *arb_csrs_info);
void (*get_admin_info)(struct admin_info *admin_csrs_info);
enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self);
@ -176,35 +189,34 @@ struct adf_hw_device_data {
bool enable);
void (*enable_ints)(struct adf_accel_dev *accel_dev);
void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);
int (*enable_pfvf_comms)(struct adf_accel_dev *accel_dev);
u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr);
void (*enable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr,
u32 vf_mask);
void (*disable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr,
u32 vf_mask);
int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr);
void (*reset_device)(struct adf_accel_dev *accel_dev);
void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
char *(*uof_get_name)(u32 obj_num);
char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num);
u32 (*uof_get_num_objs)(void);
u32 (*uof_get_ae_mask)(u32 obj_num);
u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num);
struct adf_pfvf_ops pfvf_ops;
struct adf_hw_csr_ops csr_ops;
const char *fw_name;
const char *fw_mmp_name;
u32 fuses;
u32 straps;
u32 accel_capabilities_mask;
u32 extended_dc_capabilities;
u32 clock_frequency;
u32 instance_id;
u16 accel_mask;
u32 ae_mask;
u32 admin_ae_mask;
u16 tx_rings_mask;
u16 ring_to_svc_map;
u8 tx_rx_gap;
u8 num_banks;
u16 num_banks_per_vf;
u8 num_rings_per_bank;
u8 num_accel;
u8 num_logical_accel;
u8 num_engines;
u8 min_iov_compat_ver;
};
/* CSR write macro */
@ -214,14 +226,22 @@ struct adf_hw_device_data {
/* CSR read macro */
#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset)
#define ADF_CFG_NUM_SERVICES 4
#define ADF_SRV_TYPE_BIT_LEN 3
#define ADF_SRV_TYPE_MASK 0x7
#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev)
#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars)
#define GET_HW_DATA(accel_dev) (accel_dev->hw_device)
#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks)
#define GET_NUM_RINGS_PER_BANK(accel_dev) \
GET_HW_DATA(accel_dev)->num_rings_per_bank
#define GET_SRV_TYPE(accel_dev, idx) \
(((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \
& ADF_SRV_TYPE_MASK)
#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines)
#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops)
#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops)
#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev
struct adf_admin_comms;
@ -238,6 +258,7 @@ struct adf_accel_vf_info {
struct ratelimit_state vf2pf_ratelimit;
u32 vf_nr;
bool init;
u8 vf_compat_ver;
};
struct adf_accel_dev {
@ -265,9 +286,9 @@ struct adf_accel_dev {
char irq_name[ADF_MAX_MSIX_VECTOR_NAME];
struct tasklet_struct pf2vf_bh_tasklet;
struct mutex vf2pf_lock; /* protect CSR access */
struct completion iov_msg_completion;
u8 compatible;
u8 pf_version;
struct completion msg_received;
struct pfvf_message response; /* temp field holding pf2vf response */
u8 pf_compat_ver;
} vf;
};
bool is_vf;

8
drivers/crypto/qat/qat_common/adf_accel_engine.c
View file

@ -22,8 +22,12 @@ static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr,
num_objs = hw_device->uof_get_num_objs();
for (i = 0; i < num_objs; i++) {
obj_name = hw_device->uof_get_name(i);
ae_mask = hw_device->uof_get_ae_mask(i);
obj_name = hw_device->uof_get_name(accel_dev, i);
ae_mask = hw_device->uof_get_ae_mask(accel_dev, i);
if (!obj_name || !ae_mask) {
dev_err(&GET_DEV(accel_dev), "Invalid UOF image\n");
goto out_err;
}
if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) {
dev_err(&GET_DEV(accel_dev),

47
drivers/crypto/qat/qat_common/adf_admin.c
View file

@ -194,6 +194,35 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
return adf_send_admin(accel_dev, &req, &resp, ae_mask);
}
static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev,
u32 *capabilities)
{
struct adf_hw_device_data *hw_device = accel_dev->hw_device;
struct icp_qat_fw_init_admin_resp resp;
struct icp_qat_fw_init_admin_req req;
unsigned long ae_mask;
unsigned long ae;
int ret;
/* Target only service accelerator engines */
ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask;
memset(&req, 0, sizeof(req));
memset(&resp, 0, sizeof(resp));
req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET;
*capabilities = 0;
for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) {
ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae);
if (ret)
return ret;
*capabilities |= resp.extended_features;
}
return 0;
}
/**
* adf_send_admin_init() - Function sends init message to FW
* @accel_dev: Pointer to acceleration device.
@ -204,8 +233,16 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
*/
int adf_send_admin_init(struct adf_accel_dev *accel_dev)
{
u32 dc_capabilities = 0;
int ret;
ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities);
if (ret) {
dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n");
return ret;
}
accel_dev->hw_device->extended_dc_capabilities = dc_capabilities;
ret = adf_set_fw_constants(accel_dev);
if (ret)
return ret;
@ -218,9 +255,7 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
{
struct adf_admin_comms *admin;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct adf_bar *pmisc =
&GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
void __iomem *csr = pmisc->virt_addr;
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
struct admin_info admin_csrs_info;
u32 mailbox_offset, adminmsg_u, adminmsg_l;
void __iomem *mailbox;
@ -254,13 +289,13 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
hw_data->get_admin_info(&admin_csrs_info);
mailbox_offset = admin_csrs_info.mailbox_offset;
mailbox = csr + mailbox_offset;
mailbox = pmisc_addr + mailbox_offset;
adminmsg_u = admin_csrs_info.admin_msg_ur;
adminmsg_l = admin_csrs_info.admin_msg_lr;
reg_val = (u64)admin->phy_addr;
ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val));
ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val));
ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val));
ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val));
mutex_init(&admin->lock);
admin->mailbox_addr = mailbox;

1
drivers/crypto/qat/qat_common/adf_cfg.c
View file

@ -297,3 +297,4 @@ int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev,
up_read(&cfg->lock);
return ret;
}
EXPORT_SYMBOL_GPL(adf_cfg_get_param_value);

13
drivers/crypto/qat/qat_common/adf_cfg_common.h
View file

@ -19,6 +19,19 @@
#define ADF_MAX_DEVICES (32 * 32)
#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0
#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3
#define ADF_CFG_SERV_RING_PAIR_2_SHIFT 6
#define ADF_CFG_SERV_RING_PAIR_3_SHIFT 9
enum adf_cfg_service_type {
UNUSED = 0,
CRYPTO,
COMP,
SYM,
ASYM,
USED
};
enum adf_cfg_val_type {
ADF_DEC,
ADF_HEX,

3
drivers/crypto/qat/qat_common/adf_cfg_strings.h
View file

@ -22,6 +22,9 @@
#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber"
#define ADF_CY "Cy"
#define ADF_DC "Dc"
#define ADF_CFG_DC "dc"
#define ADF_CFG_CY "sym;asym"
#define ADF_SERVICES_ENABLED "ServicesEnabled"
#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled"
#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \
ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED

42
drivers/crypto/qat/qat_common/adf_common_drv.h
View file

@ -1,5 +1,5 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2014 - 2020 Intel Corporation */
/* Copyright(c) 2014 - 2021 Intel Corporation */
#ifndef ADF_DRV_H
#define ADF_DRV_H
@ -62,9 +62,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev);
void adf_dev_stop(struct adf_accel_dev *accel_dev);
void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev);
int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev);
void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info);
void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data);
void adf_clean_vf_map(bool);
@ -117,6 +114,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev);
int qat_crypto_register(void);
int qat_crypto_unregister(void);
int qat_crypto_dev_config(struct adf_accel_dev *accel_dev);
int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev);
struct qat_crypto_instance *qat_crypto_get_instance_node(int node);
void qat_crypto_put_instance(struct qat_crypto_instance *inst);
void qat_alg_callback(void *resp);
@ -131,6 +129,8 @@ void adf_isr_resource_free(struct adf_accel_dev *accel_dev);
int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev);
void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev);
int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev);
int qat_hal_init(struct adf_accel_dev *accel_dev);
void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
@ -193,17 +193,14 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs);
void adf_disable_sriov(struct adf_accel_dev *accel_dev);
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev,
u32 vf_mask);
void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
u32 vf_mask);
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev,
u32 vf_mask);
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev);
bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev);
bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr);
int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev);
void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev);
void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev);
void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info);
int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg);
int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev);
void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev);
int adf_init_pf_wq(void);
void adf_exit_pf_wq(void);
int adf_init_vf_wq(void);
@ -212,11 +209,6 @@ void adf_flush_vf_wq(struct adf_accel_dev *accel_dev);
#else
#define adf_sriov_configure NULL
static inline int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
{
return 0;
}
static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev)
{
}
@ -229,15 +221,6 @@ static inline void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
{
}
static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev)
{
return 0;
}
static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev)
{
}
static inline int adf_init_pf_wq(void)
{
return 0;
@ -261,4 +244,15 @@ static inline void adf_flush_vf_wq(struct adf_accel_dev *accel_dev)
}
#endif
static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct adf_bar *pmisc;
pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
return pmisc->virt_addr;
}
#endif

105
drivers/crypto/qat/qat_common/adf_gen2_hw_data.c
View file

@ -1,57 +1,10 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2020 Intel Corporation */
#include "adf_common_drv.h"
#include "adf_gen2_hw_data.h"
#include "icp_qat_hw.h"
#include <linux/pci.h>
#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
u32 adf_gen2_get_pf2vf_offset(u32 i)
{
return ADF_GEN2_PF2VF_OFFSET(i);
}
EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset);
u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr)
{
u32 errsou3, errmsk3, vf_int_mask;
/* Get the interrupt sources triggered by VFs */
errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3);
/* To avoid adding duplicate entries to work queue, clear
* vf_int_mask_sets bits that are already masked in ERRMSK register.
*/
errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3);
return vf_int_mask;
}
EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources);
void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
{
/* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
if (vf_mask & 0xFFFF) {
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
& ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
}
}
EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts);
void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
{
/* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
if (vf_mask & 0xFFFF) {
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
| ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
}
}
EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts);
u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self)
{
if (!self || !self->accel_mask)
@ -73,31 +26,29 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes);
void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct adf_bar *misc_bar = &GET_BARS(accel_dev)
[hw_data->get_misc_bar_id(hw_data)];
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
unsigned long accel_mask = hw_data->accel_mask;
unsigned long ae_mask = hw_data->ae_mask;
void __iomem *csr = misc_bar->virt_addr;
unsigned int val, i;
/* Enable Accel Engine error detection & correction */
for_each_set_bit(i, &ae_mask, hw_data->num_engines) {
val = ADF_CSR_RD(csr, ADF_GEN2_AE_CTX_ENABLES(i));
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i));
val |= ADF_GEN2_ENABLE_AE_ECC_ERR;
ADF_CSR_WR(csr, ADF_GEN2_AE_CTX_ENABLES(i), val);
val = ADF_CSR_RD(csr, ADF_GEN2_AE_MISC_CONTROL(i));
ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i), val);
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i));
val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR;
ADF_CSR_WR(csr, ADF_GEN2_AE_MISC_CONTROL(i), val);
ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i), val);
}
/* Enable shared memory error detection & correction */
for_each_set_bit(i, &accel_mask, hw_data->num_accel) {
val = ADF_CSR_RD(csr, ADF_GEN2_UERRSSMSH(i));
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_UERRSSMSH(i));
val |= ADF_GEN2_ERRSSMSH_EN;
ADF_CSR_WR(csr, ADF_GEN2_UERRSSMSH(i), val);
val = ADF_CSR_RD(csr, ADF_GEN2_CERRSSMSH(i));
ADF_CSR_WR(pmisc_addr, ADF_GEN2_UERRSSMSH(i), val);
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_CERRSSMSH(i));
val |= ADF_GEN2_ERRSSMSH_EN;
ADF_CSR_WR(csr, ADF_GEN2_CERRSSMSH(i), val);
ADF_CSR_WR(pmisc_addr, ADF_GEN2_CERRSSMSH(i), val);
}
}
EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction);
@ -105,15 +56,9 @@ EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction);
void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable,
int num_a_regs, int num_b_regs)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
void __iomem *pmisc_addr;
struct adf_bar *pmisc;
int pmisc_id, i;
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u32 reg;
pmisc_id = hw_data->get_misc_bar_id(hw_data);
pmisc = &GET_BARS(accel_dev)[pmisc_id];
pmisc_addr = pmisc->virt_addr;
int i;
/* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */
for (i = 0; i < num_a_regs; i++) {
@ -259,21 +204,33 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev)
u32 legfuses;
u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
ICP_ACCEL_CAPABILITIES_CIPHER |
ICP_ACCEL_CAPABILITIES_COMPRESSION;
/* Read accelerator capabilities mask */
pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses);
if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE)
/* A set bit in legfuses means the feature is OFF in this SKU */
if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) {
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
}
if (legfuses & ICP_ACCEL_MASK_PKE_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE)
if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) {
capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
}
if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
if ((straps | fuses) & ADF_POWERGATE_PKE)
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
if ((straps | fuses) & ADF_POWERGATE_DC)
capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
return capabilities;
}
EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap);
@ -281,18 +238,12 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap);
void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
u32 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
unsigned long accel_mask = hw_data->accel_mask;
void __iomem *pmisc_addr;
struct adf_bar *pmisc;
int pmisc_id;
u32 i = 0;
pmisc_id = hw_data->get_misc_bar_id(hw_data);
pmisc = &GET_BARS(accel_dev)[pmisc_id];
pmisc_addr = pmisc->virt_addr;
/* Configures WDT timers */
for_each_set_bit(i, &accel_mask, hw_data->num_accel) {
/* Enable WDT for sym and dc */

22
drivers/crypto/qat/qat_common/adf_gen2_hw_data.h
View file

@ -4,6 +4,7 @@
#define ADF_GEN2_HW_DATA_H_
#include "adf_accel_devices.h"
#include "adf_cfg_common.h"
/* Transport access */
#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL
@ -113,8 +114,16 @@ do { \
(ADF_ARB_REG_SLOT * (index)), value)
/* Power gating */
#define ADF_POWERGATE_DC BIT(23)
#define ADF_POWERGATE_PKE BIT(24)
/* Default ring mapping */
#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \
(CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
/* WDT timers
*
* Timeout is in cycles. Clock speed may vary across products but this
@ -136,19 +145,6 @@ do { \
#define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10)
#define ADF_GEN2_ERRSSMSH_EN BIT(3)
/* VF2PF interrupts */
#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C)
#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8)
#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C)
#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC)
#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9)
u32 adf_gen2_get_pf2vf_offset(u32 i);
u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar);
void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask);
void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask);
u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self);
u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self);
void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev);

381
drivers/crypto/qat/qat_common/adf_gen2_pfvf.c Normal file
View file

@ -0,0 +1,381 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2021 Intel Corporation */
#include <linux/delay.h>
#include <linux/iopoll.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_gen2_pfvf.h"
#include "adf_pfvf_msg.h"
#include "adf_pfvf_pf_proto.h"
#include "adf_pfvf_vf_proto.h"
#include "adf_pfvf_utils.h"
/* VF2PF interrupts */
#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9)
#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
#define ADF_GEN2_VF_PF2VF_OFFSET 0x200
#define ADF_GEN2_CSR_IN_USE 0x6AC2
#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE
enum gen2_csr_pos {
ADF_GEN2_CSR_PF2VF_OFFSET = 0,
ADF_GEN2_CSR_VF2PF_OFFSET = 16,
};
#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2
#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F
#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6
#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF
static const struct pfvf_csr_format csr_gen2_fmt = {
{ ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK },
{ ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK },
};
#define ADF_PFVF_MSG_RETRY_DELAY 5
#define ADF_PFVF_MSG_MAX_RETRIES 3
static u32 adf_gen2_pf_get_pfvf_offset(u32 i)
{
return ADF_GEN2_PF_PF2VF_OFFSET(i);
}
static u32 adf_gen2_vf_get_pfvf_offset(u32 i)
{
return ADF_GEN2_VF_PF2VF_OFFSET;
}
static u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr)
{
u32 errsou3, errmsk3, vf_int_mask;
/* Get the interrupt sources triggered by VFs */
errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3);
/* To avoid adding duplicate entries to work queue, clear
* vf_int_mask_sets bits that are already masked in ERRMSK register.
*/
errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3);
return vf_int_mask;
}
static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr,
u32 vf_mask)
{
/* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
if (vf_mask & 0xFFFF) {
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
& ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
}
}
static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr,
u32 vf_mask)
{
/* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
if (vf_mask & 0xFFFF) {
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
| ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
}
}
static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset)
{
return ADF_PFVF_INT << offset;
}
static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset)
{
return (csr_msg & 0xFFFF) << offset;
}
static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset)
{
return (csr_val >> offset) & 0xFFFF;
}
static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset)
{
return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE;
}
static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset)
{
*msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset);
}
static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset)
{
*msg |= (ADF_GEN2_CSR_IN_USE << offset);
}
static bool is_legacy_user_pfvf_message(u32 msg)
{
return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM);
}
static bool is_pf2vf_notification(u8 msg_type)
{
switch (msg_type) {
case ADF_PF2VF_MSGTYPE_RESTARTING:
return true;
default:
return false;
}
}
static bool is_vf2pf_notification(u8 msg_type)
{
switch (msg_type) {
case ADF_VF2PF_MSGTYPE_INIT:
case ADF_VF2PF_MSGTYPE_SHUTDOWN:
return true;
default:
return false;
}
}
struct pfvf_gen2_params {
u32 pfvf_offset;
struct mutex *csr_lock; /* lock preventing concurrent access of CSR */
enum gen2_csr_pos local_offset;
enum gen2_csr_pos remote_offset;
bool (*is_notification_message)(u8 msg_type);
u8 compat_ver;
};
static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev,
struct pfvf_message msg,
struct pfvf_gen2_params *params)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
enum gen2_csr_pos remote_offset = params->remote_offset;
enum gen2_csr_pos local_offset = params->local_offset;
unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES;
struct mutex *lock = params->csr_lock;
u32 pfvf_offset = params->pfvf_offset;
u32 int_bit;
u32 csr_val;
u32 csr_msg;
int ret;
/* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This
* allows us to build and read messages as if they where all 0 based.
* However, send and receive are in a single shared 32 bits register,
* so we need to shift and/or mask the message half before decoding
* it and after encoding it. Which one to shift depends on the
* direction.
*/
int_bit = gen2_csr_get_int_bit(local_offset);
csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt);
if (unlikely(!csr_msg))
return -EINVAL;
/* Prepare for CSR format, shifting the wire message in place and
* setting the in use pattern
*/
csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset);
gen2_csr_set_in_use(&csr_msg, remote_offset);
mutex_lock(lock);
start:
/* Check if the PFVF CSR is in use by remote function */
csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
if (gen2_csr_is_in_use(csr_val, local_offset)) {
dev_dbg(&GET_DEV(accel_dev),
"PFVF CSR in use by remote function\n");
goto retry;
}
/* Attempt to get ownership of the PFVF CSR */
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit);
/* Wait for confirmation from remote func it received the message */
ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit),
ADF_PFVF_MSG_ACK_DELAY_US,
ADF_PFVF_MSG_ACK_MAX_DELAY_US,
true, pmisc_addr, pfvf_offset);
if (unlikely(ret < 0)) {
dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
csr_val &= ~int_bit;
}
/* For fire-and-forget notifications, the receiver does not clear
* the in-use pattern. This is used to detect collisions.
*/
if (params->is_notification_message(msg.type) && csr_val != csr_msg) {
/* Collision must have overwritten the message */
dev_err(&GET_DEV(accel_dev),
"Collision on notification - PFVF CSR overwritten by remote function\n");
goto retry;
}
/* If the far side did not clear the in-use pattern it is either
* 1) Notification - message left intact to detect collision
* 2) Older protocol (compatibility version < 3) on the far side
* where the sender is responsible for clearing the in-use
* pattern after the received has acknowledged receipt.
* In either case, clear the in-use pattern now.
*/
if (gen2_csr_is_in_use(csr_val, remote_offset)) {
gen2_csr_clear_in_use(&csr_val, remote_offset);
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
}
out:
mutex_unlock(lock);
return ret;
retry:
if (--retries) {
msleep(ADF_PFVF_MSG_RETRY_DELAY);
goto start;
} else {
ret = -EBUSY;
goto out;
}
}
static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,
struct pfvf_gen2_params *params)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
enum gen2_csr_pos remote_offset = params->remote_offset;
enum gen2_csr_pos local_offset = params->local_offset;
u32 pfvf_offset = params->pfvf_offset;
struct pfvf_message msg = { 0 };
u32 int_bit;
u32 csr_val;
u16 csr_msg;
int_bit = gen2_csr_get_int_bit(local_offset);
/* Read message */
csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
if (!(csr_val & int_bit)) {
dev_info(&GET_DEV(accel_dev),
"Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val);
return msg;
}
/* Extract the message from the CSR */
csr_msg = gen2_csr_msg_from_position(csr_val, local_offset);
/* Ignore legacy non-system (non-kernel) messages */
if (unlikely(is_legacy_user_pfvf_message(csr_msg))) {
dev_dbg(&GET_DEV(accel_dev),
"Ignored non-system message (0x%.8x);\n", csr_val);
/* Because this must be a legacy message, the far side
* must clear the in-use pattern, so don't do it.
*/
return msg;
}
/* Return the pfvf_message format */
msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt);
/* The in-use pattern is not cleared for notifications (so that
* it can be used for collision detection) or older implementations
*/
if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK &&
!params->is_notification_message(msg.type))
gen2_csr_clear_in_use(&csr_val, remote_offset);
/* To ACK, clear the INT bit */
csr_val &= ~int_bit;
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
return msg;
}
static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
u32 pfvf_offset, struct mutex *csr_lock)
{
struct pfvf_gen2_params params = {
.csr_lock = csr_lock,
.pfvf_offset = pfvf_offset,
.local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
.remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
.is_notification_message = is_pf2vf_notification,
};
return adf_gen2_pfvf_send(accel_dev, msg, &params);
}
static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
u32 pfvf_offset, struct mutex *csr_lock)
{
struct pfvf_gen2_params params = {
.csr_lock = csr_lock,
.pfvf_offset = pfvf_offset,
.local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
.remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
.is_notification_message = is_vf2pf_notification,
};
return adf_gen2_pfvf_send(accel_dev, msg, &params);
}
static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev,
u32 pfvf_offset, u8 compat_ver)
{
struct pfvf_gen2_params params = {
.pfvf_offset = pfvf_offset,
.local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
.remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
.is_notification_message = is_pf2vf_notification,
.compat_ver = compat_ver,
};
return adf_gen2_pfvf_recv(accel_dev, &params);
}
static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev,
u32 pfvf_offset, u8 compat_ver)
{
struct pfvf_gen2_params params = {
.pfvf_offset = pfvf_offset,
.local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
.remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
.is_notification_message = is_vf2pf_notification,
.compat_ver = compat_ver,
};
return adf_gen2_pfvf_recv(accel_dev, &params);
}
void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
{
pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset;
pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset;
pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
pfvf_ops->send_msg = adf_gen2_pf2vf_send;
pfvf_ops->recv_msg = adf_gen2_vf2pf_recv;
}
EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops);
void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
{
pfvf_ops->enable_comms = adf_enable_vf2pf_comms;
pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset;
pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset;
pfvf_ops->send_msg = adf_gen2_vf2pf_send;
pfvf_ops->recv_msg = adf_gen2_pf2vf_recv;
}
EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops);

29
drivers/crypto/qat/qat_common/adf_gen2_pfvf.h Normal file
View file

@ -0,0 +1,29 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2021 Intel Corporation */
#ifndef ADF_GEN2_PFVF_H
#define ADF_GEN2_PFVF_H
#include <linux/types.h>
#include "adf_accel_devices.h"
#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C)
#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8)
#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C)
#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC)
#if defined(CONFIG_PCI_IOV)
void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
#else
static inline void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
{
pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
}
static inline void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
{
pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
}
#endif
#endif /* ADF_GEN2_PFVF_H */

69
drivers/crypto/qat/qat_common/adf_gen4_hw_data.c
View file

@ -1,6 +1,8 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2020 Intel Corporation */
#include <linux/iopoll.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_gen4_hw_data.h"
static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size)
@ -109,20 +111,13 @@ static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper,
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
u32 ssm_wdt_pke_high = 0;
u32 ssm_wdt_pke_low = 0;
u32 ssm_wdt_high = 0;
u32 ssm_wdt_low = 0;
void __iomem *pmisc_addr;
struct adf_bar *pmisc;
int pmisc_id;
pmisc_id = hw_data->get_misc_bar_id(hw_data);
pmisc = &GET_BARS(accel_dev)[pmisc_id];
pmisc_addr = pmisc->virt_addr;
/* Convert 64bit WDT timer value into 32bit values for
* mmio write to 32bit CSRs.
@ -139,3 +134,61 @@ void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high);
}
EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer);
int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
{
return 0;
}
EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled);
static int reset_ring_pair(void __iomem *csr, u32 bank_number)
{
u32 status;
int ret;
/* Write rpresetctl register BIT(0) as 1
* Since rpresetctl registers have no RW fields, no need to preserve
* values for other bits. Just write directly.
*/
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
ADF_WQM_CSR_RPRESETCTL_RESET);
/* Read rpresetsts register and wait for rp reset to complete */
ret = read_poll_timeout(ADF_CSR_RD, status,
status & ADF_WQM_CSR_RPRESETSTS_STATUS,
ADF_RPRESET_POLL_DELAY_US,
ADF_RPRESET_POLL_TIMEOUT_US, true,
csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
if (!ret) {
/* When rp reset is done, clear rpresetsts */
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number),
ADF_WQM_CSR_RPRESETSTS_STATUS);
}
return ret;
}
int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data);
void __iomem *csr;
int ret;
if (bank_number >= hw_data->num_banks)
return -EINVAL;
dev_dbg(&GET_DEV(accel_dev),
"ring pair reset for bank:%d\n", bank_number);
csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr;
ret = reset_ring_pair(csr, bank_number);
if (ret)
dev_err(&GET_DEV(accel_dev),
"ring pair reset failed (timeout)\n");
else
dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
return ret;
}
EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset);

17
drivers/crypto/qat/qat_common/adf_gen4_hw_data.h
View file

@ -4,6 +4,7 @@
#define ADF_GEN4_HW_CSR_DATA_H_
#include "adf_accel_devices.h"
#include "adf_cfg_common.h"
/* Transport access */
#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL
@ -94,6 +95,13 @@ do { \
ADF_RING_BUNDLE_SIZE * (bank) + \
ADF_RING_CSR_RING_SRV_ARB_EN, (value))
/* Default ring mapping */
#define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \
(ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
/* WDT timers
*
* Timeout is in cycles. Clock speed may vary across products but this
@ -106,6 +114,15 @@ do { \
#define ADF_SSMWDTPKEL_OFFSET 0x58
#define ADF_SSMWDTPKEH_OFFSET 0x60
/* Ring reset */
#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC)
#define ADF_RPRESET_POLL_DELAY_US 20
#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0)
#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3))
#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0)
#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4)
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops);
int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number);
#endif

148
drivers/crypto/qat/qat_common/adf_gen4_pfvf.c Normal file
View file

@ -0,0 +1,148 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2021 Intel Corporation */
#include <linux/iopoll.h>
#include <linux/mutex.h>
#include <linux/types.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_gen4_pfvf.h"
#include "adf_pfvf_pf_proto.h"
#include "adf_pfvf_utils.h"
#define ADF_4XXX_MAX_NUM_VFS 16
#define ADF_4XXX_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20))
#define ADF_4XXX_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20))
/* VF2PF interrupt source registers */
#define ADF_4XXX_VM2PF_SOU(i) (0x41A180 + ((i) * 4))
#define ADF_4XXX_VM2PF_MSK(i) (0x41A1C0 + ((i) * 4))
#define ADF_4XXX_VM2PF_INT_EN_MSK BIT(0)
#define ADF_PFVF_GEN4_MSGTYPE_SHIFT 2
#define ADF_PFVF_GEN4_MSGTYPE_MASK 0x3F
#define ADF_PFVF_GEN4_MSGDATA_SHIFT 8
#define ADF_PFVF_GEN4_MSGDATA_MASK 0xFFFFFF
static const struct pfvf_csr_format csr_gen4_fmt = {
{ ADF_PFVF_GEN4_MSGTYPE_SHIFT, ADF_PFVF_GEN4_MSGTYPE_MASK },
{ ADF_PFVF_GEN4_MSGDATA_SHIFT, ADF_PFVF_GEN4_MSGDATA_MASK },
};
static u32 adf_gen4_pf_get_pf2vf_offset(u32 i)
{
return ADF_4XXX_PF2VM_OFFSET(i);
}
static u32 adf_gen4_pf_get_vf2pf_offset(u32 i)
{
return ADF_4XXX_VM2PF_OFFSET(i);
}
static u32 adf_gen4_get_vf2pf_sources(void __iomem *pmisc_addr)
{
int i;
u32 sou, mask;
int num_csrs = ADF_4XXX_MAX_NUM_VFS;
u32 vf_mask = 0;
for (i = 0; i < num_csrs; i++) {
sou = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_SOU(i));
mask = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK(i));
sou &= ~mask;
vf_mask |= sou << i;
}
return vf_mask;
}
static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr,
u32 vf_mask)
{
int num_csrs = ADF_4XXX_MAX_NUM_VFS;
unsigned long mask = vf_mask;
unsigned int val;
int i;
for_each_set_bit(i, &mask, num_csrs) {
unsigned int offset = ADF_4XXX_VM2PF_MSK(i);
val = ADF_CSR_RD(pmisc_addr, offset) & ~ADF_4XXX_VM2PF_INT_EN_MSK;
ADF_CSR_WR(pmisc_addr, offset, val);
}
}
static void adf_gen4_disable_vf2pf_interrupts(void __iomem *pmisc_addr,
u32 vf_mask)
{
int num_csrs = ADF_4XXX_MAX_NUM_VFS;
unsigned long mask = vf_mask;
unsigned int val;
int i;
for_each_set_bit(i, &mask, num_csrs) {
unsigned int offset = ADF_4XXX_VM2PF_MSK(i);
val = ADF_CSR_RD(pmisc_addr, offset) | ADF_4XXX_VM2PF_INT_EN_MSK;
ADF_CSR_WR(pmisc_addr, offset, val);
}
}
static int adf_gen4_pfvf_send(struct adf_accel_dev *accel_dev,
struct pfvf_message msg, u32 pfvf_offset,
struct mutex *csr_lock)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u32 csr_val;
int ret;
csr_val = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen4_fmt);
if (unlikely(!csr_val))
return -EINVAL;
mutex_lock(csr_lock);
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val | ADF_PFVF_INT);
/* Wait for confirmation from remote that it received the message */
ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & ADF_PFVF_INT),
ADF_PFVF_MSG_ACK_DELAY_US,
ADF_PFVF_MSG_ACK_MAX_DELAY_US,
true, pmisc_addr, pfvf_offset);
if (ret < 0)
dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
mutex_unlock(csr_lock);
return ret;
}
static struct pfvf_message adf_gen4_pfvf_recv(struct adf_accel_dev *accel_dev,
u32 pfvf_offset, u8 compat_ver)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
u32 csr_val;
/* Read message from the CSR */
csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
/* We can now acknowledge the message reception by clearing the
* interrupt bit
*/
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val & ~ADF_PFVF_INT);
/* Return the pfvf_message format */
return adf_pfvf_message_of(accel_dev, csr_val, &csr_gen4_fmt);
}
void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
{
pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
pfvf_ops->get_pf2vf_offset = adf_gen4_pf_get_pf2vf_offset;
pfvf_ops->get_vf2pf_offset = adf_gen4_pf_get_vf2pf_offset;
pfvf_ops->get_vf2pf_sources = adf_gen4_get_vf2pf_sources;
pfvf_ops->enable_vf2pf_interrupts = adf_gen4_enable_vf2pf_interrupts;
pfvf_ops->disable_vf2pf_interrupts = adf_gen4_disable_vf2pf_interrupts;
pfvf_ops->send_msg = adf_gen4_pfvf_send;
pfvf_ops->recv_msg = adf_gen4_pfvf_recv;
}
EXPORT_SYMBOL_GPL(adf_gen4_init_pf_pfvf_ops);

17
drivers/crypto/qat/qat_common/adf_gen4_pfvf.h Normal file
View file

@ -0,0 +1,17 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2021 Intel Corporation */
#ifndef ADF_GEN4_PFVF_H
#define ADF_GEN4_PFVF_H
#include "adf_accel_devices.h"
#ifdef CONFIG_PCI_IOV
void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
#else
static inline void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
{
pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
}
#endif
#endif /* ADF_GEN4_PFVF_H */

11
drivers/crypto/qat/qat_common/adf_init.c
View file

@ -69,7 +69,8 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
return -EFAULT;
}
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
!accel_dev->is_vf) {
dev_err(&GET_DEV(accel_dev), "Device not configured\n");
return -EFAULT;
}
@ -117,10 +118,16 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
hw_data->enable_ints(accel_dev);
hw_data->enable_error_correction(accel_dev);
ret = hw_data->enable_pfvf_comms(accel_dev);
ret = hw_data->pfvf_ops.enable_comms(accel_dev);
if (ret)
return ret;
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
accel_dev->is_vf) {
if (qat_crypto_vf_dev_config(accel_dev))
return -EFAULT;
}
/*
* Subservice initialisation is divided into two stages: init and start.
* This is to facilitate any ordering dependencies between services

111
drivers/crypto/qat/qat_common/adf_isr.c
View file

@ -54,52 +54,83 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
return IRQ_HANDLED;
}
#ifdef CONFIG_PCI_IOV
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
unsigned long flags;
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
}
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
unsigned long flags;
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
}
static void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
u32 vf_mask)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
spin_lock(&accel_dev->pf.vf2pf_ints_lock);
GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
}
static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev)
{
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
bool irq_handled = false;
unsigned long vf_mask;
/* Get the interrupt sources triggered by VFs */
vf_mask = GET_PFVF_OPS(accel_dev)->get_vf2pf_sources(pmisc_addr);
if (vf_mask) {
struct adf_accel_vf_info *vf_info;
int i;
/* Disable VF2PF interrupts for VFs with pending ints */
adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
/*
* Handle VF2PF interrupt unless the VF is malicious and
* is attempting to flood the host OS with VF2PF interrupts.
*/
for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
vf_info = accel_dev->pf.vf_info + i;
if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
dev_info(&GET_DEV(accel_dev),
"Too many ints from VF%d\n",
vf_info->vf_nr);
continue;
}
adf_schedule_vf2pf_handler(vf_info);
irq_handled = true;
}
}
return irq_handled;
}
#endif /* CONFIG_PCI_IOV */
static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
{
struct adf_accel_dev *accel_dev = dev_ptr;
#ifdef CONFIG_PCI_IOV
/* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
if (accel_dev->pf.vf_info) {
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct adf_bar *pmisc =
&GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
void __iomem *pmisc_addr = pmisc->virt_addr;
unsigned long vf_mask;
/* Get the interrupt sources triggered by VFs */
vf_mask = hw_data->get_vf2pf_sources(pmisc_addr);
if (vf_mask) {
struct adf_accel_vf_info *vf_info;
bool irq_handled = false;
int i;
/* Disable VF2PF interrupts for VFs with pending ints */
adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
/*
* Handle VF2PF interrupt unless the VF is malicious and
* is attempting to flood the host OS with VF2PF interrupts.
*/
for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
vf_info = accel_dev->pf.vf_info + i;
if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
dev_info(&GET_DEV(accel_dev),
"Too many ints from VF%d\n",
vf_info->vf_nr + 1);
continue;
}
adf_schedule_vf2pf_handler(vf_info);
irq_handled = true;
}
if (irq_handled)
return IRQ_HANDLED;
}
}
if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev))
return IRQ_HANDLED;
#endif /* CONFIG_PCI_IOV */
dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",

416
drivers/crypto/qat/qat_common/adf_pf2vf_msg.c
View file

@ -1,416 +0,0 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2015 - 2020 Intel Corporation */
#include <linux/delay.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_pf2vf_msg.h"
#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10
#define ADF_PFVF_MSG_ACK_DELAY 2
#define ADF_PFVF_MSG_ACK_MAX_RETRY 100
#define ADF_PFVF_MSG_RETRY_DELAY 5
#define ADF_PFVF_MSG_MAX_RETRIES 3
#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \
ADF_PFVF_MSG_ACK_MAX_RETRY + \
ADF_PFVF_MSG_COLLISION_DETECT_DELAY)
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
void __iomem *pmisc_addr = pmisc->virt_addr;
unsigned long flags;
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
}
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
void __iomem *pmisc_addr = pmisc->virt_addr;
unsigned long flags;
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
}
void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
u32 vf_mask)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
void __iomem *pmisc_addr = pmisc->virt_addr;
spin_lock(&accel_dev->pf.vf2pf_ints_lock);
hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
}
static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
{
struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
void __iomem *pmisc_bar_addr =
pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
u32 val, pf2vf_offset, count = 0;
u32 local_in_use_mask, local_in_use_pattern;
u32 remote_in_use_mask, remote_in_use_pattern;
struct mutex *lock; /* lock preventing concurrent acces of CSR */
u32 int_bit;
int ret = 0;
if (accel_dev->is_vf) {
pf2vf_offset = hw_data->get_pf2vf_offset(0);
lock = &accel_dev->vf.vf2pf_lock;
local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK;
local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF;
remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK;
remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF;
int_bit = ADF_VF2PF_INT;
} else {
pf2vf_offset = hw_data->get_pf2vf_offset(vf_nr);
lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock;
local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK;
local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF;
remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK;
remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF;
int_bit = ADF_PF2VF_INT;
}
mutex_lock(lock);
/* Check if the PFVF CSR is in use by remote function */
val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
if ((val & remote_in_use_mask) == remote_in_use_pattern) {
dev_dbg(&GET_DEV(accel_dev),
"PFVF CSR in use by remote function\n");
ret = -EBUSY;
goto out;
}
msg &= ~local_in_use_mask;
msg |= local_in_use_pattern;
/* Attempt to get ownership of the PFVF CSR */
ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit);
/* Wait for confirmation from remote func it received the message */
do {
msleep(ADF_PFVF_MSG_ACK_DELAY);
val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
} while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY));
if (val != msg) {
dev_dbg(&GET_DEV(accel_dev),
"Collision - PFVF CSR overwritten by remote function\n");
ret = -EIO;
goto out;
}
if (val & int_bit) {
dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
val &= ~int_bit;
ret = -EIO;
}
/* Finished with the PFVF CSR; relinquish it and leave msg in CSR */
ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask);
out:
mutex_unlock(lock);
return ret;
}
/**
* adf_iov_putmsg() - send PFVF message
* @accel_dev: Pointer to acceleration device.
* @msg: Message to send
* @vf_nr: VF number to which the message will be sent if on PF, ignored
* otherwise
*
* Function sends a message through the PFVF channel
*
* Return: 0 on success, error code otherwise.
*/
static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
{
u32 count = 0;
int ret;
do {
ret = __adf_iov_putmsg(accel_dev, msg, vf_nr);
if (ret)
msleep(ADF_PFVF_MSG_RETRY_DELAY);
} while (ret && (count++ < ADF_PFVF_MSG_MAX_RETRIES));
return ret;
}
/**
* adf_send_pf2vf_msg() - send PF to VF message
* @accel_dev: Pointer to acceleration device
* @vf_nr: VF number to which the message will be sent
* @msg: Message to send
*
* This function allows the PF to send a message to a specific VF.
*
* Return: 0 on success, error code otherwise.
*/
static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg)
{
return adf_iov_putmsg(accel_dev, msg, vf_nr);
}
/**
* adf_send_vf2pf_msg() - send VF to PF message
* @accel_dev: Pointer to acceleration device
* @msg: Message to send
*
* This function allows the VF to send a message to the PF.
*
* Return: 0 on success, error code otherwise.
*/
int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg)
{
return adf_iov_putmsg(accel_dev, msg, 0);
}
/**
* adf_send_vf2pf_req() - send VF2PF request message
* @accel_dev: Pointer to acceleration device.
* @msg: Request message to send
*
* This function sends a message that requires a response from the VF to the PF
* and waits for a reply.
*
* Return: 0 on success, error code otherwise.
*/
static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg)
{
unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT);
int ret;
reinit_completion(&accel_dev->vf.iov_msg_completion);
/* Send request from VF to PF */
ret = adf_send_vf2pf_msg(accel_dev, msg);
if (ret) {
dev_err(&GET_DEV(accel_dev),
"Failed to send request msg to PF\n");
return ret;
}
/* Wait for response */
if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion,
timeout)) {
dev_err(&GET_DEV(accel_dev),
"PFVF request/response message timeout expired\n");
return -EIO;
}
return 0;
}
void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info)
{
struct adf_accel_dev *accel_dev = vf_info->accel_dev;
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
int bar_id = hw_data->get_misc_bar_id(hw_data);
struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id];
void __iomem *pmisc_addr = pmisc->virt_addr;
u32 msg, resp = 0, vf_nr = vf_info->vf_nr;
/* Read message from the VF */
msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr));
if (!(msg & ADF_VF2PF_INT)) {
dev_info(&GET_DEV(accel_dev),
"Spurious VF2PF interrupt, msg %X. Ignored\n", msg);
goto out;
}
/* To ACK, clear the VF2PFINT bit */
msg &= ~ADF_VF2PF_INT;
ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg);
if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM))
/* Ignore legacy non-system (non-kernel) VF2PF messages */
goto err;
switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) {
case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
{
u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
(ADF_PF2VF_MSGTYPE_VERSION_RESP <<
ADF_PF2VF_MSGTYPE_SHIFT) |
(ADF_PFVF_COMPAT_THIS_VERSION <<
ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
dev_dbg(&GET_DEV(accel_dev),
"Compatibility Version Request from VF%d vers=%u\n",
vf_nr + 1, vf_compat_ver);
if (vf_compat_ver < hw_data->min_iov_compat_ver) {
dev_err(&GET_DEV(accel_dev),
"VF (vers %d) incompatible with PF (vers %d)\n",
vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
resp |= ADF_PF2VF_VF_INCOMPATIBLE <<
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
} else if (vf_compat_ver > ADF_PFVF_COMPAT_THIS_VERSION) {
dev_err(&GET_DEV(accel_dev),
"VF (vers %d) compat with PF (vers %d) unkn.\n",
vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN <<
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
} else {
dev_dbg(&GET_DEV(accel_dev),
"VF (vers %d) compatible with PF (vers %d)\n",
vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
resp |= ADF_PF2VF_VF_COMPATIBLE <<
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
}
}
break;
case ADF_VF2PF_MSGTYPE_VERSION_REQ:
dev_dbg(&GET_DEV(accel_dev),
"Legacy VersionRequest received from VF%d 0x%x\n",
vf_nr + 1, msg);
resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
(ADF_PF2VF_MSGTYPE_VERSION_RESP <<
ADF_PF2VF_MSGTYPE_SHIFT) |
(ADF_PFVF_COMPAT_THIS_VERSION <<
ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
resp |= ADF_PF2VF_VF_COMPATIBLE <<
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
/* Set legacy major and minor version num */
resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT |
1 << ADF_PF2VF_MINORVERSION_SHIFT;
break;
case ADF_VF2PF_MSGTYPE_INIT:
{
dev_dbg(&GET_DEV(accel_dev),
"Init message received from VF%d 0x%x\n",
vf_nr + 1, msg);
vf_info->init = true;
}
break;
case ADF_VF2PF_MSGTYPE_SHUTDOWN:
{
dev_dbg(&GET_DEV(accel_dev),
"Shutdown message received from VF%d 0x%x\n",
vf_nr + 1, msg);
vf_info->init = false;
}
break;
default:
goto err;
}
if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp))
dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n");
out:
/* re-enable interrupt on PF from this VF */
adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr));
return;
err:
dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n",
vf_nr + 1, msg);
}
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev)
{
struct adf_accel_vf_info *vf;
u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM |
(ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT));
int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev));
for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) {
if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg))
dev_err(&GET_DEV(accel_dev),
"Failed to send restarting msg to VF%d\n", i);
}
}
static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
u32 msg = 0;
int ret;
msg = ADF_VF2PF_MSGORIGIN_SYSTEM;
msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT;
msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255);
ret = adf_send_vf2pf_req(accel_dev, msg);
if (ret) {
dev_err(&GET_DEV(accel_dev),
"Failed to send Compatibility Version Request.\n");
return ret;
}
/* Response from PF received, check compatibility */
switch (accel_dev->vf.compatible) {
case ADF_PF2VF_VF_COMPATIBLE:
break;
case ADF_PF2VF_VF_COMPAT_UNKNOWN:
/* VF is newer than PF and decides whether it is compatible */
if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) {
accel_dev->vf.compatible = ADF_PF2VF_VF_COMPATIBLE;
break;
}
fallthrough;
case ADF_PF2VF_VF_INCOMPATIBLE:
dev_err(&GET_DEV(accel_dev),
"PF (vers %d) and VF (vers %d) are not compatible\n",
accel_dev->vf.pf_version,
ADF_PFVF_COMPAT_THIS_VERSION);
return -EINVAL;
default:
dev_err(&GET_DEV(accel_dev),
"Invalid response from PF; assume not compatible\n");
return -EINVAL;
}
return ret;
}
/**
* adf_enable_vf2pf_comms() - Function enables communication from vf to pf
*
* @accel_dev: Pointer to acceleration device virtual function.
*
* Return: 0 on success, error code otherwise.
*/
int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev)
{
adf_enable_pf2vf_interrupts(accel_dev);
return adf_vf2pf_request_version(accel_dev);
}
EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms);
/**
* adf_enable_pf2vf_comms() - Function enables communication from pf to vf
*
* @accel_dev: Pointer to acceleration device virtual function.
*
* This function carries out the necessary steps to setup and start the PFVF
* communication channel, if any.
*
* Return: 0 on success, error code otherwise.
*/
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
{
spin_lock_init(&accel_dev->pf.vf2pf_ints_lock);
return 0;
}
EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms);

93
drivers/crypto/qat/qat_common/adf_pf2vf_msg.h
View file

@ -1,93 +0,0 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2015 - 2020 Intel Corporation */
#ifndef ADF_PF2VF_MSG_H
#define ADF_PF2VF_MSG_H
/*
* PF<->VF Messaging
* The PF has an array of 32-bit PF2VF registers, one for each VF. The
* PF can access all these registers; each VF can access only the one
* register associated with that particular VF.
*
* The register functionally is split into two parts:
* The bottom half is for PF->VF messages. In particular when the first
* bit of this register (bit 0) gets set an interrupt will be triggered
* in the respective VF.
* The top half is for VF->PF messages. In particular when the first bit
* of this half of register (bit 16) gets set an interrupt will be triggered
* in the PF.
*
* The remaining bits within this register are available to encode messages.
* and implement a collision control mechanism to prevent concurrent use of
* the PF2VF register by both the PF and VF.
*
* 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
* _______________________________________________
* | | | | | | | | | | | | | | | | |
* +-----------------------------------------------+
* \___________________________/ \_________/ ^ ^
* ^ ^ | |
* | | | VF2PF Int
* | | Message Origin
* | Message Type
* Message-specific Data/Reserved
*
* 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
* _______________________________________________
* | | | | | | | | | | | | | | | | |
* +-----------------------------------------------+
* \___________________________/ \_________/ ^ ^
* ^ ^ | |
* | | | PF2VF Int
* | | Message Origin
* | Message Type
* Message-specific Data/Reserved
*
* Message Origin (Should always be 1)
* A legacy out-of-tree QAT driver allowed for a set of messages not supported
* by this driver; these had a Msg Origin of 0 and are ignored by this driver.
*
* When a PF or VF attempts to send a message in the lower or upper 16 bits,
* respectively, the other 16 bits are written to first with a defined
* IN_USE_BY pattern as part of a collision control scheme (see adf_iov_putmsg).
*/
#define ADF_PFVF_COMPAT_THIS_VERSION 0x1 /* PF<->VF compat */
/* PF->VF messages */
#define ADF_PF2VF_INT BIT(0)
#define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1)
#define ADF_PF2VF_MSGTYPE_MASK 0x0000003C
#define ADF_PF2VF_MSGTYPE_SHIFT 2
#define ADF_PF2VF_MSGTYPE_RESTARTING 0x01
#define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02
#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000
#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000
/* PF->VF Version Response */
#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0
#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6
#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000
#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14
#define ADF_PF2VF_MINORVERSION_SHIFT 6
#define ADF_PF2VF_MAJORVERSION_SHIFT 10
#define ADF_PF2VF_VF_COMPATIBLE 1
#define ADF_PF2VF_VF_INCOMPATIBLE 2
#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3
/* VF->PF messages */
#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2
#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE
#define ADF_VF2PF_INT BIT(16)
#define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17)
#define ADF_VF2PF_MSGTYPE_MASK 0x003C0000
#define ADF_VF2PF_MSGTYPE_SHIFT 18
#define ADF_VF2PF_MSGTYPE_INIT 0x3
#define ADF_VF2PF_MSGTYPE_SHUTDOWN 0x4
#define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5
#define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6
/* VF->PF Compatible Version Request */
#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22
#endif /* ADF_IOV_MSG_H */

259
drivers/crypto/qat/qat_common/adf_pfvf_msg.h Normal file
View file

@ -0,0 +1,259 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2015 - 2021 Intel Corporation */
#ifndef ADF_PFVF_MSG_H
#define ADF_PFVF_MSG_H
#include <linux/bits.h>
/*
* PF<->VF Gen2 Messaging format
*
* The PF has an array of 32-bit PF2VF registers, one for each VF. The
* PF can access all these registers; each VF can access only the one
* register associated with that particular VF.
*
* The register functionally is split into two parts:
* The bottom half is for PF->VF messages. In particular when the first
* bit of this register (bit 0) gets set an interrupt will be triggered
* in the respective VF.
* The top half is for VF->PF messages. In particular when the first bit
* of this half of register (bit 16) gets set an interrupt will be triggered
* in the PF.
*
* The remaining bits within this register are available to encode messages.
* and implement a collision control mechanism to prevent concurrent use of
* the PF2VF register by both the PF and VF.
*
* 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
* _______________________________________________
* | | | | | | | | | | | | | | | | |
* +-----------------------------------------------+
* \___________________________/ \_________/ ^ ^
* ^ ^ | |
* | | | VF2PF Int
* | | Message Origin
* | Message Type
* Message-specific Data/Reserved
*
* 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
* _______________________________________________
* | | | | | | | | | | | | | | | | |
* +-----------------------------------------------+
* \___________________________/ \_________/ ^ ^
* ^ ^ | |
* | | | PF2VF Int
* | | Message Origin
* | Message Type
* Message-specific Data/Reserved
*
* Message Origin (Should always be 1)
* A legacy out-of-tree QAT driver allowed for a set of messages not supported
* by this driver; these had a Msg Origin of 0 and are ignored by this driver.
*
* When a PF or VF attempts to send a message in the lower or upper 16 bits,
* respectively, the other 16 bits are written to first with a defined
* IN_USE_BY pattern as part of a collision control scheme (see function
* adf_gen2_pfvf_send() in adf_pf2vf_msg.c).
*
*
* PF<->VF Gen4 Messaging format
*
* Similarly to the gen2 messaging format, 32-bit long registers are used for
* communication between PF and VFs. However, each VF and PF share a pair of
* 32-bits register to avoid collisions: one for PV to VF messages and one
* for VF to PF messages.
*
* Both the Interrupt bit and the Message Origin bit retain the same position
* and meaning, although non-system messages are now deprecated and not
* expected.
*
* 31 30 9 8 7 6 5 4 3 2 1 0
* _______________________________________________
* | | | . . . | | | | | | | | | | |
* +-----------------------------------------------+
* \_____________________/ \_______________/ ^ ^
* ^ ^ | |
* | | | PF/VF Int
* | | Message Origin
* | Message Type
* Message-specific Data/Reserved
*
* For both formats, the message reception is acknowledged by lowering the
* interrupt bit on the register where the message was sent.
*/
/* PFVF message common bits */
#define ADF_PFVF_INT BIT(0)
#define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1)
/* Different generations have different CSR layouts, use this struct
* to abstract these differences away
*/
struct pfvf_message {
u8 type;
u32 data;
};
/* PF->VF messages */
enum pf2vf_msgtype {
ADF_PF2VF_MSGTYPE_RESTARTING = 0x01,
ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02,
ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03,
/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */
ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10,
};
/* VF->PF messages */
enum vf2pf_msgtype {
ADF_VF2PF_MSGTYPE_INIT = 0x03,
ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04,
ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05,
ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06,
ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07,
ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08,
ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09,
/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */
ADF_VF2PF_MSGTYPE_RP_RESET = 0x10,
};
/* VF/PF compatibility version. */
enum pfvf_compatibility_version {
/* Support for extended capabilities */
ADF_PFVF_COMPAT_CAPABILITIES = 0x02,
/* In-use pattern cleared by receiver */
ADF_PFVF_COMPAT_FAST_ACK = 0x03,
/* Ring to service mapping support for non-standard mappings */
ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04,
/* Reference to the latest version */
ADF_PFVF_COMPAT_THIS_VERSION = 0x04,
};
/* PF->VF Version Response */
#define ADF_PF2VF_VERSION_RESP_VERS_MASK GENMASK(7, 0)
#define ADF_PF2VF_VERSION_RESP_RESULT_MASK GENMASK(9, 8)
enum pf2vf_compat_response {
ADF_PF2VF_VF_COMPATIBLE = 0x01,
ADF_PF2VF_VF_INCOMPATIBLE = 0x02,
ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03,
};
enum ring_reset_result {
RPRESET_SUCCESS = 0x00,
RPRESET_NOT_SUPPORTED = 0x01,
RPRESET_INVAL_BANK = 0x02,
RPRESET_TIMEOUT = 0x03,
};
#define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0)
#define ADF_VF2PF_RNG_RESET_RSVD_MASK GENMASK(25, 2)
/* PF->VF Block Responses */
#define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0)
#define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2)
enum pf2vf_blkmsg_resp_type {
ADF_PF2VF_BLKMSG_RESP_TYPE_DATA = 0x00,
ADF_PF2VF_BLKMSG_RESP_TYPE_CRC = 0x01,
ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR = 0x02,
};
/* PF->VF Block Error Code */
enum pf2vf_blkmsg_error {
ADF_PF2VF_INVALID_BLOCK_TYPE = 0x00,
ADF_PF2VF_INVALID_BYTE_NUM_REQ = 0x01,
ADF_PF2VF_PAYLOAD_TRUNCATED = 0x02,
ADF_PF2VF_UNSPECIFIED_ERROR = 0x03,
};
/* VF->PF Block Requests */
#define ADF_VF2PF_LARGE_BLOCK_TYPE_MASK GENMASK(1, 0)
#define ADF_VF2PF_LARGE_BLOCK_BYTE_MASK GENMASK(8, 2)
#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK GENMASK(2, 0)
#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK GENMASK(8, 3)
#define ADF_VF2PF_SMALL_BLOCK_TYPE_MASK GENMASK(3, 0)
#define ADF_VF2PF_SMALL_BLOCK_BYTE_MASK GENMASK(8, 4)
#define ADF_VF2PF_BLOCK_CRC_REQ_MASK BIT(9)
/* PF->VF Block Request Types
* 0..15 - 32 byte message
* 16..23 - 64 byte message
* 24..27 - 128 byte message
*/
enum vf2pf_blkmsg_req_type {
ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02,
ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP = 0x03,
};
#define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \
(FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK))
#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX \
(FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK) + \
ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1)
#define ADF_VF2PF_LARGE_BLOCK_TYPE_MAX \
(FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK) + \
ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX)
#define ADF_VF2PF_SMALL_BLOCK_BYTE_MAX \
FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK)
#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX \
FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK)
#define ADF_VF2PF_LARGE_BLOCK_BYTE_MAX \
FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK)
struct pfvf_blkmsg_header {
u8 version;
u8 payload_size;
} __packed;
#define ADF_PFVF_BLKMSG_HEADER_SIZE (sizeof(struct pfvf_blkmsg_header))
#define ADF_PFVF_BLKMSG_PAYLOAD_SIZE(blkmsg) (sizeof(blkmsg) - \
ADF_PFVF_BLKMSG_HEADER_SIZE)
#define ADF_PFVF_BLKMSG_MSG_SIZE(blkmsg) (ADF_PFVF_BLKMSG_HEADER_SIZE + \
(blkmsg)->hdr.payload_size)
#define ADF_PFVF_BLKMSG_MSG_MAX_SIZE 128
/* PF->VF Block message header bytes */
#define ADF_PFVF_BLKMSG_VER_BYTE 0
#define ADF_PFVF_BLKMSG_LEN_BYTE 1
/* PF/VF Capabilities message values */
enum blkmsg_capabilities_versions {
ADF_PFVF_CAPABILITIES_V1_VERSION = 0x01,
ADF_PFVF_CAPABILITIES_V2_VERSION = 0x02,
ADF_PFVF_CAPABILITIES_V3_VERSION = 0x03,
};
struct capabilities_v1 {
struct pfvf_blkmsg_header hdr;
u32 ext_dc_caps;
} __packed;
struct capabilities_v2 {
struct pfvf_blkmsg_header hdr;
u32 ext_dc_caps;
u32 capabilities;
} __packed;
struct capabilities_v3 {
struct pfvf_blkmsg_header hdr;
u32 ext_dc_caps;
u32 capabilities;
u32 frequency;
} __packed;
/* PF/VF Ring to service mapping values */
enum blkmsg_ring_to_svc_versions {
ADF_PFVF_RING_TO_SVC_VERSION = 0x01,
};
struct ring_to_svc_map_v1 {
struct pfvf_blkmsg_header hdr;
u16 map;
} __packed;
#endif /* ADF_PFVF_MSG_H */

52
drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c Normal file
View file

@ -0,0 +1,52 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <linux/pci.h>
#include "adf_accel_devices.h"
#include "adf_pfvf_msg.h"
#include "adf_pfvf_pf_msg.h"
#include "adf_pfvf_pf_proto.h"
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev)
{
struct adf_accel_vf_info *vf;
struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTING };
int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev));
for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) {
if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg))
dev_err(&GET_DEV(accel_dev),
"Failed to send restarting msg to VF%d\n", i);
}
}
int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev,
u8 *buffer, u8 compat)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct capabilities_v2 caps_msg;
caps_msg.ext_dc_caps = hw_data->extended_dc_capabilities;
caps_msg.capabilities = hw_data->accel_capabilities_mask;
caps_msg.hdr.version = ADF_PFVF_CAPABILITIES_V2_VERSION;
caps_msg.hdr.payload_size =
ADF_PFVF_BLKMSG_PAYLOAD_SIZE(struct capabilities_v2);
memcpy(buffer, &caps_msg, sizeof(caps_msg));
return 0;
}
int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev,
u8 *buffer, u8 compat)
{
struct ring_to_svc_map_v1 rts_map_msg;
rts_map_msg.map = accel_dev->hw_device->ring_to_svc_map;
rts_map_msg.hdr.version = ADF_PFVF_RING_TO_SVC_VERSION;
rts_map_msg.hdr.payload_size = ADF_PFVF_BLKMSG_PAYLOAD_SIZE(rts_map_msg);
memcpy(buffer, &rts_map_msg, sizeof(rts_map_msg));
return 0;
}

18
drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h Normal file
View file

@ -0,0 +1,18 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2021 Intel Corporation */
#ifndef ADF_PFVF_PF_MSG_H
#define ADF_PFVF_PF_MSG_H
#include "adf_accel_devices.h"
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev);
typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev,
u8 *buffer, u8 compat);
int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev,
u8 *buffer, u8 comapt);
int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev,
u8 *buffer, u8 comapt);
#endif /* ADF_PFVF_PF_MSG_H */

346
drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c Normal file
View file

@ -0,0 +1,346 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2015 - 2021 Intel Corporation */
#include <linux/bitfield.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include "adf_accel_devices.h"
#include "adf_common_drv.h"
#include "adf_pfvf_msg.h"
#include "adf_pfvf_pf_msg.h"
#include "adf_pfvf_pf_proto.h"
#include "adf_pfvf_utils.h"
typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte);
static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = {
NULL, /* no message type defined for value 0 */
NULL, /* no message type defined for value 1 */
adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */
adf_pf_ring_to_svc_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP */
};
/**
* adf_send_pf2vf_msg() - send PF to VF message
* @accel_dev: Pointer to acceleration device
* @vf_nr: VF number to which the message will be sent
* @msg: Message to send
*
* This function allows the PF to send a message to a specific VF.
*
* Return: 0 on success, error code otherwise.
*/
int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg)
{
struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr);
return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset,
&accel_dev->pf.vf_info[vf_nr].pf2vf_lock);
}
/**
* adf_recv_vf2pf_msg() - receive a VF to PF message
* @accel_dev: Pointer to acceleration device
* @vf_nr: Number of the VF from where the message will be received
*
* This function allows the PF to receive a message from a specific VF.
*
* Return: a valid message on success, zero otherwise.
*/
static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr)
{
struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr);
return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver);
}
static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type)
{
if (type >= ARRAY_SIZE(pf2vf_blkmsg_providers))
return NULL;
return pf2vf_blkmsg_providers[type];
}
/* Byte pf2vf_blkmsg_data_getter_fn callback */
static u8 adf_pf2vf_blkmsg_get_byte(u8 const *blkmsg, u8 index)
{
return blkmsg[index];
}
/* CRC pf2vf_blkmsg_data_getter_fn callback */
static u8 adf_pf2vf_blkmsg_get_crc(u8 const *blkmsg, u8 count)
{
/* count is 0-based, turn it into a length */
return adf_pfvf_calc_blkmsg_crc(blkmsg, count + 1);
}
static int adf_pf2vf_blkmsg_get_data(struct adf_accel_vf_info *vf_info,
u8 type, u8 byte, u8 max_size, u8 *data,
pf2vf_blkmsg_data_getter_fn data_getter)
{
u8 blkmsg[ADF_PFVF_BLKMSG_MSG_MAX_SIZE] = { 0 };
struct adf_accel_dev *accel_dev = vf_info->accel_dev;
adf_pf2vf_blkmsg_provider provider;
u8 msg_size;
provider = get_blkmsg_response_provider(type);
if (unlikely(!provider)) {
pr_err("QAT: No registered provider for message %d\n", type);
*data = ADF_PF2VF_INVALID_BLOCK_TYPE;
return -EINVAL;
}
if (unlikely((*provider)(accel_dev, blkmsg, vf_info->vf_compat_ver))) {
pr_err("QAT: unknown error from provider for message %d\n", type);
*data = ADF_PF2VF_UNSPECIFIED_ERROR;
return -EINVAL;
}
msg_size = ADF_PFVF_BLKMSG_HEADER_SIZE + blkmsg[ADF_PFVF_BLKMSG_LEN_BYTE];
if (unlikely(msg_size >= max_size)) {
pr_err("QAT: Invalid size %d provided for message type %d\n",
msg_size, type);
*data = ADF_PF2VF_PAYLOAD_TRUNCATED;
return -EINVAL;
}
if (unlikely(byte >= msg_size)) {
pr_err("QAT: Out-of-bound byte number %d (msg size %d)\n",
byte, msg_size);
*data = ADF_PF2VF_INVALID_BYTE_NUM_REQ;
return -EINVAL;
}
*data = data_getter(blkmsg, byte);
return 0;
}
static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info,
struct pfvf_message req)
{
u8 resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR;
struct pfvf_message resp = { 0 };
u8 resp_data = 0;
u8 blk_type;
u8 blk_byte;
u8 byte_max;
switch (req.type) {
case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ:
blk_type = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, req.data)
+ ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX + 1;
blk_byte = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, req.data);
byte_max = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX;
break;
case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ:
blk_type = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, req.data)
+ ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1;
blk_byte = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, req.data);
byte_max = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX;
break;
case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ:
blk_type = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, req.data);
blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data);
byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX;
break;
}
/* Is this a request for CRC or data? */
if (FIELD_GET(ADF_VF2PF_BLOCK_CRC_REQ_MASK, req.data)) {
dev_dbg(&GET_DEV(vf_info->accel_dev),
"BlockMsg of type %d for CRC over %d bytes received from VF%d\n",
blk_type, blk_byte, vf_info->vf_nr);
if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte,
byte_max, &resp_data,
adf_pf2vf_blkmsg_get_crc))
resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_CRC;
} else {
dev_dbg(&GET_DEV(vf_info->accel_dev),
"BlockMsg of type %d for data byte %d received from VF%d\n",
blk_type, blk_byte, vf_info->vf_nr);
if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte,
byte_max, &resp_data,
adf_pf2vf_blkmsg_get_byte))
resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_DATA;
}
resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP;
resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp_type) |
FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp_data);
return resp;
}
static struct pfvf_message handle_rp_reset_req(struct adf_accel_dev *accel_dev, u8 vf_nr,
struct pfvf_message req)
{
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
struct pfvf_message resp = {
.type = ADF_PF2VF_MSGTYPE_RP_RESET_RESP,
.data = RPRESET_SUCCESS
};
u32 bank_number;
u32 rsvd_field;
bank_number = FIELD_GET(ADF_VF2PF_RNG_RESET_RP_MASK, req.data);
rsvd_field = FIELD_GET(ADF_VF2PF_RNG_RESET_RSVD_MASK, req.data);
dev_dbg(&GET_DEV(accel_dev),
"Ring Pair Reset Message received from VF%d for bank 0x%x\n",
vf_nr, bank_number);
if (!hw_data->ring_pair_reset || rsvd_field) {
dev_dbg(&GET_DEV(accel_dev),
"Ring Pair Reset for VF%d is not supported\n", vf_nr);
resp.data = RPRESET_NOT_SUPPORTED;
goto out;
}
if (bank_number >= hw_data->num_banks_per_vf) {
dev_err(&GET_DEV(accel_dev),
"Invalid bank number (0x%x) from VF%d for Ring Reset\n",
bank_number, vf_nr);
resp.data = RPRESET_INVAL_BANK;
goto out;
}
/* Convert the VF provided value to PF bank number */
bank_number = vf_nr * hw_data->num_banks_per_vf + bank_number;
if (hw_data->ring_pair_reset(accel_dev, bank_number)) {
dev_dbg(&GET_DEV(accel_dev),
"Ring pair reset for VF%d failure\n", vf_nr);
resp.data = RPRESET_TIMEOUT;
goto out;
}
dev_dbg(&GET_DEV(accel_dev),
"Ring pair reset for VF%d successfully\n", vf_nr);
out:
return resp;
}
static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr,
struct pfvf_message msg, struct pfvf_message *resp)
{
struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
switch (msg.type) {
case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
{
u8 vf_compat_ver = msg.data;
u8 compat;
dev_dbg(&GET_DEV(accel_dev),
"VersionRequest received from VF%d (vers %d) to PF (vers %d)\n",
vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION)
compat = ADF_PF2VF_VF_COMPATIBLE;
else
compat = ADF_PF2VF_VF_COMPAT_UNKNOWN;
vf_info->vf_compat_ver = vf_compat_ver;
resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP;
resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK,
ADF_PFVF_COMPAT_THIS_VERSION) |
FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat);
}
break;
case ADF_VF2PF_MSGTYPE_VERSION_REQ:
{
u8 compat;
dev_dbg(&GET_DEV(accel_dev),
"Legacy VersionRequest received from VF%d to PF (vers 1.1)\n",
vf_nr);
/* legacy driver, VF compat_ver is 0 */
vf_info->vf_compat_ver = 0;
/* PF always newer than legacy VF */
compat = ADF_PF2VF_VF_COMPATIBLE;
/* Set legacy major and minor version to the latest, 1.1 */
resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP;
resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, 0x11) |
FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat);
}
break;
case ADF_VF2PF_MSGTYPE_INIT:
{
dev_dbg(&GET_DEV(accel_dev),
"Init message received from VF%d\n", vf_nr);
vf_info->init = true;
}
break;
case ADF_VF2PF_MSGTYPE_SHUTDOWN:
{
dev_dbg(&GET_DEV(accel_dev),
"Shutdown message received from VF%d\n", vf_nr);
vf_info->init = false;
}
break;
case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ:
case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ:
case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ:
*resp = handle_blkmsg_req(vf_info, msg);
break;
case ADF_VF2PF_MSGTYPE_RP_RESET:
*resp = handle_rp_reset_req(accel_dev, vf_nr, msg);
break;
default:
dev_dbg(&GET_DEV(accel_dev),
"Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n",
vf_nr, msg.type, msg.data);
return -ENOMSG;
}
return 0;
}
bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr)
{
struct pfvf_message req;
struct pfvf_message resp = {0};
req = adf_recv_vf2pf_msg(accel_dev, vf_nr);
if (!req.type) /* Legacy or no message */
return true;
if (adf_handle_vf2pf_msg(accel_dev, vf_nr, req, &resp))
return false;
if (resp.type && adf_send_pf2vf_msg(accel_dev, vf_nr, resp))
dev_err(&GET_DEV(accel_dev),
"Failed to send response to VF%d\n", vf_nr);
return true;
}
/**
* adf_enable_pf2vf_comms() - Function enables communication from pf to vf
*
* @accel_dev: Pointer to acceleration device virtual function.
*
* This function carries out the necessary steps to setup and start the PFVF
* communication channel, if any.
*
* Return: 0 on success, error code otherwise.
*/
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
{
adf_pfvf_crc_init();
spin_lock_init(&accel_dev->pf.vf2pf_ints_lock);
return 0;
}
EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms);

13
drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h Normal file
View file

@ -0,0 +1,13 @@
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
/* Copyright(c) 2021 Intel Corporation */
#ifndef ADF_PFVF_PF_PROTO_H
#define ADF_PFVF_PF_PROTO_H
#include <linux/types.h>
#include "adf_accel_devices.h"
int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg);
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev);
#endif /* ADF_PFVF_PF_PROTO_H */

65
drivers/crypto/qat/qat_common/adf_pfvf_utils.c Normal file
View file

@ -0,0 +1,65 @@
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
/* Copyright(c) 2021 Intel Corporation */
#include <linux/crc8.h>
#include <linux/pci.h>
#include <linux/types.h>
#include "adf_accel_devices.h"
#include "adf_pfvf_msg.h"
#include "adf_pfvf_utils.h"
/* CRC Calculation */
DECLARE_CRC8_TABLE(pfvf_crc8_table);
#define ADF_PFVF_CRC8_POLYNOMIAL 0x97
void adf_pfvf_crc_init(void)
{
crc8_populate_msb(pfvf_crc8_table, ADF_PFVF_CRC8_POLYNOMIAL);
}
u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len)
{
return crc8(pfvf_crc8_table, buf, buf_len, CRC8_INIT_VALUE);
}
static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg,
u32 value, const struct pfvf_field_format *fmt)
{
if (unlikely((value & fmt->mask) != value)) {
dev_err(&GET_DEV(accel_dev),
"PFVF message value 0x%X out of range, %u max allowed\n",
value, fmt->mask);
return false;
}
*csr_msg |= value << fmt->offset;
return true;
}
u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev,
struct pfvf_message msg,
const struct pfvf_csr_format *fmt)
{
u32 csr_msg = 0;
if (!set_value_on_csr_msg(accel_dev, &csr_msg, msg.type, &fmt->type) ||
!set_value_on_csr_msg(accel_dev, &csr_msg, msg.data, &fmt->data))
return 0;
return csr_msg | ADF_PFVF_MSGORIGIN_SYSTEM;
}
struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 csr_msg,
const struct pfvf_csr_format *fmt)
{
struct pfvf_message msg = { 0 };
msg.type = (csr_msg >> fmt->type.offset) & fmt->type.mask;
msg.data = (csr_msg >> fmt->data.offset) & fmt->data.mask;
if (unlikely(!msg.type))
dev_err(&GET_DEV(accel_dev),
"Invalid PFVF msg with no type received\n");
return msg;
}

Some files were not shown because too many files have changed in this diff Show more