mirror of
https://github.com/Fishwaldo/linux-bl808.git
synced 2025-03-18 04:54:52 +00:00
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto updates from Herbert Xu: "Algorithms: - Drop alignment requirement for data in aesni - Use synchronous seeding from the /dev/random in DRBG - Reseed nopr DRBGs every 5 minutes from /dev/random - Add KDF algorithms currently used by security/DH - Fix lack of entropy on some AMD CPUs with jitter RNG Drivers: - Add support for the D1 variant in sun8i-ce - Add SEV_INIT_EX support in ccp - PFVF support for GEN4 host driver in qat - Compression support for GEN4 devices in qat - Add cn10k random number generator support" * 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (145 commits) crypto: af_alg - rewrite NULL pointer check lib/mpi: Add the return value check of kcalloc() crypto: qat - fix definition of ring reset results crypto: hisilicon - cleanup warning in qm_get_qos_value() crypto: kdf - select SHA-256 required for self-test crypto: x86/aesni - don't require alignment of data crypto: ccp - remove unneeded semicolon crypto: stm32/crc32 - Fix kernel BUG triggered in probe() crypto: s390/sha512 - Use macros instead of direct IV numbers crypto: sparc/sha - remove duplicate hash init function crypto: powerpc/sha - remove duplicate hash init function crypto: mips/sha - remove duplicate hash init function crypto: sha256 - remove duplicate generic hash init function crypto: jitter - add oversampling of noise source MAINTAINERS: update SEC2 driver maintainers list crypto: ux500 - Use platform_get_irq() to get the interrupt crypto: hisilicon/qm - disable qm clock-gating crypto: omap-aes - Fix broken pm_runtime_and_get() usage MAINTAINERS: update caam crypto driver maintainers list crypto: octeontx2 - prevent underflow in get_cores_bmap() ...
This commit is contained in:
commit
5c947d0dba
142 changed files with 5492 additions and 2652 deletions
|
@ -85,6 +85,12 @@ guests, such as launching, running, snapshotting, migrating and decommissioning.
|
|||
The KVM_SEV_INIT command is used by the hypervisor to initialize the SEV platform
|
||||
context. In a typical workflow, this command should be the first command issued.
|
||||
|
||||
The firmware can be initialized either by using its own non-volatile storage or
|
||||
the OS can manage the NV storage for the firmware using the module parameter
|
||||
``init_ex_path``. The file specified by ``init_ex_path`` must exist. To create
|
||||
a new NV storage file allocate the file with 32KB bytes of 0xFF as required by
|
||||
the SEV spec.
|
||||
|
||||
Returns: 0 on success, -negative on error
|
||||
|
||||
2. KVM_SEV_LAUNCH_START
|
||||
|
|
|
@ -7557,6 +7557,7 @@ F: include/video/
|
|||
FREESCALE CAAM (Cryptographic Acceleration and Assurance Module) DRIVER
|
||||
M: Horia Geantă <horia.geanta@nxp.com>
|
||||
M: Pankaj Gupta <pankaj.gupta@nxp.com>
|
||||
M: Gaurav Jain <gaurav.jain@nxp.com>
|
||||
L: linux-crypto@vger.kernel.org
|
||||
S: Maintained
|
||||
F: Documentation/devicetree/bindings/crypto/fsl-sec4.txt
|
||||
|
@ -8676,6 +8677,7 @@ F: drivers/scsi/hisi_sas/
|
|||
|
||||
HISILICON SECURITY ENGINE V2 DRIVER (SEC2)
|
||||
M: Zaibo Xu <xuzaibo@huawei.com>
|
||||
M: Kai Ye <yekai13@huawei.com>
|
||||
L: linux-crypto@vger.kernel.org
|
||||
S: Maintained
|
||||
F: Documentation/ABI/testing/debugfs-hisi-sec
|
||||
|
@ -9721,7 +9723,6 @@ F: Documentation/devicetree/bindings/crypto/intel,keembay-ocs-ecc.yaml
|
|||
F: drivers/crypto/keembay/Kconfig
|
||||
F: drivers/crypto/keembay/Makefile
|
||||
F: drivers/crypto/keembay/keembay-ocs-ecc.c
|
||||
F: drivers/crypto/keembay/ocs-ecc-curve-defs.h
|
||||
|
||||
INTEL KEEM BAY OCS HCU CRYPTO DRIVER
|
||||
M: Daniele Alessandrelli <daniele.alessandrelli@intel.com>
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include <linux/mm.h>
|
||||
#include <crypto/sha1.h>
|
||||
#include <crypto/sha1_base.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/module.h>
|
||||
|
@ -71,20 +72,6 @@ static void octeon_sha1_transform(const void *_block)
|
|||
octeon_sha1_start(block[7]);
|
||||
}
|
||||
|
||||
static int octeon_sha1_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha1_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA1_H0;
|
||||
sctx->state[1] = SHA1_H1;
|
||||
sctx->state[2] = SHA1_H2;
|
||||
sctx->state[3] = SHA1_H3;
|
||||
sctx->state[4] = SHA1_H4;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __octeon_sha1_update(struct sha1_state *sctx, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -200,7 +187,7 @@ static int octeon_sha1_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg octeon_sha1_alg = {
|
||||
.digestsize = SHA1_DIGEST_SIZE,
|
||||
.init = octeon_sha1_init,
|
||||
.init = sha1_base_init,
|
||||
.update = octeon_sha1_update,
|
||||
.final = octeon_sha1_final,
|
||||
.export = octeon_sha1_export,
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
|
||||
#include <linux/mm.h>
|
||||
#include <crypto/sha2.h>
|
||||
#include <crypto/sha256_base.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/module.h>
|
||||
|
@ -63,40 +64,6 @@ static void octeon_sha256_transform(const void *_block)
|
|||
octeon_sha256_start(block[7]);
|
||||
}
|
||||
|
||||
static int octeon_sha224_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha256_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA224_H0;
|
||||
sctx->state[1] = SHA224_H1;
|
||||
sctx->state[2] = SHA224_H2;
|
||||
sctx->state[3] = SHA224_H3;
|
||||
sctx->state[4] = SHA224_H4;
|
||||
sctx->state[5] = SHA224_H5;
|
||||
sctx->state[6] = SHA224_H6;
|
||||
sctx->state[7] = SHA224_H7;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int octeon_sha256_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha256_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA256_H0;
|
||||
sctx->state[1] = SHA256_H1;
|
||||
sctx->state[2] = SHA256_H2;
|
||||
sctx->state[3] = SHA256_H3;
|
||||
sctx->state[4] = SHA256_H4;
|
||||
sctx->state[5] = SHA256_H5;
|
||||
sctx->state[6] = SHA256_H6;
|
||||
sctx->state[7] = SHA256_H7;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __octeon_sha256_update(struct sha256_state *sctx, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -224,7 +191,7 @@ static int octeon_sha256_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg octeon_sha256_algs[2] = { {
|
||||
.digestsize = SHA256_DIGEST_SIZE,
|
||||
.init = octeon_sha256_init,
|
||||
.init = sha256_base_init,
|
||||
.update = octeon_sha256_update,
|
||||
.final = octeon_sha256_final,
|
||||
.export = octeon_sha256_export,
|
||||
|
@ -240,7 +207,7 @@ static struct shash_alg octeon_sha256_algs[2] = { {
|
|||
}
|
||||
}, {
|
||||
.digestsize = SHA224_DIGEST_SIZE,
|
||||
.init = octeon_sha224_init,
|
||||
.init = sha224_base_init,
|
||||
.update = octeon_sha256_update,
|
||||
.final = octeon_sha224_final,
|
||||
.descsize = sizeof(struct sha256_state),
|
||||
|
|
|
@ -15,6 +15,7 @@
|
|||
|
||||
#include <linux/mm.h>
|
||||
#include <crypto/sha2.h>
|
||||
#include <crypto/sha512_base.h>
|
||||
#include <linux/init.h>
|
||||
#include <linux/types.h>
|
||||
#include <linux/module.h>
|
||||
|
@ -74,40 +75,6 @@ static void octeon_sha512_transform(const void *_block)
|
|||
octeon_sha512_start(block[15]);
|
||||
}
|
||||
|
||||
static int octeon_sha512_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA512_H0;
|
||||
sctx->state[1] = SHA512_H1;
|
||||
sctx->state[2] = SHA512_H2;
|
||||
sctx->state[3] = SHA512_H3;
|
||||
sctx->state[4] = SHA512_H4;
|
||||
sctx->state[5] = SHA512_H5;
|
||||
sctx->state[6] = SHA512_H6;
|
||||
sctx->state[7] = SHA512_H7;
|
||||
sctx->count[0] = sctx->count[1] = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int octeon_sha384_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA384_H0;
|
||||
sctx->state[1] = SHA384_H1;
|
||||
sctx->state[2] = SHA384_H2;
|
||||
sctx->state[3] = SHA384_H3;
|
||||
sctx->state[4] = SHA384_H4;
|
||||
sctx->state[5] = SHA384_H5;
|
||||
sctx->state[6] = SHA384_H6;
|
||||
sctx->state[7] = SHA384_H7;
|
||||
sctx->count[0] = sctx->count[1] = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __octeon_sha512_update(struct sha512_state *sctx, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -223,7 +190,7 @@ static int octeon_sha384_final(struct shash_desc *desc, u8 *hash)
|
|||
|
||||
static struct shash_alg octeon_sha512_algs[2] = { {
|
||||
.digestsize = SHA512_DIGEST_SIZE,
|
||||
.init = octeon_sha512_init,
|
||||
.init = sha512_base_init,
|
||||
.update = octeon_sha512_update,
|
||||
.final = octeon_sha512_final,
|
||||
.descsize = sizeof(struct sha512_state),
|
||||
|
@ -236,7 +203,7 @@ static struct shash_alg octeon_sha512_algs[2] = { {
|
|||
}
|
||||
}, {
|
||||
.digestsize = SHA384_DIGEST_SIZE,
|
||||
.init = octeon_sha384_init,
|
||||
.init = sha384_base_init,
|
||||
.update = octeon_sha512_update,
|
||||
.final = octeon_sha384_final,
|
||||
.descsize = sizeof(struct sha512_state),
|
||||
|
|
|
@ -13,6 +13,7 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha1.h>
|
||||
#include <crypto/sha1_base.h>
|
||||
#include <asm/byteorder.h>
|
||||
#include <asm/switch_to.h>
|
||||
#include <linux/hardirq.h>
|
||||
|
@ -55,20 +56,6 @@ static inline void ppc_sha1_clear_context(struct sha1_state *sctx)
|
|||
do { *ptr++ = 0; } while (--count);
|
||||
}
|
||||
|
||||
static int ppc_spe_sha1_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha1_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA1_H0;
|
||||
sctx->state[1] = SHA1_H1;
|
||||
sctx->state[2] = SHA1_H2;
|
||||
sctx->state[3] = SHA1_H3;
|
||||
sctx->state[4] = SHA1_H4;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ppc_spe_sha1_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -168,7 +155,7 @@ static int ppc_spe_sha1_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg alg = {
|
||||
.digestsize = SHA1_DIGEST_SIZE,
|
||||
.init = ppc_spe_sha1_init,
|
||||
.init = sha1_base_init,
|
||||
.update = ppc_spe_sha1_update,
|
||||
.final = ppc_spe_sha1_final,
|
||||
.export = ppc_spe_sha1_export,
|
||||
|
|
|
@ -18,21 +18,11 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha1.h>
|
||||
#include <crypto/sha1_base.h>
|
||||
#include <asm/byteorder.h>
|
||||
|
||||
void powerpc_sha_transform(u32 *state, const u8 *src);
|
||||
|
||||
static int powerpc_sha1_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha1_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
*sctx = (struct sha1_state){
|
||||
.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
|
||||
};
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int powerpc_sha1_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -114,7 +104,7 @@ static int powerpc_sha1_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg alg = {
|
||||
.digestsize = SHA1_DIGEST_SIZE,
|
||||
.init = powerpc_sha1_init,
|
||||
.init = sha1_base_init,
|
||||
.update = powerpc_sha1_update,
|
||||
.final = powerpc_sha1_final,
|
||||
.export = powerpc_sha1_export,
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha2.h>
|
||||
#include <crypto/sha256_base.h>
|
||||
#include <asm/byteorder.h>
|
||||
#include <asm/switch_to.h>
|
||||
#include <linux/hardirq.h>
|
||||
|
@ -56,40 +57,6 @@ static inline void ppc_sha256_clear_context(struct sha256_state *sctx)
|
|||
do { *ptr++ = 0; } while (--count);
|
||||
}
|
||||
|
||||
static int ppc_spe_sha256_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha256_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA256_H0;
|
||||
sctx->state[1] = SHA256_H1;
|
||||
sctx->state[2] = SHA256_H2;
|
||||
sctx->state[3] = SHA256_H3;
|
||||
sctx->state[4] = SHA256_H4;
|
||||
sctx->state[5] = SHA256_H5;
|
||||
sctx->state[6] = SHA256_H6;
|
||||
sctx->state[7] = SHA256_H7;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ppc_spe_sha224_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha256_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
sctx->state[0] = SHA224_H0;
|
||||
sctx->state[1] = SHA224_H1;
|
||||
sctx->state[2] = SHA224_H2;
|
||||
sctx->state[3] = SHA224_H3;
|
||||
sctx->state[4] = SHA224_H4;
|
||||
sctx->state[5] = SHA224_H5;
|
||||
sctx->state[6] = SHA224_H6;
|
||||
sctx->state[7] = SHA224_H7;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int ppc_spe_sha256_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -214,7 +181,7 @@ static int ppc_spe_sha256_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg algs[2] = { {
|
||||
.digestsize = SHA256_DIGEST_SIZE,
|
||||
.init = ppc_spe_sha256_init,
|
||||
.init = sha256_base_init,
|
||||
.update = ppc_spe_sha256_update,
|
||||
.final = ppc_spe_sha256_final,
|
||||
.export = ppc_spe_sha256_export,
|
||||
|
@ -230,7 +197,7 @@ static struct shash_alg algs[2] = { {
|
|||
}
|
||||
}, {
|
||||
.digestsize = SHA224_DIGEST_SIZE,
|
||||
.init = ppc_spe_sha224_init,
|
||||
.init = sha224_base_init,
|
||||
.update = ppc_spe_sha256_update,
|
||||
.final = ppc_spe_sha224_final,
|
||||
.export = ppc_spe_sha256_export,
|
||||
|
|
|
@ -22,14 +22,14 @@ static int sha512_init(struct shash_desc *desc)
|
|||
{
|
||||
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
|
||||
|
||||
*(__u64 *)&ctx->state[0] = 0x6a09e667f3bcc908ULL;
|
||||
*(__u64 *)&ctx->state[2] = 0xbb67ae8584caa73bULL;
|
||||
*(__u64 *)&ctx->state[4] = 0x3c6ef372fe94f82bULL;
|
||||
*(__u64 *)&ctx->state[6] = 0xa54ff53a5f1d36f1ULL;
|
||||
*(__u64 *)&ctx->state[8] = 0x510e527fade682d1ULL;
|
||||
*(__u64 *)&ctx->state[10] = 0x9b05688c2b3e6c1fULL;
|
||||
*(__u64 *)&ctx->state[12] = 0x1f83d9abfb41bd6bULL;
|
||||
*(__u64 *)&ctx->state[14] = 0x5be0cd19137e2179ULL;
|
||||
*(__u64 *)&ctx->state[0] = SHA512_H0;
|
||||
*(__u64 *)&ctx->state[2] = SHA512_H1;
|
||||
*(__u64 *)&ctx->state[4] = SHA512_H2;
|
||||
*(__u64 *)&ctx->state[6] = SHA512_H3;
|
||||
*(__u64 *)&ctx->state[8] = SHA512_H4;
|
||||
*(__u64 *)&ctx->state[10] = SHA512_H5;
|
||||
*(__u64 *)&ctx->state[12] = SHA512_H6;
|
||||
*(__u64 *)&ctx->state[14] = SHA512_H7;
|
||||
ctx->count = 0;
|
||||
ctx->func = CPACF_KIMD_SHA_512;
|
||||
|
||||
|
@ -87,14 +87,14 @@ static int sha384_init(struct shash_desc *desc)
|
|||
{
|
||||
struct s390_sha_ctx *ctx = shash_desc_ctx(desc);
|
||||
|
||||
*(__u64 *)&ctx->state[0] = 0xcbbb9d5dc1059ed8ULL;
|
||||
*(__u64 *)&ctx->state[2] = 0x629a292a367cd507ULL;
|
||||
*(__u64 *)&ctx->state[4] = 0x9159015a3070dd17ULL;
|
||||
*(__u64 *)&ctx->state[6] = 0x152fecd8f70e5939ULL;
|
||||
*(__u64 *)&ctx->state[8] = 0x67332667ffc00b31ULL;
|
||||
*(__u64 *)&ctx->state[10] = 0x8eb44a8768581511ULL;
|
||||
*(__u64 *)&ctx->state[12] = 0xdb0c2e0d64f98fa7ULL;
|
||||
*(__u64 *)&ctx->state[14] = 0x47b5481dbefa4fa4ULL;
|
||||
*(__u64 *)&ctx->state[0] = SHA384_H0;
|
||||
*(__u64 *)&ctx->state[2] = SHA384_H1;
|
||||
*(__u64 *)&ctx->state[4] = SHA384_H2;
|
||||
*(__u64 *)&ctx->state[6] = SHA384_H3;
|
||||
*(__u64 *)&ctx->state[8] = SHA384_H4;
|
||||
*(__u64 *)&ctx->state[10] = SHA384_H5;
|
||||
*(__u64 *)&ctx->state[12] = SHA384_H6;
|
||||
*(__u64 *)&ctx->state[14] = SHA384_H7;
|
||||
ctx->count = 0;
|
||||
ctx->func = CPACF_KIMD_SHA_512;
|
||||
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha1.h>
|
||||
#include <crypto/sha1_base.h>
|
||||
|
||||
#include <asm/pstate.h>
|
||||
#include <asm/elf.h>
|
||||
|
@ -26,17 +27,6 @@
|
|||
asmlinkage void sha1_sparc64_transform(u32 *digest, const char *data,
|
||||
unsigned int rounds);
|
||||
|
||||
static int sha1_sparc64_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha1_state *sctx = shash_desc_ctx(desc);
|
||||
|
||||
*sctx = (struct sha1_state){
|
||||
.state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 },
|
||||
};
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __sha1_sparc64_update(struct sha1_state *sctx, const u8 *data,
|
||||
unsigned int len, unsigned int partial)
|
||||
{
|
||||
|
@ -128,7 +118,7 @@ static int sha1_sparc64_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg alg = {
|
||||
.digestsize = SHA1_DIGEST_SIZE,
|
||||
.init = sha1_sparc64_init,
|
||||
.init = sha1_base_init,
|
||||
.update = sha1_sparc64_update,
|
||||
.final = sha1_sparc64_final,
|
||||
.export = sha1_sparc64_export,
|
||||
|
|
|
@ -17,6 +17,7 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha2.h>
|
||||
#include <crypto/sha256_base.h>
|
||||
|
||||
#include <asm/pstate.h>
|
||||
#include <asm/elf.h>
|
||||
|
@ -26,38 +27,6 @@
|
|||
asmlinkage void sha256_sparc64_transform(u32 *digest, const char *data,
|
||||
unsigned int rounds);
|
||||
|
||||
static int sha224_sparc64_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha256_state *sctx = shash_desc_ctx(desc);
|
||||
sctx->state[0] = SHA224_H0;
|
||||
sctx->state[1] = SHA224_H1;
|
||||
sctx->state[2] = SHA224_H2;
|
||||
sctx->state[3] = SHA224_H3;
|
||||
sctx->state[4] = SHA224_H4;
|
||||
sctx->state[5] = SHA224_H5;
|
||||
sctx->state[6] = SHA224_H6;
|
||||
sctx->state[7] = SHA224_H7;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha256_sparc64_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha256_state *sctx = shash_desc_ctx(desc);
|
||||
sctx->state[0] = SHA256_H0;
|
||||
sctx->state[1] = SHA256_H1;
|
||||
sctx->state[2] = SHA256_H2;
|
||||
sctx->state[3] = SHA256_H3;
|
||||
sctx->state[4] = SHA256_H4;
|
||||
sctx->state[5] = SHA256_H5;
|
||||
sctx->state[6] = SHA256_H6;
|
||||
sctx->state[7] = SHA256_H7;
|
||||
sctx->count = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __sha256_sparc64_update(struct sha256_state *sctx, const u8 *data,
|
||||
unsigned int len, unsigned int partial)
|
||||
{
|
||||
|
@ -158,7 +127,7 @@ static int sha256_sparc64_import(struct shash_desc *desc, const void *in)
|
|||
|
||||
static struct shash_alg sha256_alg = {
|
||||
.digestsize = SHA256_DIGEST_SIZE,
|
||||
.init = sha256_sparc64_init,
|
||||
.init = sha256_base_init,
|
||||
.update = sha256_sparc64_update,
|
||||
.final = sha256_sparc64_final,
|
||||
.export = sha256_sparc64_export,
|
||||
|
@ -176,7 +145,7 @@ static struct shash_alg sha256_alg = {
|
|||
|
||||
static struct shash_alg sha224_alg = {
|
||||
.digestsize = SHA224_DIGEST_SIZE,
|
||||
.init = sha224_sparc64_init,
|
||||
.init = sha224_base_init,
|
||||
.update = sha256_sparc64_update,
|
||||
.final = sha224_sparc64_final,
|
||||
.descsize = sizeof(struct sha256_state),
|
||||
|
|
|
@ -16,6 +16,7 @@
|
|||
#include <linux/mm.h>
|
||||
#include <linux/types.h>
|
||||
#include <crypto/sha2.h>
|
||||
#include <crypto/sha512_base.h>
|
||||
|
||||
#include <asm/pstate.h>
|
||||
#include <asm/elf.h>
|
||||
|
@ -25,38 +26,6 @@
|
|||
asmlinkage void sha512_sparc64_transform(u64 *digest, const char *data,
|
||||
unsigned int rounds);
|
||||
|
||||
static int sha512_sparc64_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
sctx->state[0] = SHA512_H0;
|
||||
sctx->state[1] = SHA512_H1;
|
||||
sctx->state[2] = SHA512_H2;
|
||||
sctx->state[3] = SHA512_H3;
|
||||
sctx->state[4] = SHA512_H4;
|
||||
sctx->state[5] = SHA512_H5;
|
||||
sctx->state[6] = SHA512_H6;
|
||||
sctx->state[7] = SHA512_H7;
|
||||
sctx->count[0] = sctx->count[1] = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int sha384_sparc64_init(struct shash_desc *desc)
|
||||
{
|
||||
struct sha512_state *sctx = shash_desc_ctx(desc);
|
||||
sctx->state[0] = SHA384_H0;
|
||||
sctx->state[1] = SHA384_H1;
|
||||
sctx->state[2] = SHA384_H2;
|
||||
sctx->state[3] = SHA384_H3;
|
||||
sctx->state[4] = SHA384_H4;
|
||||
sctx->state[5] = SHA384_H5;
|
||||
sctx->state[6] = SHA384_H6;
|
||||
sctx->state[7] = SHA384_H7;
|
||||
sctx->count[0] = sctx->count[1] = 0;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void __sha512_sparc64_update(struct sha512_state *sctx, const u8 *data,
|
||||
unsigned int len, unsigned int partial)
|
||||
{
|
||||
|
@ -146,7 +115,7 @@ static int sha384_sparc64_final(struct shash_desc *desc, u8 *hash)
|
|||
|
||||
static struct shash_alg sha512 = {
|
||||
.digestsize = SHA512_DIGEST_SIZE,
|
||||
.init = sha512_sparc64_init,
|
||||
.init = sha512_base_init,
|
||||
.update = sha512_sparc64_update,
|
||||
.final = sha512_sparc64_final,
|
||||
.descsize = sizeof(struct sha512_state),
|
||||
|
@ -161,7 +130,7 @@ static struct shash_alg sha512 = {
|
|||
|
||||
static struct shash_alg sha384 = {
|
||||
.digestsize = SHA384_DIGEST_SIZE,
|
||||
.init = sha384_sparc64_init,
|
||||
.init = sha384_base_init,
|
||||
.update = sha512_sparc64_update,
|
||||
.final = sha384_sparc64_final,
|
||||
.descsize = sizeof(struct sha512_state),
|
||||
|
|
|
@ -1107,7 +1107,7 @@ static struct aead_alg aesni_aeads[] = { {
|
|||
.cra_flags = CRYPTO_ALG_INTERNAL,
|
||||
.cra_blocksize = 1,
|
||||
.cra_ctxsize = sizeof(struct aesni_rfc4106_gcm_ctx),
|
||||
.cra_alignmask = AESNI_ALIGN - 1,
|
||||
.cra_alignmask = 0,
|
||||
.cra_module = THIS_MODULE,
|
||||
},
|
||||
}, {
|
||||
|
@ -1124,7 +1124,7 @@ static struct aead_alg aesni_aeads[] = { {
|
|||
.cra_flags = CRYPTO_ALG_INTERNAL,
|
||||
.cra_blocksize = 1,
|
||||
.cra_ctxsize = sizeof(struct generic_gcmaes_ctx),
|
||||
.cra_alignmask = AESNI_ALIGN - 1,
|
||||
.cra_alignmask = 0,
|
||||
.cra_module = THIS_MODULE,
|
||||
},
|
||||
} };
|
||||
|
|
|
@ -64,10 +64,9 @@ static inline u64 add_scalar(u64 *out, const u64 *f1, u64 f2)
|
|||
|
||||
/* Return the carry bit in a register */
|
||||
" adcx %%r11, %1;"
|
||||
: "+&r" (f2), "=&r" (carry_r)
|
||||
: "r" (out), "r" (f1)
|
||||
: "%r8", "%r9", "%r10", "%r11", "memory", "cc"
|
||||
);
|
||||
: "+&r"(f2), "=&r"(carry_r)
|
||||
: "r"(out), "r"(f1)
|
||||
: "%r8", "%r9", "%r10", "%r11", "memory", "cc");
|
||||
|
||||
return carry_r;
|
||||
}
|
||||
|
@ -108,10 +107,9 @@ static inline void fadd(u64 *out, const u64 *f1, const u64 *f2)
|
|||
" cmovc %0, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 0(%1);"
|
||||
: "+&r" (f2)
|
||||
: "r" (out), "r" (f1)
|
||||
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"
|
||||
);
|
||||
: "+&r"(f2)
|
||||
: "r"(out), "r"(f1)
|
||||
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes the field subtraction of two field elements */
|
||||
|
@ -151,10 +149,9 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
|
|||
" movq %%r9, 8(%0);"
|
||||
" movq %%r10, 16(%0);"
|
||||
" movq %%r11, 24(%0);"
|
||||
:
|
||||
: "r" (out), "r" (f1), "r" (f2)
|
||||
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc"
|
||||
);
|
||||
:
|
||||
: "r"(out), "r"(f1), "r"(f2)
|
||||
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes a field multiplication: out <- f1 * f2
|
||||
|
@ -162,239 +159,400 @@ static inline void fsub(u64 *out, const u64 *f1, const u64 *f2)
|
|||
static inline void fmul(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
|
||||
{
|
||||
asm volatile(
|
||||
|
||||
/* Compute the raw multiplication: tmp <- src1 * src2 */
|
||||
|
||||
/* Compute src1[0] * src2 */
|
||||
" movq 0(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" movq 0(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" movq %%r8, 0(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" movq %%r10, 8(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
|
||||
/* Compute src1[1] * src2 */
|
||||
" movq 8(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
|
||||
" movq 8(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 8(%2), %%r8;"
|
||||
" movq %%r8, 8(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 16(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
|
||||
/* Compute src1[2] * src2 */
|
||||
" movq 16(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
|
||||
" movq 16(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 16(%2), %%r8;"
|
||||
" movq %%r8, 16(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 24(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
|
||||
/* Compute src1[3] * src2 */
|
||||
" movq 24(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);"
|
||||
" movq 24(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 24(%2), %%r8;"
|
||||
" movq %%r8, 24(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 32(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" movq %%rbx, 40(%2);"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" movq %%r14, 48(%2);"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
" movq %%rax, 56(%2);"
|
||||
|
||||
/* Line up pointers */
|
||||
" mov %0, %1;"
|
||||
" mov %2, %0;"
|
||||
" mov %3, %2;"
|
||||
|
||||
/* Wrap the result back into the field */
|
||||
|
||||
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
|
||||
" mov $38, %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r13;"
|
||||
" xor %k3, %k3;"
|
||||
" adoxq 0(%1), %%r8;"
|
||||
" mulxq 40(%1), %%r9, %%rbx;"
|
||||
" mulxq 32(%0), %%r8, %%r13;"
|
||||
" xor %k1, %k1;"
|
||||
" adoxq 0(%0), %%r8;"
|
||||
" mulxq 40(%0), %%r9, %%rbx;"
|
||||
" adcx %%r13, %%r9;"
|
||||
" adoxq 8(%1), %%r9;"
|
||||
" mulxq 48(%1), %%r10, %%r13;"
|
||||
" adoxq 8(%0), %%r9;"
|
||||
" mulxq 48(%0), %%r10, %%r13;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" adoxq 16(%1), %%r10;"
|
||||
" mulxq 56(%1), %%r11, %%rax;"
|
||||
" adoxq 16(%0), %%r10;"
|
||||
" mulxq 56(%0), %%r11, %%rax;"
|
||||
" adcx %%r13, %%r11;"
|
||||
" adoxq 24(%1), %%r11;"
|
||||
" adcx %3, %%rax;"
|
||||
" adox %3, %%rax;"
|
||||
" adoxq 24(%0), %%r11;"
|
||||
" adcx %1, %%rax;"
|
||||
" adox %1, %%rax;"
|
||||
" imul %%rdx, %%rax;"
|
||||
|
||||
/* Step 2: Fold the carry back into dst */
|
||||
" add %%rax, %%r8;"
|
||||
" adcx %3, %%r9;"
|
||||
" movq %%r9, 8(%0);"
|
||||
" adcx %3, %%r10;"
|
||||
" movq %%r10, 16(%0);"
|
||||
" adcx %3, %%r11;"
|
||||
" movq %%r11, 24(%0);"
|
||||
" adcx %1, %%r9;"
|
||||
" movq %%r9, 8(%2);"
|
||||
" adcx %1, %%r10;"
|
||||
" movq %%r10, 16(%2);"
|
||||
" adcx %1, %%r11;"
|
||||
" movq %%r11, 24(%2);"
|
||||
|
||||
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
|
||||
" mov $0, %%rax;"
|
||||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 0(%0);"
|
||||
: "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2)
|
||||
:
|
||||
: "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc"
|
||||
);
|
||||
" movq %%r8, 0(%2);"
|
||||
: "+&r"(f1), "+&r"(f2), "+&r"(tmp)
|
||||
: "r"(out)
|
||||
: "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
|
||||
"%r14", "memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes two field multiplications:
|
||||
* out[0] <- f1[0] * f2[0]
|
||||
* out[1] <- f1[1] * f2[1]
|
||||
* Uses the 16-element buffer tmp for intermediate results. */
|
||||
* out[0] <- f1[0] * f2[0]
|
||||
* out[1] <- f1[1] * f2[1]
|
||||
* Uses the 16-element buffer tmp for intermediate results: */
|
||||
static inline void fmul2(u64 *out, const u64 *f1, const u64 *f2, u64 *tmp)
|
||||
{
|
||||
asm volatile(
|
||||
|
||||
/* Compute the raw multiplication tmp[0] <- f1[0] * f2[0] */
|
||||
|
||||
/* Compute src1[0] * src2 */
|
||||
" movq 0(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 0(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 8(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" movq 0(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" movq %%r8, 0(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" movq %%r10, 8(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
|
||||
/* Compute src1[1] * src2 */
|
||||
" movq 8(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 8(%0), %%r8;" " movq %%r8, 8(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 16(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
|
||||
" movq 8(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 8(%2), %%r8;"
|
||||
" movq %%r8, 8(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 16(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
|
||||
/* Compute src1[2] * src2 */
|
||||
" movq 16(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 16(%0), %%r8;" " movq %%r8, 16(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 24(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
|
||||
" movq 16(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 16(%2), %%r8;"
|
||||
" movq %%r8, 16(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 24(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
|
||||
/* Compute src1[3] * src2 */
|
||||
" movq 24(%1), %%rdx;"
|
||||
" mulxq 0(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 24(%0), %%r8;" " movq %%r8, 24(%0);"
|
||||
" mulxq 8(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 32(%0);"
|
||||
" mulxq 16(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 40(%0);" " mov $0, %%r8;"
|
||||
" mulxq 24(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 48(%0);" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 56(%0);"
|
||||
" movq 24(%0), %%rdx;"
|
||||
" mulxq 0(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 24(%2), %%r8;"
|
||||
" movq %%r8, 24(%2);"
|
||||
" mulxq 8(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 32(%2);"
|
||||
" mulxq 16(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" movq %%rbx, 40(%2);"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 24(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" movq %%r14, 48(%2);"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
" movq %%rax, 56(%2);"
|
||||
|
||||
/* Compute the raw multiplication tmp[1] <- f1[1] * f2[1] */
|
||||
|
||||
/* Compute src1[0] * src2 */
|
||||
" movq 32(%1), %%rdx;"
|
||||
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " movq %%r8, 64(%0);"
|
||||
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " movq %%r10, 72(%0);"
|
||||
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;"
|
||||
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" movq 32(%0), %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" movq %%r8, 64(%2);"
|
||||
" mulxq 40(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" movq %%r10, 72(%2);"
|
||||
" mulxq 48(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" mulxq 56(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
|
||||
/* Compute src1[1] * src2 */
|
||||
" movq 40(%1), %%rdx;"
|
||||
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 72(%0), %%r8;" " movq %%r8, 72(%0);"
|
||||
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 80(%0);"
|
||||
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
|
||||
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
|
||||
" movq 40(%0), %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 72(%2), %%r8;"
|
||||
" movq %%r8, 72(%2);"
|
||||
" mulxq 40(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 80(%2);"
|
||||
" mulxq 48(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 56(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
|
||||
/* Compute src1[2] * src2 */
|
||||
" movq 48(%1), %%rdx;"
|
||||
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 80(%0), %%r8;" " movq %%r8, 80(%0);"
|
||||
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 88(%0);"
|
||||
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " mov $0, %%r8;"
|
||||
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;"
|
||||
" movq 48(%0), %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 80(%2), %%r8;"
|
||||
" movq %%r8, 80(%2);"
|
||||
" mulxq 40(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 88(%2);"
|
||||
" mulxq 48(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 56(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
|
||||
/* Compute src1[3] * src2 */
|
||||
" movq 56(%1), %%rdx;"
|
||||
" mulxq 32(%3), %%r8, %%r9;" " xor %%r10d, %%r10d;" " adcxq 88(%0), %%r8;" " movq %%r8, 88(%0);"
|
||||
" mulxq 40(%3), %%r10, %%r11;" " adox %%r9, %%r10;" " adcx %%rbx, %%r10;" " movq %%r10, 96(%0);"
|
||||
" mulxq 48(%3), %%rbx, %%r13;" " adox %%r11, %%rbx;" " adcx %%r14, %%rbx;" " movq %%rbx, 104(%0);" " mov $0, %%r8;"
|
||||
" mulxq 56(%3), %%r14, %%rdx;" " adox %%r13, %%r14;" " adcx %%rax, %%r14;" " movq %%r14, 112(%0);" " mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;" " adcx %%r8, %%rax;" " movq %%rax, 120(%0);"
|
||||
" movq 56(%0), %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r9;"
|
||||
" xor %%r10d, %%r10d;"
|
||||
" adcxq 88(%2), %%r8;"
|
||||
" movq %%r8, 88(%2);"
|
||||
" mulxq 40(%1), %%r10, %%r11;"
|
||||
" adox %%r9, %%r10;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" movq %%r10, 96(%2);"
|
||||
" mulxq 48(%1), %%rbx, %%r13;"
|
||||
" adox %%r11, %%rbx;"
|
||||
" adcx %%r14, %%rbx;"
|
||||
" movq %%rbx, 104(%2);"
|
||||
" mov $0, %%r8;"
|
||||
" mulxq 56(%1), %%r14, %%rdx;"
|
||||
" adox %%r13, %%r14;"
|
||||
" adcx %%rax, %%r14;"
|
||||
" movq %%r14, 112(%2);"
|
||||
" mov $0, %%rax;"
|
||||
" adox %%rdx, %%rax;"
|
||||
" adcx %%r8, %%rax;"
|
||||
" movq %%rax, 120(%2);"
|
||||
|
||||
/* Line up pointers */
|
||||
" mov %0, %1;"
|
||||
" mov %2, %0;"
|
||||
" mov %3, %2;"
|
||||
|
||||
/* Wrap the results back into the field */
|
||||
|
||||
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
|
||||
" mov $38, %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r13;"
|
||||
" xor %k3, %k3;"
|
||||
" adoxq 0(%1), %%r8;"
|
||||
" mulxq 40(%1), %%r9, %%rbx;"
|
||||
" mulxq 32(%0), %%r8, %%r13;"
|
||||
" xor %k1, %k1;"
|
||||
" adoxq 0(%0), %%r8;"
|
||||
" mulxq 40(%0), %%r9, %%rbx;"
|
||||
" adcx %%r13, %%r9;"
|
||||
" adoxq 8(%1), %%r9;"
|
||||
" mulxq 48(%1), %%r10, %%r13;"
|
||||
" adoxq 8(%0), %%r9;"
|
||||
" mulxq 48(%0), %%r10, %%r13;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" adoxq 16(%1), %%r10;"
|
||||
" mulxq 56(%1), %%r11, %%rax;"
|
||||
" adoxq 16(%0), %%r10;"
|
||||
" mulxq 56(%0), %%r11, %%rax;"
|
||||
" adcx %%r13, %%r11;"
|
||||
" adoxq 24(%1), %%r11;"
|
||||
" adcx %3, %%rax;"
|
||||
" adox %3, %%rax;"
|
||||
" adoxq 24(%0), %%r11;"
|
||||
" adcx %1, %%rax;"
|
||||
" adox %1, %%rax;"
|
||||
" imul %%rdx, %%rax;"
|
||||
|
||||
/* Step 2: Fold the carry back into dst */
|
||||
" add %%rax, %%r8;"
|
||||
" adcx %3, %%r9;"
|
||||
" movq %%r9, 8(%0);"
|
||||
" adcx %3, %%r10;"
|
||||
" movq %%r10, 16(%0);"
|
||||
" adcx %3, %%r11;"
|
||||
" movq %%r11, 24(%0);"
|
||||
" adcx %1, %%r9;"
|
||||
" movq %%r9, 8(%2);"
|
||||
" adcx %1, %%r10;"
|
||||
" movq %%r10, 16(%2);"
|
||||
" adcx %1, %%r11;"
|
||||
" movq %%r11, 24(%2);"
|
||||
|
||||
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
|
||||
" mov $0, %%rax;"
|
||||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 0(%0);"
|
||||
" movq %%r8, 0(%2);"
|
||||
|
||||
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
|
||||
" mov $38, %%rdx;"
|
||||
" mulxq 96(%1), %%r8, %%r13;"
|
||||
" xor %k3, %k3;"
|
||||
" adoxq 64(%1), %%r8;"
|
||||
" mulxq 104(%1), %%r9, %%rbx;"
|
||||
" mulxq 96(%0), %%r8, %%r13;"
|
||||
" xor %k1, %k1;"
|
||||
" adoxq 64(%0), %%r8;"
|
||||
" mulxq 104(%0), %%r9, %%rbx;"
|
||||
" adcx %%r13, %%r9;"
|
||||
" adoxq 72(%1), %%r9;"
|
||||
" mulxq 112(%1), %%r10, %%r13;"
|
||||
" adoxq 72(%0), %%r9;"
|
||||
" mulxq 112(%0), %%r10, %%r13;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" adoxq 80(%1), %%r10;"
|
||||
" mulxq 120(%1), %%r11, %%rax;"
|
||||
" adoxq 80(%0), %%r10;"
|
||||
" mulxq 120(%0), %%r11, %%rax;"
|
||||
" adcx %%r13, %%r11;"
|
||||
" adoxq 88(%1), %%r11;"
|
||||
" adcx %3, %%rax;"
|
||||
" adox %3, %%rax;"
|
||||
" adoxq 88(%0), %%r11;"
|
||||
" adcx %1, %%rax;"
|
||||
" adox %1, %%rax;"
|
||||
" imul %%rdx, %%rax;"
|
||||
|
||||
/* Step 2: Fold the carry back into dst */
|
||||
" add %%rax, %%r8;"
|
||||
" adcx %3, %%r9;"
|
||||
" movq %%r9, 40(%0);"
|
||||
" adcx %3, %%r10;"
|
||||
" movq %%r10, 48(%0);"
|
||||
" adcx %3, %%r11;"
|
||||
" movq %%r11, 56(%0);"
|
||||
" adcx %1, %%r9;"
|
||||
" movq %%r9, 40(%2);"
|
||||
" adcx %1, %%r10;"
|
||||
" movq %%r10, 48(%2);"
|
||||
" adcx %1, %%r11;"
|
||||
" movq %%r11, 56(%2);"
|
||||
|
||||
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
|
||||
" mov $0, %%rax;"
|
||||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 32(%0);"
|
||||
: "+&r" (tmp), "+&r" (f1), "+&r" (out), "+&r" (f2)
|
||||
:
|
||||
: "%rax", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "memory", "cc"
|
||||
);
|
||||
" movq %%r8, 32(%2);"
|
||||
: "+&r"(f1), "+&r"(f2), "+&r"(tmp)
|
||||
: "r"(out)
|
||||
: "%rax", "%rbx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r13",
|
||||
"%r14", "memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes the field multiplication of four-element f1 with value in f2 */
|
||||
/* Computes the field multiplication of four-element f1 with value in f2
|
||||
* Requires f2 to be smaller than 2^17 */
|
||||
static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
|
||||
{
|
||||
register u64 f2_r asm("rdx") = f2;
|
||||
|
||||
asm volatile(
|
||||
/* Compute the raw multiplication of f1*f2 */
|
||||
" mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
|
||||
" mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
|
||||
" mulxq 0(%2), %%r8, %%rcx;" /* f1[0]*f2 */
|
||||
" mulxq 8(%2), %%r9, %%rbx;" /* f1[1]*f2 */
|
||||
" add %%rcx, %%r9;"
|
||||
" mov $0, %%rcx;"
|
||||
" mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
|
||||
" mulxq 16(%2), %%r10, %%r13;" /* f1[2]*f2 */
|
||||
" adcx %%rbx, %%r10;"
|
||||
" mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
|
||||
" mulxq 24(%2), %%r11, %%rax;" /* f1[3]*f2 */
|
||||
" adcx %%r13, %%r11;"
|
||||
" adcx %%rcx, %%rax;"
|
||||
|
||||
|
@ -418,17 +576,17 @@ static inline void fmul_scalar(u64 *out, const u64 *f1, u64 f2)
|
|||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 0(%1);"
|
||||
: "+&r" (f2_r)
|
||||
: "r" (out), "r" (f1)
|
||||
: "%rax", "%rcx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "memory", "cc"
|
||||
);
|
||||
: "+&r"(f2_r)
|
||||
: "r"(out), "r"(f1)
|
||||
: "%rax", "%rbx", "%rcx", "%r8", "%r9", "%r10", "%r11", "%r13",
|
||||
"memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes p1 <- bit ? p2 : p1 in constant time */
|
||||
static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
|
||||
{
|
||||
asm volatile(
|
||||
/* Invert the polarity of bit to match cmov expectations */
|
||||
/* Transfer bit into CF flag */
|
||||
" add $18446744073709551615, %0;"
|
||||
|
||||
/* cswap p1[0], p2[0] */
|
||||
|
@ -502,10 +660,9 @@ static inline void cswap2(u64 bit, const u64 *p1, const u64 *p2)
|
|||
" cmovc %%r10, %%r9;"
|
||||
" movq %%r8, 56(%1);"
|
||||
" movq %%r9, 56(%2);"
|
||||
: "+&r" (bit)
|
||||
: "r" (p1), "r" (p2)
|
||||
: "%r8", "%r9", "%r10", "memory", "cc"
|
||||
);
|
||||
: "+&r"(bit)
|
||||
: "r"(p1), "r"(p2)
|
||||
: "%r8", "%r9", "%r10", "memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes the square of a field element: out <- f * f
|
||||
|
@ -516,15 +673,22 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
|
|||
/* Compute the raw multiplication: tmp <- f * f */
|
||||
|
||||
/* Step 1: Compute all partial products */
|
||||
" movq 0(%1), %%rdx;" /* f[0] */
|
||||
" mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
|
||||
" mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
|
||||
" mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
|
||||
" movq 24(%1), %%rdx;" /* f[3] */
|
||||
" mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
|
||||
" mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
|
||||
" movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
|
||||
" mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
|
||||
" movq 0(%0), %%rdx;" /* f[0] */
|
||||
" mulxq 8(%0), %%r8, %%r14;"
|
||||
" xor %%r15d, %%r15d;" /* f[1]*f[0] */
|
||||
" mulxq 16(%0), %%r9, %%r10;"
|
||||
" adcx %%r14, %%r9;" /* f[2]*f[0] */
|
||||
" mulxq 24(%0), %%rax, %%rcx;"
|
||||
" adcx %%rax, %%r10;" /* f[3]*f[0] */
|
||||
" movq 24(%0), %%rdx;" /* f[3] */
|
||||
" mulxq 8(%0), %%r11, %%rbx;"
|
||||
" adcx %%rcx, %%r11;" /* f[1]*f[3] */
|
||||
" mulxq 16(%0), %%rax, %%r13;"
|
||||
" adcx %%rax, %%rbx;" /* f[2]*f[3] */
|
||||
" movq 8(%0), %%rdx;"
|
||||
" adcx %%r15, %%r13;" /* f1 */
|
||||
" mulxq 16(%0), %%rax, %%rcx;"
|
||||
" mov $0, %%r14;" /* f[2]*f[1] */
|
||||
|
||||
/* Step 2: Compute two parallel carry chains */
|
||||
" xor %%r15d, %%r15d;"
|
||||
|
@ -542,39 +706,50 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
|
|||
" adcx %%r14, %%r14;"
|
||||
|
||||
/* Step 3: Compute intermediate squares */
|
||||
" movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
|
||||
" movq %%rax, 0(%0);"
|
||||
" add %%rcx, %%r8;" " movq %%r8, 8(%0);"
|
||||
" movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
|
||||
" adcx %%rax, %%r9;" " movq %%r9, 16(%0);"
|
||||
" adcx %%rcx, %%r10;" " movq %%r10, 24(%0);"
|
||||
" movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
|
||||
" adcx %%rax, %%r11;" " movq %%r11, 32(%0);"
|
||||
" adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);"
|
||||
" movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
|
||||
" adcx %%rax, %%r13;" " movq %%r13, 48(%0);"
|
||||
" adcx %%rcx, %%r14;" " movq %%r14, 56(%0);"
|
||||
" movq 0(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
|
||||
" movq %%rax, 0(%1);"
|
||||
" add %%rcx, %%r8;"
|
||||
" movq %%r8, 8(%1);"
|
||||
" movq 8(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
|
||||
" adcx %%rax, %%r9;"
|
||||
" movq %%r9, 16(%1);"
|
||||
" adcx %%rcx, %%r10;"
|
||||
" movq %%r10, 24(%1);"
|
||||
" movq 16(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
|
||||
" adcx %%rax, %%r11;"
|
||||
" movq %%r11, 32(%1);"
|
||||
" adcx %%rcx, %%rbx;"
|
||||
" movq %%rbx, 40(%1);"
|
||||
" movq 24(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
|
||||
" adcx %%rax, %%r13;"
|
||||
" movq %%r13, 48(%1);"
|
||||
" adcx %%rcx, %%r14;"
|
||||
" movq %%r14, 56(%1);"
|
||||
|
||||
/* Line up pointers */
|
||||
" mov %0, %1;"
|
||||
" mov %2, %0;"
|
||||
" mov %1, %0;"
|
||||
" mov %2, %1;"
|
||||
|
||||
/* Wrap the result back into the field */
|
||||
|
||||
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
|
||||
" mov $38, %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r13;"
|
||||
" mulxq 32(%0), %%r8, %%r13;"
|
||||
" xor %%ecx, %%ecx;"
|
||||
" adoxq 0(%1), %%r8;"
|
||||
" mulxq 40(%1), %%r9, %%rbx;"
|
||||
" adoxq 0(%0), %%r8;"
|
||||
" mulxq 40(%0), %%r9, %%rbx;"
|
||||
" adcx %%r13, %%r9;"
|
||||
" adoxq 8(%1), %%r9;"
|
||||
" mulxq 48(%1), %%r10, %%r13;"
|
||||
" adoxq 8(%0), %%r9;"
|
||||
" mulxq 48(%0), %%r10, %%r13;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" adoxq 16(%1), %%r10;"
|
||||
" mulxq 56(%1), %%r11, %%rax;"
|
||||
" adoxq 16(%0), %%r10;"
|
||||
" mulxq 56(%0), %%r11, %%rax;"
|
||||
" adcx %%r13, %%r11;"
|
||||
" adoxq 24(%1), %%r11;"
|
||||
" adoxq 24(%0), %%r11;"
|
||||
" adcx %%rcx, %%rax;"
|
||||
" adox %%rcx, %%rax;"
|
||||
" imul %%rdx, %%rax;"
|
||||
|
@ -582,40 +757,47 @@ static inline void fsqr(u64 *out, const u64 *f, u64 *tmp)
|
|||
/* Step 2: Fold the carry back into dst */
|
||||
" add %%rax, %%r8;"
|
||||
" adcx %%rcx, %%r9;"
|
||||
" movq %%r9, 8(%0);"
|
||||
" movq %%r9, 8(%1);"
|
||||
" adcx %%rcx, %%r10;"
|
||||
" movq %%r10, 16(%0);"
|
||||
" movq %%r10, 16(%1);"
|
||||
" adcx %%rcx, %%r11;"
|
||||
" movq %%r11, 24(%0);"
|
||||
" movq %%r11, 24(%1);"
|
||||
|
||||
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
|
||||
" mov $0, %%rax;"
|
||||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 0(%0);"
|
||||
: "+&r" (tmp), "+&r" (f), "+&r" (out)
|
||||
:
|
||||
: "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc"
|
||||
);
|
||||
" movq %%r8, 0(%1);"
|
||||
: "+&r"(f), "+&r"(tmp)
|
||||
: "r"(out)
|
||||
: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
|
||||
"%r13", "%r14", "%r15", "memory", "cc");
|
||||
}
|
||||
|
||||
/* Computes two field squarings:
|
||||
* out[0] <- f[0] * f[0]
|
||||
* out[1] <- f[1] * f[1]
|
||||
* out[0] <- f[0] * f[0]
|
||||
* out[1] <- f[1] * f[1]
|
||||
* Uses the 16-element buffer tmp for intermediate results */
|
||||
static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
|
||||
{
|
||||
asm volatile(
|
||||
/* Step 1: Compute all partial products */
|
||||
" movq 0(%1), %%rdx;" /* f[0] */
|
||||
" mulxq 8(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
|
||||
" mulxq 16(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
|
||||
" mulxq 24(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
|
||||
" movq 24(%1), %%rdx;" /* f[3] */
|
||||
" mulxq 8(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
|
||||
" mulxq 16(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
|
||||
" movq 8(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
|
||||
" mulxq 16(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
|
||||
" movq 0(%0), %%rdx;" /* f[0] */
|
||||
" mulxq 8(%0), %%r8, %%r14;"
|
||||
" xor %%r15d, %%r15d;" /* f[1]*f[0] */
|
||||
" mulxq 16(%0), %%r9, %%r10;"
|
||||
" adcx %%r14, %%r9;" /* f[2]*f[0] */
|
||||
" mulxq 24(%0), %%rax, %%rcx;"
|
||||
" adcx %%rax, %%r10;" /* f[3]*f[0] */
|
||||
" movq 24(%0), %%rdx;" /* f[3] */
|
||||
" mulxq 8(%0), %%r11, %%rbx;"
|
||||
" adcx %%rcx, %%r11;" /* f[1]*f[3] */
|
||||
" mulxq 16(%0), %%rax, %%r13;"
|
||||
" adcx %%rax, %%rbx;" /* f[2]*f[3] */
|
||||
" movq 8(%0), %%rdx;"
|
||||
" adcx %%r15, %%r13;" /* f1 */
|
||||
" mulxq 16(%0), %%rax, %%rcx;"
|
||||
" mov $0, %%r14;" /* f[2]*f[1] */
|
||||
|
||||
/* Step 2: Compute two parallel carry chains */
|
||||
" xor %%r15d, %%r15d;"
|
||||
|
@ -633,29 +815,47 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
|
|||
" adcx %%r14, %%r14;"
|
||||
|
||||
/* Step 3: Compute intermediate squares */
|
||||
" movq 0(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
|
||||
" movq %%rax, 0(%0);"
|
||||
" add %%rcx, %%r8;" " movq %%r8, 8(%0);"
|
||||
" movq 8(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
|
||||
" adcx %%rax, %%r9;" " movq %%r9, 16(%0);"
|
||||
" adcx %%rcx, %%r10;" " movq %%r10, 24(%0);"
|
||||
" movq 16(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
|
||||
" adcx %%rax, %%r11;" " movq %%r11, 32(%0);"
|
||||
" adcx %%rcx, %%rbx;" " movq %%rbx, 40(%0);"
|
||||
" movq 24(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
|
||||
" adcx %%rax, %%r13;" " movq %%r13, 48(%0);"
|
||||
" adcx %%rcx, %%r14;" " movq %%r14, 56(%0);"
|
||||
" movq 0(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
|
||||
" movq %%rax, 0(%1);"
|
||||
" add %%rcx, %%r8;"
|
||||
" movq %%r8, 8(%1);"
|
||||
" movq 8(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
|
||||
" adcx %%rax, %%r9;"
|
||||
" movq %%r9, 16(%1);"
|
||||
" adcx %%rcx, %%r10;"
|
||||
" movq %%r10, 24(%1);"
|
||||
" movq 16(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
|
||||
" adcx %%rax, %%r11;"
|
||||
" movq %%r11, 32(%1);"
|
||||
" adcx %%rcx, %%rbx;"
|
||||
" movq %%rbx, 40(%1);"
|
||||
" movq 24(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
|
||||
" adcx %%rax, %%r13;"
|
||||
" movq %%r13, 48(%1);"
|
||||
" adcx %%rcx, %%r14;"
|
||||
" movq %%r14, 56(%1);"
|
||||
|
||||
/* Step 1: Compute all partial products */
|
||||
" movq 32(%1), %%rdx;" /* f[0] */
|
||||
" mulxq 40(%1), %%r8, %%r14;" " xor %%r15d, %%r15d;" /* f[1]*f[0] */
|
||||
" mulxq 48(%1), %%r9, %%r10;" " adcx %%r14, %%r9;" /* f[2]*f[0] */
|
||||
" mulxq 56(%1), %%rax, %%rcx;" " adcx %%rax, %%r10;" /* f[3]*f[0] */
|
||||
" movq 56(%1), %%rdx;" /* f[3] */
|
||||
" mulxq 40(%1), %%r11, %%rbx;" " adcx %%rcx, %%r11;" /* f[1]*f[3] */
|
||||
" mulxq 48(%1), %%rax, %%r13;" " adcx %%rax, %%rbx;" /* f[2]*f[3] */
|
||||
" movq 40(%1), %%rdx;" " adcx %%r15, %%r13;" /* f1 */
|
||||
" mulxq 48(%1), %%rax, %%rcx;" " mov $0, %%r14;" /* f[2]*f[1] */
|
||||
" movq 32(%0), %%rdx;" /* f[0] */
|
||||
" mulxq 40(%0), %%r8, %%r14;"
|
||||
" xor %%r15d, %%r15d;" /* f[1]*f[0] */
|
||||
" mulxq 48(%0), %%r9, %%r10;"
|
||||
" adcx %%r14, %%r9;" /* f[2]*f[0] */
|
||||
" mulxq 56(%0), %%rax, %%rcx;"
|
||||
" adcx %%rax, %%r10;" /* f[3]*f[0] */
|
||||
" movq 56(%0), %%rdx;" /* f[3] */
|
||||
" mulxq 40(%0), %%r11, %%rbx;"
|
||||
" adcx %%rcx, %%r11;" /* f[1]*f[3] */
|
||||
" mulxq 48(%0), %%rax, %%r13;"
|
||||
" adcx %%rax, %%rbx;" /* f[2]*f[3] */
|
||||
" movq 40(%0), %%rdx;"
|
||||
" adcx %%r15, %%r13;" /* f1 */
|
||||
" mulxq 48(%0), %%rax, %%rcx;"
|
||||
" mov $0, %%r14;" /* f[2]*f[1] */
|
||||
|
||||
/* Step 2: Compute two parallel carry chains */
|
||||
" xor %%r15d, %%r15d;"
|
||||
|
@ -673,37 +873,48 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
|
|||
" adcx %%r14, %%r14;"
|
||||
|
||||
/* Step 3: Compute intermediate squares */
|
||||
" movq 32(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
|
||||
" movq %%rax, 64(%0);"
|
||||
" add %%rcx, %%r8;" " movq %%r8, 72(%0);"
|
||||
" movq 40(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
|
||||
" adcx %%rax, %%r9;" " movq %%r9, 80(%0);"
|
||||
" adcx %%rcx, %%r10;" " movq %%r10, 88(%0);"
|
||||
" movq 48(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
|
||||
" adcx %%rax, %%r11;" " movq %%r11, 96(%0);"
|
||||
" adcx %%rcx, %%rbx;" " movq %%rbx, 104(%0);"
|
||||
" movq 56(%1), %%rdx;" " mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
|
||||
" adcx %%rax, %%r13;" " movq %%r13, 112(%0);"
|
||||
" adcx %%rcx, %%r14;" " movq %%r14, 120(%0);"
|
||||
" movq 32(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[0]^2 */
|
||||
" movq %%rax, 64(%1);"
|
||||
" add %%rcx, %%r8;"
|
||||
" movq %%r8, 72(%1);"
|
||||
" movq 40(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[1]^2 */
|
||||
" adcx %%rax, %%r9;"
|
||||
" movq %%r9, 80(%1);"
|
||||
" adcx %%rcx, %%r10;"
|
||||
" movq %%r10, 88(%1);"
|
||||
" movq 48(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[2]^2 */
|
||||
" adcx %%rax, %%r11;"
|
||||
" movq %%r11, 96(%1);"
|
||||
" adcx %%rcx, %%rbx;"
|
||||
" movq %%rbx, 104(%1);"
|
||||
" movq 56(%0), %%rdx;"
|
||||
" mulx %%rdx, %%rax, %%rcx;" /* f[3]^2 */
|
||||
" adcx %%rax, %%r13;"
|
||||
" movq %%r13, 112(%1);"
|
||||
" adcx %%rcx, %%r14;"
|
||||
" movq %%r14, 120(%1);"
|
||||
|
||||
/* Line up pointers */
|
||||
" mov %0, %1;"
|
||||
" mov %2, %0;"
|
||||
" mov %1, %0;"
|
||||
" mov %2, %1;"
|
||||
|
||||
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
|
||||
" mov $38, %%rdx;"
|
||||
" mulxq 32(%1), %%r8, %%r13;"
|
||||
" mulxq 32(%0), %%r8, %%r13;"
|
||||
" xor %%ecx, %%ecx;"
|
||||
" adoxq 0(%1), %%r8;"
|
||||
" mulxq 40(%1), %%r9, %%rbx;"
|
||||
" adoxq 0(%0), %%r8;"
|
||||
" mulxq 40(%0), %%r9, %%rbx;"
|
||||
" adcx %%r13, %%r9;"
|
||||
" adoxq 8(%1), %%r9;"
|
||||
" mulxq 48(%1), %%r10, %%r13;"
|
||||
" adoxq 8(%0), %%r9;"
|
||||
" mulxq 48(%0), %%r10, %%r13;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" adoxq 16(%1), %%r10;"
|
||||
" mulxq 56(%1), %%r11, %%rax;"
|
||||
" adoxq 16(%0), %%r10;"
|
||||
" mulxq 56(%0), %%r11, %%rax;"
|
||||
" adcx %%r13, %%r11;"
|
||||
" adoxq 24(%1), %%r11;"
|
||||
" adoxq 24(%0), %%r11;"
|
||||
" adcx %%rcx, %%rax;"
|
||||
" adox %%rcx, %%rax;"
|
||||
" imul %%rdx, %%rax;"
|
||||
|
@ -711,32 +922,32 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
|
|||
/* Step 2: Fold the carry back into dst */
|
||||
" add %%rax, %%r8;"
|
||||
" adcx %%rcx, %%r9;"
|
||||
" movq %%r9, 8(%0);"
|
||||
" movq %%r9, 8(%1);"
|
||||
" adcx %%rcx, %%r10;"
|
||||
" movq %%r10, 16(%0);"
|
||||
" movq %%r10, 16(%1);"
|
||||
" adcx %%rcx, %%r11;"
|
||||
" movq %%r11, 24(%0);"
|
||||
" movq %%r11, 24(%1);"
|
||||
|
||||
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
|
||||
" mov $0, %%rax;"
|
||||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 0(%0);"
|
||||
" movq %%r8, 0(%1);"
|
||||
|
||||
/* Step 1: Compute dst + carry == tmp_hi * 38 + tmp_lo */
|
||||
" mov $38, %%rdx;"
|
||||
" mulxq 96(%1), %%r8, %%r13;"
|
||||
" mulxq 96(%0), %%r8, %%r13;"
|
||||
" xor %%ecx, %%ecx;"
|
||||
" adoxq 64(%1), %%r8;"
|
||||
" mulxq 104(%1), %%r9, %%rbx;"
|
||||
" adoxq 64(%0), %%r8;"
|
||||
" mulxq 104(%0), %%r9, %%rbx;"
|
||||
" adcx %%r13, %%r9;"
|
||||
" adoxq 72(%1), %%r9;"
|
||||
" mulxq 112(%1), %%r10, %%r13;"
|
||||
" adoxq 72(%0), %%r9;"
|
||||
" mulxq 112(%0), %%r10, %%r13;"
|
||||
" adcx %%rbx, %%r10;"
|
||||
" adoxq 80(%1), %%r10;"
|
||||
" mulxq 120(%1), %%r11, %%rax;"
|
||||
" adoxq 80(%0), %%r10;"
|
||||
" mulxq 120(%0), %%r11, %%rax;"
|
||||
" adcx %%r13, %%r11;"
|
||||
" adoxq 88(%1), %%r11;"
|
||||
" adoxq 88(%0), %%r11;"
|
||||
" adcx %%rcx, %%rax;"
|
||||
" adox %%rcx, %%rax;"
|
||||
" imul %%rdx, %%rax;"
|
||||
|
@ -744,21 +955,21 @@ static inline void fsqr2(u64 *out, const u64 *f, u64 *tmp)
|
|||
/* Step 2: Fold the carry back into dst */
|
||||
" add %%rax, %%r8;"
|
||||
" adcx %%rcx, %%r9;"
|
||||
" movq %%r9, 40(%0);"
|
||||
" movq %%r9, 40(%1);"
|
||||
" adcx %%rcx, %%r10;"
|
||||
" movq %%r10, 48(%0);"
|
||||
" movq %%r10, 48(%1);"
|
||||
" adcx %%rcx, %%r11;"
|
||||
" movq %%r11, 56(%0);"
|
||||
" movq %%r11, 56(%1);"
|
||||
|
||||
/* Step 3: Fold the carry bit back in; guaranteed not to carry at this point */
|
||||
" mov $0, %%rax;"
|
||||
" cmovc %%rdx, %%rax;"
|
||||
" add %%rax, %%r8;"
|
||||
" movq %%r8, 32(%0);"
|
||||
: "+&r" (tmp), "+&r" (f), "+&r" (out)
|
||||
:
|
||||
: "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%rbx", "%r13", "%r14", "%r15", "memory", "cc"
|
||||
);
|
||||
" movq %%r8, 32(%1);"
|
||||
: "+&r"(f), "+&r"(tmp)
|
||||
: "r"(out)
|
||||
: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11",
|
||||
"%r13", "%r14", "%r15", "memory", "cc");
|
||||
}
|
||||
|
||||
static void point_add_and_double(u64 *q, u64 *p01_tmp1, u64 *tmp2)
|
||||
|
|
|
@ -164,7 +164,7 @@ static int cbc_encrypt(struct skcipher_request *req)
|
|||
|
||||
err = skcipher_walk_virt(&walk, req, false);
|
||||
|
||||
while ((nbytes = walk.nbytes)) {
|
||||
while (walk.nbytes) {
|
||||
nbytes = __cbc_encrypt(ctx, &walk);
|
||||
err = skcipher_walk_done(&walk, nbytes);
|
||||
}
|
||||
|
@ -243,7 +243,7 @@ static int cbc_decrypt(struct skcipher_request *req)
|
|||
|
||||
err = skcipher_walk_virt(&walk, req, false);
|
||||
|
||||
while ((nbytes = walk.nbytes)) {
|
||||
while (walk.nbytes) {
|
||||
nbytes = __cbc_decrypt(ctx, &walk);
|
||||
err = skcipher_walk_done(&walk, nbytes);
|
||||
}
|
||||
|
|
|
@ -1845,6 +1845,10 @@ config CRYPTO_JITTERENTROPY
|
|||
random numbers. This Jitterentropy RNG registers with
|
||||
the kernel crypto API and can be used by any caller.
|
||||
|
||||
config CRYPTO_KDF800108_CTR
|
||||
tristate
|
||||
select CRYPTO_SHA256
|
||||
|
||||
config CRYPTO_USER_API
|
||||
tristate
|
||||
|
||||
|
|
|
@ -200,3 +200,8 @@ obj-$(CONFIG_ASYMMETRIC_KEY_TYPE) += asymmetric_keys/
|
|||
obj-$(CONFIG_CRYPTO_HASH_INFO) += hash_info.o
|
||||
crypto_simd-y := simd.o
|
||||
obj-$(CONFIG_CRYPTO_SIMD) += crypto_simd.o
|
||||
|
||||
#
|
||||
# Key derivation function
|
||||
#
|
||||
obj-$(CONFIG_CRYPTO_KDF800108_CTR) += kdf_sp800108.o
|
||||
|
|
|
@ -931,16 +931,19 @@ int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size,
|
|||
sg_unmark_end(sg + sgl->cur - 1);
|
||||
|
||||
do {
|
||||
struct page *pg;
|
||||
unsigned int i = sgl->cur;
|
||||
|
||||
plen = min_t(size_t, len, PAGE_SIZE);
|
||||
|
||||
sg_assign_page(sg + i, alloc_page(GFP_KERNEL));
|
||||
if (!sg_page(sg + i)) {
|
||||
pg = alloc_page(GFP_KERNEL);
|
||||
if (!pg) {
|
||||
err = -ENOMEM;
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
sg_assign_page(sg + i, pg);
|
||||
|
||||
err = memcpy_from_msg(page_address(sg_page(sg + i)),
|
||||
msg, plen);
|
||||
if (err) {
|
||||
|
|
|
@ -5,11 +5,11 @@
|
|||
* Authors: Salvatore Benedetto <salvatore.benedetto@intel.com>
|
||||
*/
|
||||
|
||||
#include <linux/fips.h>
|
||||
#include <linux/module.h>
|
||||
#include <crypto/internal/kpp.h>
|
||||
#include <crypto/kpp.h>
|
||||
#include <crypto/dh.h>
|
||||
#include <linux/fips.h>
|
||||
#include <linux/mpi.h>
|
||||
|
||||
struct dh_ctx {
|
||||
|
@ -47,6 +47,9 @@ static inline struct dh_ctx *dh_get_ctx(struct crypto_kpp *tfm)
|
|||
|
||||
static int dh_check_params_length(unsigned int p_len)
|
||||
{
|
||||
if (fips_enabled)
|
||||
return (p_len < 2048) ? -EINVAL : 0;
|
||||
|
||||
return (p_len < 1536) ? -EINVAL : 0;
|
||||
}
|
||||
|
||||
|
|
154
crypto/drbg.c
154
crypto/drbg.c
|
@ -100,6 +100,7 @@
|
|||
#include <crypto/drbg.h>
|
||||
#include <crypto/internal/cipher.h>
|
||||
#include <linux/kernel.h>
|
||||
#include <linux/jiffies.h>
|
||||
|
||||
/***************************************************************
|
||||
* Backend cipher definitions available to DRBG
|
||||
|
@ -1036,17 +1037,39 @@ static const struct drbg_state_ops drbg_hash_ops = {
|
|||
******************************************************************/
|
||||
|
||||
static inline int __drbg_seed(struct drbg_state *drbg, struct list_head *seed,
|
||||
int reseed)
|
||||
int reseed, enum drbg_seed_state new_seed_state)
|
||||
{
|
||||
int ret = drbg->d_ops->update(drbg, seed, reseed);
|
||||
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
drbg->seeded = true;
|
||||
drbg->seeded = new_seed_state;
|
||||
drbg->last_seed_time = jiffies;
|
||||
/* 10.1.1.2 / 10.1.1.3 step 5 */
|
||||
drbg->reseed_ctr = 1;
|
||||
|
||||
switch (drbg->seeded) {
|
||||
case DRBG_SEED_STATE_UNSEEDED:
|
||||
/* Impossible, but handle it to silence compiler warnings. */
|
||||
fallthrough;
|
||||
case DRBG_SEED_STATE_PARTIAL:
|
||||
/*
|
||||
* Require frequent reseeds until the seed source is
|
||||
* fully initialized.
|
||||
*/
|
||||
drbg->reseed_threshold = 50;
|
||||
break;
|
||||
|
||||
case DRBG_SEED_STATE_FULL:
|
||||
/*
|
||||
* Seed source has become fully initialized, frequent
|
||||
* reseeds no longer required.
|
||||
*/
|
||||
drbg->reseed_threshold = drbg_max_requests(drbg);
|
||||
break;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
@ -1066,12 +1089,10 @@ static inline int drbg_get_random_bytes(struct drbg_state *drbg,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void drbg_async_seed(struct work_struct *work)
|
||||
static int drbg_seed_from_random(struct drbg_state *drbg)
|
||||
{
|
||||
struct drbg_string data;
|
||||
LIST_HEAD(seedlist);
|
||||
struct drbg_state *drbg = container_of(work, struct drbg_state,
|
||||
seed_work);
|
||||
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
|
||||
unsigned char entropy[32];
|
||||
int ret;
|
||||
|
@ -1082,26 +1103,35 @@ static void drbg_async_seed(struct work_struct *work)
|
|||
drbg_string_fill(&data, entropy, entropylen);
|
||||
list_add_tail(&data.list, &seedlist);
|
||||
|
||||
mutex_lock(&drbg->drbg_mutex);
|
||||
|
||||
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
|
||||
if (ret)
|
||||
goto unlock;
|
||||
goto out;
|
||||
|
||||
/* Set seeded to false so that if __drbg_seed fails the
|
||||
* next generate call will trigger a reseed.
|
||||
*/
|
||||
drbg->seeded = false;
|
||||
|
||||
__drbg_seed(drbg, &seedlist, true);
|
||||
|
||||
if (drbg->seeded)
|
||||
drbg->reseed_threshold = drbg_max_requests(drbg);
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&drbg->drbg_mutex);
|
||||
ret = __drbg_seed(drbg, &seedlist, true, DRBG_SEED_STATE_FULL);
|
||||
|
||||
out:
|
||||
memzero_explicit(entropy, entropylen);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static bool drbg_nopr_reseed_interval_elapsed(struct drbg_state *drbg)
|
||||
{
|
||||
unsigned long next_reseed;
|
||||
|
||||
/* Don't ever reseed from get_random_bytes() in test mode. */
|
||||
if (list_empty(&drbg->test_data.list))
|
||||
return false;
|
||||
|
||||
/*
|
||||
* Obtain fresh entropy for the nopr DRBGs after 300s have
|
||||
* elapsed in order to still achieve sort of partial
|
||||
* prediction resistance over the time domain at least. Note
|
||||
* that the period of 300s has been chosen to match the
|
||||
* CRNG_RESEED_INTERVAL of the get_random_bytes()' chacha
|
||||
* rngs.
|
||||
*/
|
||||
next_reseed = drbg->last_seed_time + 300 * HZ;
|
||||
return time_after(jiffies, next_reseed);
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -1123,6 +1153,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
|
|||
unsigned int entropylen = drbg_sec_strength(drbg->core->flags);
|
||||
struct drbg_string data1;
|
||||
LIST_HEAD(seedlist);
|
||||
enum drbg_seed_state new_seed_state = DRBG_SEED_STATE_FULL;
|
||||
|
||||
/* 9.1 / 9.2 / 9.3.1 step 3 */
|
||||
if (pers && pers->len > (drbg_max_addtl(drbg))) {
|
||||
|
@ -1150,6 +1181,9 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
|
|||
BUG_ON((entropylen * 2) > sizeof(entropy));
|
||||
|
||||
/* Get seed from in-kernel /dev/urandom */
|
||||
if (!rng_is_initialized())
|
||||
new_seed_state = DRBG_SEED_STATE_PARTIAL;
|
||||
|
||||
ret = drbg_get_random_bytes(drbg, entropy, entropylen);
|
||||
if (ret)
|
||||
goto out;
|
||||
|
@ -1159,11 +1193,14 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
|
|||
pr_devel("DRBG: (re)seeding with %u bytes of entropy\n",
|
||||
entropylen);
|
||||
} else {
|
||||
/* Get seed from Jitter RNG */
|
||||
/*
|
||||
* Get seed from Jitter RNG, failures are
|
||||
* fatal only in FIPS mode.
|
||||
*/
|
||||
ret = crypto_rng_get_bytes(drbg->jent,
|
||||
entropy + entropylen,
|
||||
entropylen);
|
||||
if (ret) {
|
||||
if (fips_enabled && ret) {
|
||||
pr_devel("DRBG: jent failed with %d\n", ret);
|
||||
|
||||
/*
|
||||
|
@ -1206,7 +1243,7 @@ static int drbg_seed(struct drbg_state *drbg, struct drbg_string *pers,
|
|||
memset(drbg->C, 0, drbg_statelen(drbg));
|
||||
}
|
||||
|
||||
ret = __drbg_seed(drbg, &seedlist, reseed);
|
||||
ret = __drbg_seed(drbg, &seedlist, reseed, new_seed_state);
|
||||
|
||||
out:
|
||||
memzero_explicit(entropy, entropylen * 2);
|
||||
|
@ -1386,19 +1423,26 @@ static int drbg_generate(struct drbg_state *drbg,
|
|||
* here. The spec is a bit convoluted here, we make it simpler.
|
||||
*/
|
||||
if (drbg->reseed_threshold < drbg->reseed_ctr)
|
||||
drbg->seeded = false;
|
||||
drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
|
||||
|
||||
if (drbg->pr || !drbg->seeded) {
|
||||
if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) {
|
||||
pr_devel("DRBG: reseeding before generation (prediction "
|
||||
"resistance: %s, state %s)\n",
|
||||
drbg->pr ? "true" : "false",
|
||||
drbg->seeded ? "seeded" : "unseeded");
|
||||
(drbg->seeded == DRBG_SEED_STATE_FULL ?
|
||||
"seeded" : "unseeded"));
|
||||
/* 9.3.1 steps 7.1 through 7.3 */
|
||||
len = drbg_seed(drbg, addtl, true);
|
||||
if (len)
|
||||
goto err;
|
||||
/* 9.3.1 step 7.4 */
|
||||
addtl = NULL;
|
||||
} else if (rng_is_initialized() &&
|
||||
(drbg->seeded == DRBG_SEED_STATE_PARTIAL ||
|
||||
drbg_nopr_reseed_interval_elapsed(drbg))) {
|
||||
len = drbg_seed_from_random(drbg);
|
||||
if (len)
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (addtl && 0 < addtl->len)
|
||||
|
@ -1491,51 +1535,23 @@ static int drbg_generate_long(struct drbg_state *drbg,
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void drbg_schedule_async_seed(struct random_ready_callback *rdy)
|
||||
{
|
||||
struct drbg_state *drbg = container_of(rdy, struct drbg_state,
|
||||
random_ready);
|
||||
|
||||
schedule_work(&drbg->seed_work);
|
||||
}
|
||||
|
||||
static int drbg_prepare_hrng(struct drbg_state *drbg)
|
||||
{
|
||||
int err;
|
||||
|
||||
/* We do not need an HRNG in test mode. */
|
||||
if (list_empty(&drbg->test_data.list))
|
||||
return 0;
|
||||
|
||||
drbg->jent = crypto_alloc_rng("jitterentropy_rng", 0, 0);
|
||||
if (IS_ERR(drbg->jent)) {
|
||||
const int err = PTR_ERR(drbg->jent);
|
||||
|
||||
INIT_WORK(&drbg->seed_work, drbg_async_seed);
|
||||
|
||||
drbg->random_ready.owner = THIS_MODULE;
|
||||
drbg->random_ready.func = drbg_schedule_async_seed;
|
||||
|
||||
err = add_random_ready_callback(&drbg->random_ready);
|
||||
|
||||
switch (err) {
|
||||
case 0:
|
||||
break;
|
||||
|
||||
case -EALREADY:
|
||||
err = 0;
|
||||
fallthrough;
|
||||
|
||||
default:
|
||||
drbg->random_ready.func = NULL;
|
||||
return err;
|
||||
drbg->jent = NULL;
|
||||
if (fips_enabled || err != -ENOENT)
|
||||
return err;
|
||||
pr_info("DRBG: Continuing without Jitter RNG\n");
|
||||
}
|
||||
|
||||
/*
|
||||
* Require frequent reseeds until the seed source is fully
|
||||
* initialized.
|
||||
*/
|
||||
drbg->reseed_threshold = 50;
|
||||
|
||||
return err;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/*
|
||||
|
@ -1578,7 +1594,8 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
|
|||
if (!drbg->core) {
|
||||
drbg->core = &drbg_cores[coreref];
|
||||
drbg->pr = pr;
|
||||
drbg->seeded = false;
|
||||
drbg->seeded = DRBG_SEED_STATE_UNSEEDED;
|
||||
drbg->last_seed_time = 0;
|
||||
drbg->reseed_threshold = drbg_max_requests(drbg);
|
||||
|
||||
ret = drbg_alloc_state(drbg);
|
||||
|
@ -1589,14 +1606,6 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers,
|
|||
if (ret)
|
||||
goto free_everything;
|
||||
|
||||
if (IS_ERR(drbg->jent)) {
|
||||
ret = PTR_ERR(drbg->jent);
|
||||
drbg->jent = NULL;
|
||||
if (fips_enabled || ret != -ENOENT)
|
||||
goto free_everything;
|
||||
pr_info("DRBG: Continuing without Jitter RNG\n");
|
||||
}
|
||||
|
||||
reseed = false;
|
||||
}
|
||||
|
||||
|
@ -1629,11 +1638,6 @@ free_everything:
|
|||
*/
|
||||
static int drbg_uninstantiate(struct drbg_state *drbg)
|
||||
{
|
||||
if (drbg->random_ready.func) {
|
||||
del_random_ready_callback(&drbg->random_ready);
|
||||
cancel_work_sync(&drbg->seed_work);
|
||||
}
|
||||
|
||||
if (!IS_ERR_OR_NULL(drbg->jent))
|
||||
crypto_free_rng(drbg->jent);
|
||||
drbg->jent = NULL;
|
||||
|
|
|
@ -40,7 +40,6 @@
|
|||
#include <linux/kernel.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/slab.h>
|
||||
#include <linux/fips.h>
|
||||
#include <linux/time.h>
|
||||
#include <crypto/internal/rng.h>
|
||||
|
||||
|
@ -60,11 +59,6 @@ void jent_zfree(void *ptr)
|
|||
kfree_sensitive(ptr);
|
||||
}
|
||||
|
||||
int jent_fips_enabled(void)
|
||||
{
|
||||
return fips_enabled;
|
||||
}
|
||||
|
||||
void jent_panic(char *s)
|
||||
{
|
||||
panic("%s", s);
|
||||
|
|
|
@ -117,6 +117,22 @@ struct rand_data {
|
|||
#define JENT_EHEALTH 9 /* Health test failed during initialization */
|
||||
#define JENT_ERCT 10 /* RCT failed during initialization */
|
||||
|
||||
/*
|
||||
* The output n bits can receive more than n bits of min entropy, of course,
|
||||
* but the fixed output of the conditioning function can only asymptotically
|
||||
* approach the output size bits of min entropy, not attain that bound. Random
|
||||
* maps will tend to have output collisions, which reduces the creditable
|
||||
* output entropy (that is what SP 800-90B Section 3.1.5.1.2 attempts to bound).
|
||||
*
|
||||
* The value "64" is justified in Appendix A.4 of the current 90C draft,
|
||||
* and aligns with NIST's in "epsilon" definition in this document, which is
|
||||
* that a string can be considered "full entropy" if you can bound the min
|
||||
* entropy in each bit of output to at least 1-epsilon, where epsilon is
|
||||
* required to be <= 2^(-32).
|
||||
*/
|
||||
#define JENT_ENTROPY_SAFETY_FACTOR 64
|
||||
|
||||
#include <linux/fips.h>
|
||||
#include "jitterentropy.h"
|
||||
|
||||
/***************************************************************************
|
||||
|
@ -265,7 +281,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
|
|||
{
|
||||
__u64 delta2 = jent_delta(ec->last_delta, current_delta);
|
||||
__u64 delta3 = jent_delta(ec->last_delta2, delta2);
|
||||
unsigned int delta_masked = current_delta & JENT_APT_WORD_MASK;
|
||||
|
||||
ec->last_delta = current_delta;
|
||||
ec->last_delta2 = delta2;
|
||||
|
@ -274,7 +289,7 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
|
|||
* Insert the result of the comparison of two back-to-back time
|
||||
* deltas.
|
||||
*/
|
||||
jent_apt_insert(ec, delta_masked);
|
||||
jent_apt_insert(ec, current_delta);
|
||||
|
||||
if (!current_delta || !delta2 || !delta3) {
|
||||
/* RCT with a stuck bit */
|
||||
|
@ -299,10 +314,6 @@ static int jent_stuck(struct rand_data *ec, __u64 current_delta)
|
|||
*/
|
||||
static int jent_health_failure(struct rand_data *ec)
|
||||
{
|
||||
/* Test is only enabled in FIPS mode */
|
||||
if (!jent_fips_enabled())
|
||||
return 0;
|
||||
|
||||
return ec->health_failure;
|
||||
}
|
||||
|
||||
|
@ -547,12 +558,15 @@ static int jent_measure_jitter(struct rand_data *ec)
|
|||
*/
|
||||
static void jent_gen_entropy(struct rand_data *ec)
|
||||
{
|
||||
unsigned int k = 0;
|
||||
unsigned int k = 0, safety_factor = 0;
|
||||
|
||||
if (fips_enabled)
|
||||
safety_factor = JENT_ENTROPY_SAFETY_FACTOR;
|
||||
|
||||
/* priming of the ->prev_time value */
|
||||
jent_measure_jitter(ec);
|
||||
|
||||
while (1) {
|
||||
while (!jent_health_failure(ec)) {
|
||||
/* If a stuck measurement is received, repeat measurement */
|
||||
if (jent_measure_jitter(ec))
|
||||
continue;
|
||||
|
@ -561,7 +575,7 @@ static void jent_gen_entropy(struct rand_data *ec)
|
|||
* We multiply the loop value with ->osr to obtain the
|
||||
* oversampling rate requested by the caller
|
||||
*/
|
||||
if (++k >= (DATA_SIZE_BITS * ec->osr))
|
||||
if (++k >= ((DATA_SIZE_BITS + safety_factor) * ec->osr))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
extern void *jent_zalloc(unsigned int len);
|
||||
extern void jent_zfree(void *ptr);
|
||||
extern int jent_fips_enabled(void);
|
||||
extern void jent_panic(char *s);
|
||||
extern void jent_memcpy(void *dest, const void *src, unsigned int n);
|
||||
extern void jent_get_nstime(__u64 *out);
|
||||
|
|
153
crypto/kdf_sp800108.c
Normal file
153
crypto/kdf_sp800108.c
Normal file
|
@ -0,0 +1,153 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
|
||||
/*
|
||||
* SP800-108 Key-derivation function
|
||||
*
|
||||
* Copyright (C) 2021, Stephan Mueller <smueller@chronox.de>
|
||||
*/
|
||||
|
||||
#include <linux/fips.h>
|
||||
#include <linux/module.h>
|
||||
#include <crypto/kdf_sp800108.h>
|
||||
#include <crypto/internal/kdf_selftest.h>
|
||||
|
||||
/*
|
||||
* SP800-108 CTR KDF implementation
|
||||
*/
|
||||
int crypto_kdf108_ctr_generate(struct crypto_shash *kmd,
|
||||
const struct kvec *info, unsigned int info_nvec,
|
||||
u8 *dst, unsigned int dlen)
|
||||
{
|
||||
SHASH_DESC_ON_STACK(desc, kmd);
|
||||
__be32 counter = cpu_to_be32(1);
|
||||
const unsigned int h = crypto_shash_digestsize(kmd), dlen_orig = dlen;
|
||||
unsigned int i;
|
||||
int err = 0;
|
||||
u8 *dst_orig = dst;
|
||||
|
||||
desc->tfm = kmd;
|
||||
|
||||
while (dlen) {
|
||||
err = crypto_shash_init(desc);
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
err = crypto_shash_update(desc, (u8 *)&counter, sizeof(__be32));
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
for (i = 0; i < info_nvec; i++) {
|
||||
err = crypto_shash_update(desc, info[i].iov_base,
|
||||
info[i].iov_len);
|
||||
if (err)
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (dlen < h) {
|
||||
u8 tmpbuffer[HASH_MAX_DIGESTSIZE];
|
||||
|
||||
err = crypto_shash_final(desc, tmpbuffer);
|
||||
if (err)
|
||||
goto out;
|
||||
memcpy(dst, tmpbuffer, dlen);
|
||||
memzero_explicit(tmpbuffer, h);
|
||||
goto out;
|
||||
}
|
||||
|
||||
err = crypto_shash_final(desc, dst);
|
||||
if (err)
|
||||
goto out;
|
||||
|
||||
dlen -= h;
|
||||
dst += h;
|
||||
counter = cpu_to_be32(be32_to_cpu(counter) + 1);
|
||||
}
|
||||
|
||||
out:
|
||||
if (err)
|
||||
memzero_explicit(dst_orig, dlen_orig);
|
||||
shash_desc_zero(desc);
|
||||
return err;
|
||||
}
|
||||
EXPORT_SYMBOL(crypto_kdf108_ctr_generate);
|
||||
|
||||
/*
|
||||
* The seeding of the KDF
|
||||
*/
|
||||
int crypto_kdf108_setkey(struct crypto_shash *kmd,
|
||||
const u8 *key, size_t keylen,
|
||||
const u8 *ikm, size_t ikmlen)
|
||||
{
|
||||
unsigned int ds = crypto_shash_digestsize(kmd);
|
||||
|
||||
/* SP800-108 does not support IKM */
|
||||
if (ikm || ikmlen)
|
||||
return -EINVAL;
|
||||
|
||||
/* Check according to SP800-108 section 7.2 */
|
||||
if (ds > keylen)
|
||||
return -EINVAL;
|
||||
|
||||
/* Set the key for the MAC used for the KDF. */
|
||||
return crypto_shash_setkey(kmd, key, keylen);
|
||||
}
|
||||
EXPORT_SYMBOL(crypto_kdf108_setkey);
|
||||
|
||||
/*
|
||||
* Test vector obtained from
|
||||
* http://csrc.nist.gov/groups/STM/cavp/documents/KBKDF800-108/CounterMode.zip
|
||||
*/
|
||||
static const struct kdf_testvec kdf_ctr_hmac_sha256_tv_template[] = {
|
||||
{
|
||||
.key = "\xdd\x1d\x91\xb7\xd9\x0b\x2b\xd3"
|
||||
"\x13\x85\x33\xce\x92\xb2\x72\xfb"
|
||||
"\xf8\xa3\x69\x31\x6a\xef\xe2\x42"
|
||||
"\xe6\x59\xcc\x0a\xe2\x38\xaf\xe0",
|
||||
.keylen = 32,
|
||||
.ikm = NULL,
|
||||
.ikmlen = 0,
|
||||
.info = {
|
||||
.iov_base = "\x01\x32\x2b\x96\xb3\x0a\xcd\x19"
|
||||
"\x79\x79\x44\x4e\x46\x8e\x1c\x5c"
|
||||
"\x68\x59\xbf\x1b\x1c\xf9\x51\xb7"
|
||||
"\xe7\x25\x30\x3e\x23\x7e\x46\xb8"
|
||||
"\x64\xa1\x45\xfa\xb2\x5e\x51\x7b"
|
||||
"\x08\xf8\x68\x3d\x03\x15\xbb\x29"
|
||||
"\x11\xd8\x0a\x0e\x8a\xba\x17\xf3"
|
||||
"\xb4\x13\xfa\xac",
|
||||
.iov_len = 60
|
||||
},
|
||||
.expected = "\x10\x62\x13\x42\xbf\xb0\xfd\x40"
|
||||
"\x04\x6c\x0e\x29\xf2\xcf\xdb\xf0",
|
||||
.expectedlen = 16
|
||||
}
|
||||
};
|
||||
|
||||
static int __init crypto_kdf108_init(void)
|
||||
{
|
||||
int ret = kdf_test(&kdf_ctr_hmac_sha256_tv_template[0], "hmac(sha256)",
|
||||
crypto_kdf108_setkey, crypto_kdf108_ctr_generate);
|
||||
|
||||
if (ret) {
|
||||
if (fips_enabled)
|
||||
panic("alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
|
||||
ret);
|
||||
|
||||
WARN(1,
|
||||
"alg: self-tests for CTR-KDF (hmac(sha256)) failed (rc=%d)\n",
|
||||
ret);
|
||||
} else {
|
||||
pr_info("alg: self-tests for CTR-KDF (hmac(sha256)) passed\n");
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static void __exit crypto_kdf108_exit(void) { }
|
||||
|
||||
module_init(crypto_kdf108_init);
|
||||
module_exit(crypto_kdf108_exit);
|
||||
|
||||
MODULE_LICENSE("GPL v2");
|
||||
MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>");
|
||||
MODULE_DESCRIPTION("Key Derivation Function conformant to SP800-108");
|
|
@ -5,6 +5,7 @@
|
|||
* Authors: Tadeusz Struk <tadeusz.struk@intel.com>
|
||||
*/
|
||||
|
||||
#include <linux/fips.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/mpi.h>
|
||||
#include <crypto/internal/rsa.h>
|
||||
|
@ -144,6 +145,9 @@ static int rsa_check_key_length(unsigned int len)
|
|||
case 512:
|
||||
case 1024:
|
||||
case 1536:
|
||||
if (fips_enabled)
|
||||
return -EINVAL;
|
||||
fallthrough;
|
||||
case 2048:
|
||||
case 3072:
|
||||
case 4096:
|
||||
|
|
|
@ -33,18 +33,6 @@ const u8 sha256_zero_message_hash[SHA256_DIGEST_SIZE] = {
|
|||
};
|
||||
EXPORT_SYMBOL_GPL(sha256_zero_message_hash);
|
||||
|
||||
static int crypto_sha256_init(struct shash_desc *desc)
|
||||
{
|
||||
sha256_init(shash_desc_ctx(desc));
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int crypto_sha224_init(struct shash_desc *desc)
|
||||
{
|
||||
sha224_init(shash_desc_ctx(desc));
|
||||
return 0;
|
||||
}
|
||||
|
||||
int crypto_sha256_update(struct shash_desc *desc, const u8 *data,
|
||||
unsigned int len)
|
||||
{
|
||||
|
@ -72,7 +60,7 @@ EXPORT_SYMBOL(crypto_sha256_finup);
|
|||
|
||||
static struct shash_alg sha256_algs[2] = { {
|
||||
.digestsize = SHA256_DIGEST_SIZE,
|
||||
.init = crypto_sha256_init,
|
||||
.init = sha256_base_init,
|
||||
.update = crypto_sha256_update,
|
||||
.final = crypto_sha256_final,
|
||||
.finup = crypto_sha256_finup,
|
||||
|
@ -86,7 +74,7 @@ static struct shash_alg sha256_algs[2] = { {
|
|||
}
|
||||
}, {
|
||||
.digestsize = SHA224_DIGEST_SIZE,
|
||||
.init = crypto_sha224_init,
|
||||
.init = sha224_base_init,
|
||||
.update = crypto_sha256_update,
|
||||
.final = crypto_sha256_final,
|
||||
.finup = crypto_sha256_finup,
|
||||
|
|
|
@ -4193,7 +4193,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "authenc(hmac(sha1),cbc(des3_ede))",
|
||||
.test = alg_test_aead,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.aead = __VECS(hmac_sha1_des3_ede_cbc_tv_temp)
|
||||
}
|
||||
|
@ -4220,7 +4219,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "authenc(hmac(sha224),cbc(des3_ede))",
|
||||
.test = alg_test_aead,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.aead = __VECS(hmac_sha224_des3_ede_cbc_tv_temp)
|
||||
}
|
||||
|
@ -4240,7 +4238,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "authenc(hmac(sha256),cbc(des3_ede))",
|
||||
.test = alg_test_aead,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.aead = __VECS(hmac_sha256_des3_ede_cbc_tv_temp)
|
||||
}
|
||||
|
@ -4261,7 +4258,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "authenc(hmac(sha384),cbc(des3_ede))",
|
||||
.test = alg_test_aead,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.aead = __VECS(hmac_sha384_des3_ede_cbc_tv_temp)
|
||||
}
|
||||
|
@ -4289,7 +4285,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "authenc(hmac(sha512),cbc(des3_ede))",
|
||||
.test = alg_test_aead,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.aead = __VECS(hmac_sha512_des3_ede_cbc_tv_temp)
|
||||
}
|
||||
|
@ -4399,7 +4394,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "cbc(des3_ede)",
|
||||
.test = alg_test_skcipher,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.cipher = __VECS(des3_ede_cbc_tv_template)
|
||||
},
|
||||
|
@ -4505,7 +4499,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}
|
||||
}, {
|
||||
.alg = "cmac(des3_ede)",
|
||||
.fips_allowed = 1,
|
||||
.test = alg_test_hash,
|
||||
.suite = {
|
||||
.hash = __VECS(des3_ede_cmac64_tv_template)
|
||||
|
@ -4580,7 +4573,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "ctr(des3_ede)",
|
||||
.test = alg_test_skcipher,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.cipher = __VECS(des3_ede_ctr_tv_template)
|
||||
}
|
||||
|
@ -4846,7 +4838,6 @@ static const struct alg_test_desc alg_test_descs[] = {
|
|||
}, {
|
||||
.alg = "ecb(des3_ede)",
|
||||
.test = alg_test_skcipher,
|
||||
.fips_allowed = 1,
|
||||
.suite = {
|
||||
.cipher = __VECS(des3_ede_tv_template)
|
||||
}
|
||||
|
|
|
@ -257,9 +257,9 @@ static const struct akcipher_testvec rsa_tv_template[] = {
|
|||
}, {
|
||||
#endif
|
||||
.key =
|
||||
"\x30\x82\x02\x1F" /* sequence of 543 bytes */
|
||||
"\x30\x82\x02\x20" /* sequence of 544 bytes */
|
||||
"\x02\x01\x01" /* version - integer of 1 byte */
|
||||
"\x02\x82\x01\x00" /* modulus - integer of 256 bytes */
|
||||
"\x02\x82\x01\x01\x00" /* modulus - integer of 256 bytes */
|
||||
"\xDB\x10\x1A\xC2\xA3\xF1\xDC\xFF\x13\x6B\xED\x44\xDF\xF0\x02\x6D"
|
||||
"\x13\xC7\x88\xDA\x70\x6B\x54\xF1\xE8\x27\xDC\xC3\x0F\x99\x6A\xFA"
|
||||
"\xC6\x67\xFF\x1D\x1E\x3C\x1D\xC1\xB5\x5F\x6C\xC0\xB2\x07\x3A\x6D"
|
||||
|
@ -299,7 +299,7 @@ static const struct akcipher_testvec rsa_tv_template[] = {
|
|||
"\x02\x01\x00" /* exponent1 - integer of 1 byte */
|
||||
"\x02\x01\x00" /* exponent2 - integer of 1 byte */
|
||||
"\x02\x01\x00", /* coefficient - integer of 1 byte */
|
||||
.key_len = 547,
|
||||
.key_len = 548,
|
||||
.m = "\x54\x85\x9b\x34\x2c\x49\xea\x2a",
|
||||
.c =
|
||||
"\xb2\x97\x76\xb4\xae\x3e\x38\x3c\x7e\x64\x1f\xcc\xa2\x7f\xf6\xbe"
|
||||
|
|
|
@ -414,7 +414,7 @@ config HW_RANDOM_MESON
|
|||
|
||||
config HW_RANDOM_CAVIUM
|
||||
tristate "Cavium ThunderX Random Number Generator support"
|
||||
depends on HW_RANDOM && PCI && (ARM64 || (COMPILE_TEST && 64BIT))
|
||||
depends on HW_RANDOM && PCI && ARM64
|
||||
default HW_RANDOM
|
||||
help
|
||||
This driver provides kernel-side support for the Random Number
|
||||
|
@ -538,6 +538,17 @@ config HW_RANDOM_ARM_SMCCC_TRNG
|
|||
To compile this driver as a module, choose M here: the
|
||||
module will be called arm_smccc_trng.
|
||||
|
||||
config HW_RANDOM_CN10K
|
||||
tristate "Marvell CN10K Random Number Generator support"
|
||||
depends on HW_RANDOM && PCI && ARM64
|
||||
default HW_RANDOM
|
||||
help
|
||||
This driver provides support for the True Random Number
|
||||
generator available in Marvell CN10K SoCs.
|
||||
|
||||
To compile this driver as a module, choose M here.
|
||||
The module will be called cn10k_rng. If unsure, say Y.
|
||||
|
||||
endif # HW_RANDOM
|
||||
|
||||
config UML_RANDOM
|
||||
|
|
|
@ -46,3 +46,4 @@ obj-$(CONFIG_HW_RANDOM_NPCM) += npcm-rng.o
|
|||
obj-$(CONFIG_HW_RANDOM_CCTRNG) += cctrng.o
|
||||
obj-$(CONFIG_HW_RANDOM_XIPHERA) += xiphera-trng.o
|
||||
obj-$(CONFIG_HW_RANDOM_ARM_SMCCC_TRNG) += arm_smccc_trng.o
|
||||
obj-$(CONFIG_HW_RANDOM_CN10K) += cn10k-rng.o
|
||||
|
|
|
@ -1,10 +1,7 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Hardware Random Number Generator support for Cavium, Inc.
|
||||
* Thunder processor family.
|
||||
*
|
||||
* This file is subject to the terms and conditions of the GNU General Public
|
||||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
* Hardware Random Number Generator support.
|
||||
* Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
|
||||
*
|
||||
* Copyright (C) 2016 Cavium, Inc.
|
||||
*/
|
||||
|
@ -15,16 +12,146 @@
|
|||
#include <linux/pci.h>
|
||||
#include <linux/pci_ids.h>
|
||||
|
||||
#include <asm/arch_timer.h>
|
||||
|
||||
/* PCI device IDs */
|
||||
#define PCI_DEVID_CAVIUM_RNG_PF 0xA018
|
||||
#define PCI_DEVID_CAVIUM_RNG_VF 0xA033
|
||||
|
||||
#define HEALTH_STATUS_REG 0x38
|
||||
|
||||
/* RST device info */
|
||||
#define PCI_DEVICE_ID_RST_OTX2 0xA085
|
||||
#define RST_BOOT_REG 0x1600ULL
|
||||
#define CLOCK_BASE_RATE 50000000ULL
|
||||
#define MSEC_TO_NSEC(x) (x * 1000000)
|
||||
|
||||
struct cavium_rng {
|
||||
struct hwrng ops;
|
||||
void __iomem *result;
|
||||
void __iomem *pf_regbase;
|
||||
struct pci_dev *pdev;
|
||||
u64 clock_rate;
|
||||
u64 prev_error;
|
||||
u64 prev_time;
|
||||
};
|
||||
|
||||
static inline bool is_octeontx(struct pci_dev *pdev)
|
||||
{
|
||||
if (midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_83XX,
|
||||
MIDR_CPU_VAR_REV(0, 0),
|
||||
MIDR_CPU_VAR_REV(3, 0)) ||
|
||||
midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX_81XX,
|
||||
MIDR_CPU_VAR_REV(0, 0),
|
||||
MIDR_CPU_VAR_REV(3, 0)) ||
|
||||
midr_is_cpu_model_range(read_cpuid_id(), MIDR_THUNDERX,
|
||||
MIDR_CPU_VAR_REV(0, 0),
|
||||
MIDR_CPU_VAR_REV(3, 0)))
|
||||
return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
static u64 rng_get_coprocessor_clkrate(void)
|
||||
{
|
||||
u64 ret = CLOCK_BASE_RATE * 16; /* Assume 800Mhz as default */
|
||||
struct pci_dev *pdev;
|
||||
void __iomem *base;
|
||||
|
||||
pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
|
||||
PCI_DEVICE_ID_RST_OTX2, NULL);
|
||||
if (!pdev)
|
||||
goto error;
|
||||
|
||||
base = pci_ioremap_bar(pdev, 0);
|
||||
if (!base)
|
||||
goto error_put_pdev;
|
||||
|
||||
/* RST: PNR_MUL * 50Mhz gives clockrate */
|
||||
ret = CLOCK_BASE_RATE * ((readq(base + RST_BOOT_REG) >> 33) & 0x3F);
|
||||
|
||||
iounmap(base);
|
||||
|
||||
error_put_pdev:
|
||||
pci_dev_put(pdev);
|
||||
|
||||
error:
|
||||
return ret;
|
||||
}
|
||||
|
||||
static int check_rng_health(struct cavium_rng *rng)
|
||||
{
|
||||
u64 cur_err, cur_time;
|
||||
u64 status, cycles;
|
||||
u64 time_elapsed;
|
||||
|
||||
|
||||
/* Skip checking health for OcteonTx */
|
||||
if (!rng->pf_regbase)
|
||||
return 0;
|
||||
|
||||
status = readq(rng->pf_regbase + HEALTH_STATUS_REG);
|
||||
if (status & BIT_ULL(0)) {
|
||||
dev_err(&rng->pdev->dev, "HWRNG: Startup health test failed\n");
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
cycles = status >> 1;
|
||||
if (!cycles)
|
||||
return 0;
|
||||
|
||||
cur_time = arch_timer_read_counter();
|
||||
|
||||
/* RNM_HEALTH_STATUS[CYCLES_SINCE_HEALTH_FAILURE]
|
||||
* Number of coprocessor cycles times 2 since the last failure.
|
||||
* This field doesn't get cleared/updated until another failure.
|
||||
*/
|
||||
cycles = cycles / 2;
|
||||
cur_err = (cycles * 1000000000) / rng->clock_rate; /* In nanosec */
|
||||
|
||||
/* Ignore errors that happenned a long time ago, these
|
||||
* are most likely false positive errors.
|
||||
*/
|
||||
if (cur_err > MSEC_TO_NSEC(10)) {
|
||||
rng->prev_error = 0;
|
||||
rng->prev_time = 0;
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (rng->prev_error) {
|
||||
/* Calculate time elapsed since last error
|
||||
* '1' tick of CNTVCT is 10ns, since it runs at 100Mhz.
|
||||
*/
|
||||
time_elapsed = (cur_time - rng->prev_time) * 10;
|
||||
time_elapsed += rng->prev_error;
|
||||
|
||||
/* Check if current error is a new one or the old one itself.
|
||||
* If error is a new one then consider there is a persistent
|
||||
* issue with entropy, declare hardware failure.
|
||||
*/
|
||||
if (cur_err < time_elapsed) {
|
||||
dev_err(&rng->pdev->dev, "HWRNG failure detected\n");
|
||||
rng->prev_error = cur_err;
|
||||
rng->prev_time = cur_time;
|
||||
return -EIO;
|
||||
}
|
||||
}
|
||||
|
||||
rng->prev_error = cur_err;
|
||||
rng->prev_time = cur_time;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Read data from the RNG unit */
|
||||
static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
|
||||
{
|
||||
struct cavium_rng *p = container_of(rng, struct cavium_rng, ops);
|
||||
unsigned int size = max;
|
||||
int err = 0;
|
||||
|
||||
err = check_rng_health(p);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
while (size >= 8) {
|
||||
*((u64 *)dat) = readq(p->result);
|
||||
|
@ -39,6 +166,39 @@ static int cavium_rng_read(struct hwrng *rng, void *dat, size_t max, bool wait)
|
|||
return max;
|
||||
}
|
||||
|
||||
static int cavium_map_pf_regs(struct cavium_rng *rng)
|
||||
{
|
||||
struct pci_dev *pdev;
|
||||
|
||||
/* Health status is not supported on 83xx, skip mapping PF CSRs */
|
||||
if (is_octeontx(rng->pdev)) {
|
||||
rng->pf_regbase = NULL;
|
||||
return 0;
|
||||
}
|
||||
|
||||
pdev = pci_get_device(PCI_VENDOR_ID_CAVIUM,
|
||||
PCI_DEVID_CAVIUM_RNG_PF, NULL);
|
||||
if (!pdev) {
|
||||
dev_err(&pdev->dev, "Cannot find RNG PF device\n");
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
rng->pf_regbase = ioremap(pci_resource_start(pdev, 0),
|
||||
pci_resource_len(pdev, 0));
|
||||
if (!rng->pf_regbase) {
|
||||
dev_err(&pdev->dev, "Failed to map PF CSR region\n");
|
||||
pci_dev_put(pdev);
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
pci_dev_put(pdev);
|
||||
|
||||
/* Get co-processor clock rate */
|
||||
rng->clock_rate = rng_get_coprocessor_clkrate();
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Map Cavium RNG to an HWRNG object */
|
||||
static int cavium_rng_probe_vf(struct pci_dev *pdev,
|
||||
const struct pci_device_id *id)
|
||||
|
@ -50,6 +210,8 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
|
|||
if (!rng)
|
||||
return -ENOMEM;
|
||||
|
||||
rng->pdev = pdev;
|
||||
|
||||
/* Map the RNG result */
|
||||
rng->result = pcim_iomap(pdev, 0, 0);
|
||||
if (!rng->result) {
|
||||
|
@ -67,6 +229,11 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
|
|||
|
||||
pci_set_drvdata(pdev, rng);
|
||||
|
||||
/* Health status is available only at PF, hence map PF registers. */
|
||||
ret = cavium_map_pf_regs(rng);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = devm_hwrng_register(&pdev->dev, &rng->ops);
|
||||
if (ret) {
|
||||
dev_err(&pdev->dev, "Error registering device as HWRNG.\n");
|
||||
|
@ -76,10 +243,18 @@ static int cavium_rng_probe_vf(struct pci_dev *pdev,
|
|||
return 0;
|
||||
}
|
||||
|
||||
/* Remove the VF */
|
||||
static void cavium_rng_remove_vf(struct pci_dev *pdev)
|
||||
{
|
||||
struct cavium_rng *rng;
|
||||
|
||||
rng = pci_get_drvdata(pdev);
|
||||
iounmap(rng->pf_regbase);
|
||||
}
|
||||
|
||||
static const struct pci_device_id cavium_rng_vf_id_table[] = {
|
||||
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xa033), 0, 0, 0},
|
||||
{0,},
|
||||
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, PCI_DEVID_CAVIUM_RNG_VF) },
|
||||
{ 0, }
|
||||
};
|
||||
MODULE_DEVICE_TABLE(pci, cavium_rng_vf_id_table);
|
||||
|
||||
|
@ -87,8 +262,9 @@ static struct pci_driver cavium_rng_vf_driver = {
|
|||
.name = "cavium_rng_vf",
|
||||
.id_table = cavium_rng_vf_id_table,
|
||||
.probe = cavium_rng_probe_vf,
|
||||
.remove = cavium_rng_remove_vf,
|
||||
};
|
||||
module_pci_driver(cavium_rng_vf_driver);
|
||||
|
||||
MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_LICENSE("GPL v2");
|
||||
|
|
|
@ -1,10 +1,7 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/*
|
||||
* Hardware Random Number Generator support for Cavium Inc.
|
||||
* Thunder processor family.
|
||||
*
|
||||
* This file is subject to the terms and conditions of the GNU General Public
|
||||
* License. See the file "COPYING" in the main directory of this archive
|
||||
* for more details.
|
||||
* Hardware Random Number Generator support.
|
||||
* Cavium Thunder, Marvell OcteonTx/Tx2 processor families.
|
||||
*
|
||||
* Copyright (C) 2016 Cavium, Inc.
|
||||
*/
|
||||
|
@ -91,4 +88,4 @@ static struct pci_driver cavium_rng_pf_driver = {
|
|||
|
||||
module_pci_driver(cavium_rng_pf_driver);
|
||||
MODULE_AUTHOR("Omer Khaliq <okhaliq@caviumnetworks.com>");
|
||||
MODULE_LICENSE("GPL");
|
||||
MODULE_LICENSE("GPL v2");
|
||||
|
|
181
drivers/char/hw_random/cn10k-rng.c
Normal file
181
drivers/char/hw_random/cn10k-rng.c
Normal file
|
@ -0,0 +1,181 @@
|
|||
// SPDX-License-Identifier: GPL-2.0
|
||||
/* Marvell CN10K RVU Hardware Random Number Generator.
|
||||
*
|
||||
* Copyright (C) 2021 Marvell.
|
||||
*
|
||||
*/
|
||||
|
||||
#include <linux/hw_random.h>
|
||||
#include <linux/io.h>
|
||||
#include <linux/module.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/pci_ids.h>
|
||||
#include <linux/delay.h>
|
||||
|
||||
#include <linux/arm-smccc.h>
|
||||
|
||||
/* CSRs */
|
||||
#define RNM_CTL_STATUS 0x000
|
||||
#define RNM_ENTROPY_STATUS 0x008
|
||||
#define RNM_CONST 0x030
|
||||
#define RNM_EBG_ENT 0x048
|
||||
#define RNM_PF_EBG_HEALTH 0x050
|
||||
#define RNM_PF_RANDOM 0x400
|
||||
#define RNM_TRNG_RESULT 0x408
|
||||
|
||||
struct cn10k_rng {
|
||||
void __iomem *reg_base;
|
||||
struct hwrng ops;
|
||||
struct pci_dev *pdev;
|
||||
};
|
||||
|
||||
#define PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE 0xc2000b0f
|
||||
|
||||
static int reset_rng_health_state(struct cn10k_rng *rng)
|
||||
{
|
||||
struct arm_smccc_res res;
|
||||
|
||||
/* Send SMC service call to reset EBG health state */
|
||||
arm_smccc_smc(PLAT_OCTEONTX_RESET_RNG_EBG_HEALTH_STATE, 0, 0, 0, 0, 0, 0, 0, &res);
|
||||
if (res.a0 != 0UL)
|
||||
return -EIO;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int check_rng_health(struct cn10k_rng *rng)
|
||||
{
|
||||
u64 status;
|
||||
int err;
|
||||
|
||||
/* Skip checking health */
|
||||
if (!rng->reg_base)
|
||||
return 0;
|
||||
|
||||
status = readq(rng->reg_base + RNM_PF_EBG_HEALTH);
|
||||
if (status & BIT_ULL(20)) {
|
||||
err = reset_rng_health_state(rng);
|
||||
if (err) {
|
||||
dev_err(&rng->pdev->dev, "HWRNG: Health test failed (status=%llx)\n",
|
||||
status);
|
||||
dev_err(&rng->pdev->dev, "HWRNG: error during reset\n");
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void cn10k_read_trng(struct cn10k_rng *rng, u64 *value)
|
||||
{
|
||||
u64 upper, lower;
|
||||
|
||||
*value = readq(rng->reg_base + RNM_PF_RANDOM);
|
||||
|
||||
/* HW can run out of entropy if large amount random data is read in
|
||||
* quick succession. Zeros may not be real random data from HW.
|
||||
*/
|
||||
if (!*value) {
|
||||
upper = readq(rng->reg_base + RNM_PF_RANDOM);
|
||||
lower = readq(rng->reg_base + RNM_PF_RANDOM);
|
||||
while (!(upper & 0x00000000FFFFFFFFULL))
|
||||
upper = readq(rng->reg_base + RNM_PF_RANDOM);
|
||||
while (!(lower & 0xFFFFFFFF00000000ULL))
|
||||
lower = readq(rng->reg_base + RNM_PF_RANDOM);
|
||||
|
||||
*value = (upper & 0xFFFFFFFF00000000) | (lower & 0xFFFFFFFF);
|
||||
}
|
||||
}
|
||||
|
||||
static int cn10k_rng_read(struct hwrng *hwrng, void *data,
|
||||
size_t max, bool wait)
|
||||
{
|
||||
struct cn10k_rng *rng = (struct cn10k_rng *)hwrng->priv;
|
||||
unsigned int size;
|
||||
int err = 0;
|
||||
u64 value;
|
||||
|
||||
err = check_rng_health(rng);
|
||||
if (err)
|
||||
return err;
|
||||
|
||||
size = max;
|
||||
|
||||
while (size >= 8) {
|
||||
cn10k_read_trng(rng, &value);
|
||||
|
||||
*((u64 *)data) = (u64)value;
|
||||
size -= 8;
|
||||
data += 8;
|
||||
}
|
||||
|
||||
while (size > 0) {
|
||||
cn10k_read_trng(rng, &value);
|
||||
|
||||
*((u8 *)data) = (u8)value;
|
||||
size--;
|
||||
data++;
|
||||
}
|
||||
|
||||
return max - size;
|
||||
}
|
||||
|
||||
static int cn10k_rng_probe(struct pci_dev *pdev, const struct pci_device_id *id)
|
||||
{
|
||||
struct cn10k_rng *rng;
|
||||
int err;
|
||||
|
||||
rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL);
|
||||
if (!rng)
|
||||
return -ENOMEM;
|
||||
|
||||
rng->pdev = pdev;
|
||||
pci_set_drvdata(pdev, rng);
|
||||
|
||||
rng->reg_base = pcim_iomap(pdev, 0, 0);
|
||||
if (!rng->reg_base) {
|
||||
dev_err(&pdev->dev, "Error while mapping CSRs, exiting\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
rng->ops.name = devm_kasprintf(&pdev->dev, GFP_KERNEL,
|
||||
"cn10k-rng-%s", dev_name(&pdev->dev));
|
||||
if (!rng->ops.name)
|
||||
return -ENOMEM;
|
||||
|
||||
rng->ops.read = cn10k_rng_read;
|
||||
rng->ops.quality = 1000;
|
||||
rng->ops.priv = (unsigned long)rng;
|
||||
|
||||
reset_rng_health_state(rng);
|
||||
|
||||
err = devm_hwrng_register(&pdev->dev, &rng->ops);
|
||||
if (err) {
|
||||
dev_err(&pdev->dev, "Could not register hwrng device.\n");
|
||||
return err;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void cn10k_rng_remove(struct pci_dev *pdev)
|
||||
{
|
||||
/* Nothing to do */
|
||||
}
|
||||
|
||||
static const struct pci_device_id cn10k_rng_id_table[] = {
|
||||
{ PCI_DEVICE(PCI_VENDOR_ID_CAVIUM, 0xA098) }, /* RNG PF */
|
||||
{0,},
|
||||
};
|
||||
|
||||
MODULE_DEVICE_TABLE(pci, cn10k_rng_id_table);
|
||||
|
||||
static struct pci_driver cn10k_rng_driver = {
|
||||
.name = "cn10k_rng",
|
||||
.id_table = cn10k_rng_id_table,
|
||||
.probe = cn10k_rng_probe,
|
||||
.remove = cn10k_rng_remove,
|
||||
};
|
||||
|
||||
module_pci_driver(cn10k_rng_driver);
|
||||
MODULE_AUTHOR("Sunil Goutham <sgoutham@marvell.com>");
|
||||
MODULE_DESCRIPTION("Marvell CN10K HW RNG Driver");
|
||||
MODULE_LICENSE("GPL v2");
|
|
@ -106,6 +106,24 @@ static const struct ce_variant ce_a64_variant = {
|
|||
.trng = CE_ID_NOTSUPP,
|
||||
};
|
||||
|
||||
static const struct ce_variant ce_d1_variant = {
|
||||
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
|
||||
},
|
||||
.alg_hash = { CE_ALG_MD5, CE_ALG_SHA1, CE_ALG_SHA224, CE_ALG_SHA256,
|
||||
CE_ALG_SHA384, CE_ALG_SHA512
|
||||
},
|
||||
.op_mode = { CE_OP_ECB, CE_OP_CBC
|
||||
},
|
||||
.ce_clks = {
|
||||
{ "bus", 0, 200000000 },
|
||||
{ "mod", 300000000, 0 },
|
||||
{ "ram", 0, 400000000 },
|
||||
},
|
||||
.esr = ESR_D1,
|
||||
.prng = CE_ALG_PRNG,
|
||||
.trng = CE_ALG_TRNG,
|
||||
};
|
||||
|
||||
static const struct ce_variant ce_r40_variant = {
|
||||
.alg_cipher = { CE_ALG_AES, CE_ALG_DES, CE_ALG_3DES,
|
||||
},
|
||||
|
@ -192,6 +210,7 @@ int sun8i_ce_run_task(struct sun8i_ce_dev *ce, int flow, const char *name)
|
|||
dev_err(ce->dev, "CE ERROR: keysram access error for AES\n");
|
||||
break;
|
||||
case ESR_A64:
|
||||
case ESR_D1:
|
||||
case ESR_H5:
|
||||
case ESR_R40:
|
||||
v >>= (flow * 4);
|
||||
|
@ -990,6 +1009,8 @@ static const struct of_device_id sun8i_ce_crypto_of_match_table[] = {
|
|||
.data = &ce_h3_variant },
|
||||
{ .compatible = "allwinner,sun8i-r40-crypto",
|
||||
.data = &ce_r40_variant },
|
||||
{ .compatible = "allwinner,sun20i-d1-crypto",
|
||||
.data = &ce_d1_variant },
|
||||
{ .compatible = "allwinner,sun50i-a64-crypto",
|
||||
.data = &ce_a64_variant },
|
||||
{ .compatible = "allwinner,sun50i-h5-crypto",
|
||||
|
|
|
@ -94,6 +94,7 @@
|
|||
#define ESR_R40 2
|
||||
#define ESR_H5 3
|
||||
#define ESR_H6 4
|
||||
#define ESR_D1 5
|
||||
|
||||
#define PRNG_DATA_SIZE (160 / 8)
|
||||
#define PRNG_SEED_SIZE DIV_ROUND_UP(175, 8)
|
||||
|
|
|
@ -960,6 +960,7 @@ static int atmel_aes_handle_queue(struct atmel_aes_dev *dd,
|
|||
ctx = crypto_tfm_ctx(areq->tfm);
|
||||
|
||||
dd->areq = areq;
|
||||
dd->ctx = ctx;
|
||||
start_async = (areq != new_areq);
|
||||
dd->is_async = start_async;
|
||||
|
||||
|
@ -1274,7 +1275,6 @@ static int atmel_aes_init_tfm(struct crypto_skcipher *tfm)
|
|||
|
||||
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
|
||||
ctx->base.dd = dd;
|
||||
ctx->base.dd->ctx = &ctx->base;
|
||||
ctx->base.start = atmel_aes_start;
|
||||
|
||||
return 0;
|
||||
|
@ -1291,7 +1291,6 @@ static int atmel_aes_ctr_init_tfm(struct crypto_skcipher *tfm)
|
|||
|
||||
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
|
||||
ctx->base.dd = dd;
|
||||
ctx->base.dd->ctx = &ctx->base;
|
||||
ctx->base.start = atmel_aes_ctr_start;
|
||||
|
||||
return 0;
|
||||
|
@ -1783,7 +1782,6 @@ static int atmel_aes_gcm_init(struct crypto_aead *tfm)
|
|||
|
||||
crypto_aead_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx));
|
||||
ctx->base.dd = dd;
|
||||
ctx->base.dd->ctx = &ctx->base;
|
||||
ctx->base.start = atmel_aes_gcm_start;
|
||||
|
||||
return 0;
|
||||
|
@ -1927,7 +1925,6 @@ static int atmel_aes_xts_init_tfm(struct crypto_skcipher *tfm)
|
|||
crypto_skcipher_set_reqsize(tfm, sizeof(struct atmel_aes_reqctx) +
|
||||
crypto_skcipher_reqsize(ctx->fallback_tfm));
|
||||
ctx->base.dd = dd;
|
||||
ctx->base.dd->ctx = &ctx->base;
|
||||
ctx->base.start = atmel_aes_xts_start;
|
||||
|
||||
return 0;
|
||||
|
@ -2154,7 +2151,6 @@ static int atmel_aes_authenc_init_tfm(struct crypto_aead *tfm,
|
|||
crypto_aead_set_reqsize(tfm, (sizeof(struct atmel_aes_authenc_reqctx) +
|
||||
auth_reqsize));
|
||||
ctx->base.dd = dd;
|
||||
ctx->base.dd->ctx = &ctx->base;
|
||||
ctx->base.start = atmel_aes_authenc_start;
|
||||
|
||||
return 0;
|
||||
|
|
|
@ -1533,6 +1533,9 @@ static int aead_do_one_req(struct crypto_engine *engine, void *areq)
|
|||
|
||||
ret = caam_jr_enqueue(ctx->jrdev, desc, aead_crypt_done, req);
|
||||
|
||||
if (ret == -ENOSPC && engine->retry_support)
|
||||
return ret;
|
||||
|
||||
if (ret != -EINPROGRESS) {
|
||||
aead_unmap(ctx->jrdev, rctx->edesc, req);
|
||||
kfree(rctx->edesc);
|
||||
|
@ -1762,6 +1765,9 @@ static int skcipher_do_one_req(struct crypto_engine *engine, void *areq)
|
|||
|
||||
ret = caam_jr_enqueue(ctx->jrdev, desc, skcipher_crypt_done, req);
|
||||
|
||||
if (ret == -ENOSPC && engine->retry_support)
|
||||
return ret;
|
||||
|
||||
if (ret != -EINPROGRESS) {
|
||||
skcipher_unmap(ctx->jrdev, rctx->edesc, req);
|
||||
kfree(rctx->edesc);
|
||||
|
|
|
@ -5470,7 +5470,7 @@ int dpaa2_caam_enqueue(struct device *dev, struct caam_request *req)
|
|||
dpaa2_fd_set_len(&fd, dpaa2_fl_get_len(&req->fd_flt[1]));
|
||||
dpaa2_fd_set_flc(&fd, req->flc_dma);
|
||||
|
||||
ppriv = this_cpu_ptr(priv->ppriv);
|
||||
ppriv = raw_cpu_ptr(priv->ppriv);
|
||||
for (i = 0; i < (priv->dpseci_attr.num_tx_queues << 1); i++) {
|
||||
err = dpaa2_io_service_enqueue_fq(ppriv->dpio, ppriv->req_fqid,
|
||||
&fd);
|
||||
|
|
|
@ -765,6 +765,9 @@ static int ahash_do_one_req(struct crypto_engine *engine, void *areq)
|
|||
|
||||
ret = caam_jr_enqueue(jrdev, desc, state->ahash_op_done, req);
|
||||
|
||||
if (ret == -ENOSPC && engine->retry_support)
|
||||
return ret;
|
||||
|
||||
if (ret != -EINPROGRESS) {
|
||||
ahash_unmap(jrdev, state->edesc, req, 0);
|
||||
kfree(state->edesc);
|
||||
|
|
|
@ -380,6 +380,9 @@ static int akcipher_do_one_req(struct crypto_engine *engine, void *areq)
|
|||
|
||||
ret = caam_jr_enqueue(jrdev, desc, req_ctx->akcipher_op_done, req);
|
||||
|
||||
if (ret == -ENOSPC && engine->retry_support)
|
||||
return ret;
|
||||
|
||||
if (ret != -EINPROGRESS) {
|
||||
rsa_pub_unmap(jrdev, req_ctx->edesc, req);
|
||||
rsa_io_unmap(jrdev, req_ctx->edesc, req);
|
||||
|
|
|
@ -104,17 +104,14 @@ static int alloc_pending_queues(struct pending_qinfo *pqinfo, u32 qlen,
|
|||
u32 nr_queues)
|
||||
{
|
||||
u32 i;
|
||||
size_t size;
|
||||
int ret;
|
||||
struct pending_queue *queue = NULL;
|
||||
|
||||
pqinfo->nr_queues = nr_queues;
|
||||
pqinfo->qlen = qlen;
|
||||
|
||||
size = (qlen * sizeof(struct pending_entry));
|
||||
|
||||
for_each_pending_queue(pqinfo, queue, i) {
|
||||
queue->head = kzalloc((size), GFP_KERNEL);
|
||||
queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
|
||||
if (!queue->head) {
|
||||
ret = -ENOMEM;
|
||||
goto pending_qfail;
|
||||
|
|
|
@ -31,7 +31,7 @@
|
|||
#define MAX_CCPS 32
|
||||
|
||||
/* Limit CCP use to a specifed number of queues per device */
|
||||
static unsigned int nqueues = 0;
|
||||
static unsigned int nqueues;
|
||||
module_param(nqueues, uint, 0444);
|
||||
MODULE_PARM_DESC(nqueues, "Number of queues per CCP (minimum 1; default: all available)");
|
||||
|
||||
|
|
|
@ -22,6 +22,7 @@
|
|||
#include <linux/firmware.h>
|
||||
#include <linux/gfp.h>
|
||||
#include <linux/cpufeature.h>
|
||||
#include <linux/fs.h>
|
||||
|
||||
#include <asm/smp.h>
|
||||
|
||||
|
@ -43,6 +44,14 @@ static int psp_probe_timeout = 5;
|
|||
module_param(psp_probe_timeout, int, 0644);
|
||||
MODULE_PARM_DESC(psp_probe_timeout, " default timeout value, in seconds, during PSP device probe");
|
||||
|
||||
static char *init_ex_path;
|
||||
module_param(init_ex_path, charp, 0444);
|
||||
MODULE_PARM_DESC(init_ex_path, " Path for INIT_EX data; if set try INIT_EX");
|
||||
|
||||
static bool psp_init_on_probe = true;
|
||||
module_param(psp_init_on_probe, bool, 0444);
|
||||
MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it");
|
||||
|
||||
MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */
|
||||
MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */
|
||||
MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */
|
||||
|
@ -58,6 +67,14 @@ static int psp_timeout;
|
|||
#define SEV_ES_TMR_SIZE (1024 * 1024)
|
||||
static void *sev_es_tmr;
|
||||
|
||||
/* INIT_EX NV Storage:
|
||||
* The NV Storage is a 32Kb area and must be 4Kb page aligned. Use the page
|
||||
* allocator to allocate the memory, which will return aligned memory for the
|
||||
* specified allocation order.
|
||||
*/
|
||||
#define NV_LENGTH (32 * 1024)
|
||||
static void *sev_init_ex_buffer;
|
||||
|
||||
static inline bool sev_version_greater_or_equal(u8 maj, u8 min)
|
||||
{
|
||||
struct sev_device *sev = psp_master->sev_data;
|
||||
|
@ -107,6 +124,7 @@ static int sev_cmd_buffer_len(int cmd)
|
|||
{
|
||||
switch (cmd) {
|
||||
case SEV_CMD_INIT: return sizeof(struct sev_data_init);
|
||||
case SEV_CMD_INIT_EX: return sizeof(struct sev_data_init_ex);
|
||||
case SEV_CMD_PLATFORM_STATUS: return sizeof(struct sev_user_data_status);
|
||||
case SEV_CMD_PEK_CSR: return sizeof(struct sev_data_pek_csr);
|
||||
case SEV_CMD_PEK_CERT_IMPORT: return sizeof(struct sev_data_pek_cert_import);
|
||||
|
@ -141,6 +159,112 @@ static int sev_cmd_buffer_len(int cmd)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void *sev_fw_alloc(unsigned long len)
|
||||
{
|
||||
struct page *page;
|
||||
|
||||
page = alloc_pages(GFP_KERNEL, get_order(len));
|
||||
if (!page)
|
||||
return NULL;
|
||||
|
||||
return page_address(page);
|
||||
}
|
||||
|
||||
static int sev_read_init_ex_file(void)
|
||||
{
|
||||
struct sev_device *sev = psp_master->sev_data;
|
||||
struct file *fp;
|
||||
ssize_t nread;
|
||||
|
||||
lockdep_assert_held(&sev_cmd_mutex);
|
||||
|
||||
if (!sev_init_ex_buffer)
|
||||
return -EOPNOTSUPP;
|
||||
|
||||
fp = filp_open(init_ex_path, O_RDONLY, 0);
|
||||
if (IS_ERR(fp)) {
|
||||
int ret = PTR_ERR(fp);
|
||||
|
||||
dev_err(sev->dev,
|
||||
"SEV: could not open %s for read, error %d\n",
|
||||
init_ex_path, ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
nread = kernel_read(fp, sev_init_ex_buffer, NV_LENGTH, NULL);
|
||||
if (nread != NV_LENGTH) {
|
||||
dev_err(sev->dev,
|
||||
"SEV: failed to read %u bytes to non volatile memory area, ret %ld\n",
|
||||
NV_LENGTH, nread);
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
dev_dbg(sev->dev, "SEV: read %ld bytes from NV file\n", nread);
|
||||
filp_close(fp, NULL);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static void sev_write_init_ex_file(void)
|
||||
{
|
||||
struct sev_device *sev = psp_master->sev_data;
|
||||
struct file *fp;
|
||||
loff_t offset = 0;
|
||||
ssize_t nwrite;
|
||||
|
||||
lockdep_assert_held(&sev_cmd_mutex);
|
||||
|
||||
if (!sev_init_ex_buffer)
|
||||
return;
|
||||
|
||||
fp = filp_open(init_ex_path, O_CREAT | O_WRONLY, 0600);
|
||||
if (IS_ERR(fp)) {
|
||||
dev_err(sev->dev,
|
||||
"SEV: could not open file for write, error %ld\n",
|
||||
PTR_ERR(fp));
|
||||
return;
|
||||
}
|
||||
|
||||
nwrite = kernel_write(fp, sev_init_ex_buffer, NV_LENGTH, &offset);
|
||||
vfs_fsync(fp, 0);
|
||||
filp_close(fp, NULL);
|
||||
|
||||
if (nwrite != NV_LENGTH) {
|
||||
dev_err(sev->dev,
|
||||
"SEV: failed to write %u bytes to non volatile memory area, ret %ld\n",
|
||||
NV_LENGTH, nwrite);
|
||||
return;
|
||||
}
|
||||
|
||||
dev_dbg(sev->dev, "SEV: write successful to NV file\n");
|
||||
}
|
||||
|
||||
static void sev_write_init_ex_file_if_required(int cmd_id)
|
||||
{
|
||||
lockdep_assert_held(&sev_cmd_mutex);
|
||||
|
||||
if (!sev_init_ex_buffer)
|
||||
return;
|
||||
|
||||
/*
|
||||
* Only a few platform commands modify the SPI/NV area, but none of the
|
||||
* non-platform commands do. Only INIT(_EX), PLATFORM_RESET, PEK_GEN,
|
||||
* PEK_CERT_IMPORT, and PDH_GEN do.
|
||||
*/
|
||||
switch (cmd_id) {
|
||||
case SEV_CMD_FACTORY_RESET:
|
||||
case SEV_CMD_INIT_EX:
|
||||
case SEV_CMD_PDH_GEN:
|
||||
case SEV_CMD_PEK_CERT_IMPORT:
|
||||
case SEV_CMD_PEK_GEN:
|
||||
break;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
|
||||
sev_write_init_ex_file();
|
||||
}
|
||||
|
||||
static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
|
||||
{
|
||||
struct psp_device *psp = psp_master;
|
||||
|
@ -210,6 +334,8 @@ static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret)
|
|||
dev_dbg(sev->dev, "sev command %#x failed (%#010x)\n",
|
||||
cmd, reg & PSP_CMDRESP_ERR_MASK);
|
||||
ret = -EIO;
|
||||
} else {
|
||||
sev_write_init_ex_file_if_required(cmd);
|
||||
}
|
||||
|
||||
print_hex_dump_debug("(out): ", DUMP_PREFIX_OFFSET, 16, 2, data,
|
||||
|
@ -236,12 +362,59 @@ static int sev_do_cmd(int cmd, void *data, int *psp_ret)
|
|||
return rc;
|
||||
}
|
||||
|
||||
static int __sev_init_locked(int *error)
|
||||
{
|
||||
struct sev_data_init data;
|
||||
|
||||
memset(&data, 0, sizeof(data));
|
||||
if (sev_es_tmr) {
|
||||
/*
|
||||
* Do not include the encryption mask on the physical
|
||||
* address of the TMR (firmware should clear it anyway).
|
||||
*/
|
||||
data.tmr_address = __pa(sev_es_tmr);
|
||||
|
||||
data.flags |= SEV_INIT_FLAGS_SEV_ES;
|
||||
data.tmr_len = SEV_ES_TMR_SIZE;
|
||||
}
|
||||
|
||||
return __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
|
||||
}
|
||||
|
||||
static int __sev_init_ex_locked(int *error)
|
||||
{
|
||||
struct sev_data_init_ex data;
|
||||
int ret;
|
||||
|
||||
memset(&data, 0, sizeof(data));
|
||||
data.length = sizeof(data);
|
||||
data.nv_address = __psp_pa(sev_init_ex_buffer);
|
||||
data.nv_len = NV_LENGTH;
|
||||
|
||||
ret = sev_read_init_ex_file();
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (sev_es_tmr) {
|
||||
/*
|
||||
* Do not include the encryption mask on the physical
|
||||
* address of the TMR (firmware should clear it anyway).
|
||||
*/
|
||||
data.tmr_address = __pa(sev_es_tmr);
|
||||
|
||||
data.flags |= SEV_INIT_FLAGS_SEV_ES;
|
||||
data.tmr_len = SEV_ES_TMR_SIZE;
|
||||
}
|
||||
|
||||
return __sev_do_cmd_locked(SEV_CMD_INIT_EX, &data, error);
|
||||
}
|
||||
|
||||
static int __sev_platform_init_locked(int *error)
|
||||
{
|
||||
struct psp_device *psp = psp_master;
|
||||
struct sev_data_init data;
|
||||
struct sev_device *sev;
|
||||
int rc = 0;
|
||||
int rc, psp_ret;
|
||||
int (*init_function)(int *error);
|
||||
|
||||
if (!psp || !psp->sev_data)
|
||||
return -ENODEV;
|
||||
|
@ -251,22 +424,23 @@ static int __sev_platform_init_locked(int *error)
|
|||
if (sev->state == SEV_STATE_INIT)
|
||||
return 0;
|
||||
|
||||
memset(&data, 0, sizeof(data));
|
||||
if (sev_es_tmr) {
|
||||
u64 tmr_pa;
|
||||
|
||||
init_function = sev_init_ex_buffer ? __sev_init_ex_locked :
|
||||
__sev_init_locked;
|
||||
rc = init_function(&psp_ret);
|
||||
if (rc && psp_ret == SEV_RET_SECURE_DATA_INVALID) {
|
||||
/*
|
||||
* Do not include the encryption mask on the physical
|
||||
* address of the TMR (firmware should clear it anyway).
|
||||
* Initialization command returned an integrity check failure
|
||||
* status code, meaning that firmware load and validation of SEV
|
||||
* related persistent data has failed. Retrying the
|
||||
* initialization function should succeed by replacing the state
|
||||
* with a reset state.
|
||||
*/
|
||||
tmr_pa = __pa(sev_es_tmr);
|
||||
|
||||
data.flags |= SEV_INIT_FLAGS_SEV_ES;
|
||||
data.tmr_address = tmr_pa;
|
||||
data.tmr_len = SEV_ES_TMR_SIZE;
|
||||
dev_dbg(sev->dev, "SEV: retrying INIT command");
|
||||
rc = init_function(&psp_ret);
|
||||
}
|
||||
if (error)
|
||||
*error = psp_ret;
|
||||
|
||||
rc = __sev_do_cmd_locked(SEV_CMD_INIT, &data, error);
|
||||
if (rc)
|
||||
return rc;
|
||||
|
||||
|
@ -280,7 +454,10 @@ static int __sev_platform_init_locked(int *error)
|
|||
|
||||
dev_dbg(sev->dev, "SEV firmware initialized\n");
|
||||
|
||||
return rc;
|
||||
dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
|
||||
sev->api_minor, sev->build);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int sev_platform_init(int *error)
|
||||
|
@ -1034,6 +1211,12 @@ static void sev_firmware_shutdown(struct sev_device *sev)
|
|||
get_order(SEV_ES_TMR_SIZE));
|
||||
sev_es_tmr = NULL;
|
||||
}
|
||||
|
||||
if (sev_init_ex_buffer) {
|
||||
free_pages((unsigned long)sev_init_ex_buffer,
|
||||
get_order(NV_LENGTH));
|
||||
sev_init_ex_buffer = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void sev_dev_destroy(struct psp_device *psp)
|
||||
|
@ -1064,7 +1247,6 @@ EXPORT_SYMBOL_GPL(sev_issue_cmd_external_user);
|
|||
void sev_pci_init(void)
|
||||
{
|
||||
struct sev_device *sev = psp_master->sev_data;
|
||||
struct page *tmr_page;
|
||||
int error, rc;
|
||||
|
||||
if (!sev)
|
||||
|
@ -1079,37 +1261,32 @@ void sev_pci_init(void)
|
|||
sev_update_firmware(sev->dev) == 0)
|
||||
sev_get_api_version();
|
||||
|
||||
/* If an init_ex_path is provided rely on INIT_EX for PSP initialization
|
||||
* instead of INIT.
|
||||
*/
|
||||
if (init_ex_path) {
|
||||
sev_init_ex_buffer = sev_fw_alloc(NV_LENGTH);
|
||||
if (!sev_init_ex_buffer) {
|
||||
dev_err(sev->dev,
|
||||
"SEV: INIT_EX NV memory allocation failed\n");
|
||||
goto err;
|
||||
}
|
||||
}
|
||||
|
||||
/* Obtain the TMR memory area for SEV-ES use */
|
||||
tmr_page = alloc_pages(GFP_KERNEL, get_order(SEV_ES_TMR_SIZE));
|
||||
if (tmr_page) {
|
||||
sev_es_tmr = page_address(tmr_page);
|
||||
} else {
|
||||
sev_es_tmr = NULL;
|
||||
sev_es_tmr = sev_fw_alloc(SEV_ES_TMR_SIZE);
|
||||
if (!sev_es_tmr)
|
||||
dev_warn(sev->dev,
|
||||
"SEV: TMR allocation failed, SEV-ES support unavailable\n");
|
||||
}
|
||||
|
||||
if (!psp_init_on_probe)
|
||||
return;
|
||||
|
||||
/* Initialize the platform */
|
||||
rc = sev_platform_init(&error);
|
||||
if (rc && (error == SEV_RET_SECURE_DATA_INVALID)) {
|
||||
/*
|
||||
* INIT command returned an integrity check failure
|
||||
* status code, meaning that firmware load and
|
||||
* validation of SEV related persistent data has
|
||||
* failed and persistent state has been erased.
|
||||
* Retrying INIT command here should succeed.
|
||||
*/
|
||||
dev_dbg(sev->dev, "SEV: retrying INIT command");
|
||||
rc = sev_platform_init(&error);
|
||||
}
|
||||
|
||||
if (rc) {
|
||||
dev_err(sev->dev, "SEV: failed to INIT error %#x\n", error);
|
||||
return;
|
||||
}
|
||||
|
||||
dev_info(sev->dev, "SEV API:%d.%d build:%d\n", sev->api_major,
|
||||
sev->api_minor, sev->build);
|
||||
if (rc)
|
||||
dev_err(sev->dev, "SEV: failed to INIT error %#x, rc %d\n",
|
||||
error, rc);
|
||||
|
||||
return;
|
||||
|
||||
|
|
|
@ -101,7 +101,6 @@ void cc_req_mgr_fini(struct cc_drvdata *drvdata)
|
|||
dev_dbg(dev, "max_used_sw_slots=%d\n", req_mgr_h->max_used_sw_slots);
|
||||
|
||||
#ifdef COMP_IN_WQ
|
||||
flush_workqueue(req_mgr_h->workq);
|
||||
destroy_workqueue(req_mgr_h->workq);
|
||||
#else
|
||||
/* Kill tasklet */
|
||||
|
|
|
@ -1177,13 +1177,10 @@ static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
|
|||
static void hpre_key_to_big_end(u8 *data, int len)
|
||||
{
|
||||
int i, j;
|
||||
u8 tmp;
|
||||
|
||||
for (i = 0; i < len / 2; i++) {
|
||||
j = len - i - 1;
|
||||
tmp = data[j];
|
||||
data[j] = data[i];
|
||||
data[i] = tmp;
|
||||
swap(data[j], data[i]);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1865,7 +1862,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req,
|
|||
*/
|
||||
if (memcmp(ptr, p, ctx->key_sz) == 0) {
|
||||
dev_err(dev, "gx is p!\n");
|
||||
return -EINVAL;
|
||||
goto err;
|
||||
} else if (memcmp(ptr, p, ctx->key_sz) > 0) {
|
||||
hpre_curve25519_src_modulo_p(ptr);
|
||||
}
|
||||
|
|
|
@ -103,7 +103,7 @@
|
|||
#define HPRE_QM_PM_FLR BIT(11)
|
||||
#define HPRE_QM_SRIOV_FLR BIT(12)
|
||||
|
||||
#define HPRE_SHAPER_TYPE_RATE 128
|
||||
#define HPRE_SHAPER_TYPE_RATE 640
|
||||
#define HPRE_VIA_MSI_DSM 1
|
||||
#define HPRE_SQE_MASK_OFFSET 8
|
||||
#define HPRE_SQE_MASK_LEN 24
|
||||
|
|
|
@ -89,6 +89,10 @@
|
|||
|
||||
#define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1)
|
||||
#define QM_AEQE_TYPE_SHIFT 17
|
||||
#define QM_AEQE_CQN_MASK GENMASK(15, 0)
|
||||
#define QM_CQ_OVERFLOW 0
|
||||
#define QM_EQ_OVERFLOW 1
|
||||
#define QM_CQE_ERROR 2
|
||||
|
||||
#define QM_DOORBELL_CMD_SQ 0
|
||||
#define QM_DOORBELL_CMD_CQ 1
|
||||
|
@ -122,6 +126,8 @@
|
|||
#define QM_CQC_VFT 0x1
|
||||
#define QM_VFT_CFG 0x100060
|
||||
#define QM_VFT_CFG_OP_ENABLE 0x100054
|
||||
#define QM_PM_CTRL 0x100148
|
||||
#define QM_IDLE_DISABLE BIT(9)
|
||||
|
||||
#define QM_VFT_CFG_DATA_L 0x100064
|
||||
#define QM_VFT_CFG_DATA_H 0x100068
|
||||
|
@ -501,10 +507,30 @@ static const char * const qp_s[] = {
|
|||
"none", "init", "start", "stop", "close",
|
||||
};
|
||||
|
||||
static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000,
|
||||
10000, 25000, 50000, 100000};
|
||||
static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16,
|
||||
17, 18, 19};
|
||||
struct qm_typical_qos_table {
|
||||
u32 start;
|
||||
u32 end;
|
||||
u32 val;
|
||||
};
|
||||
|
||||
/* the qos step is 100 */
|
||||
static struct qm_typical_qos_table shaper_cir_s[] = {
|
||||
{100, 100, 4},
|
||||
{200, 200, 3},
|
||||
{300, 500, 2},
|
||||
{600, 1000, 1},
|
||||
{1100, 100000, 0},
|
||||
};
|
||||
|
||||
static struct qm_typical_qos_table shaper_cbs_s[] = {
|
||||
{100, 200, 9},
|
||||
{300, 500, 11},
|
||||
{600, 1000, 12},
|
||||
{1100, 10000, 16},
|
||||
{10100, 25000, 17},
|
||||
{25100, 50000, 18},
|
||||
{50100, 100000, 19}
|
||||
};
|
||||
|
||||
static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new)
|
||||
{
|
||||
|
@ -585,6 +611,75 @@ static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp,
|
|||
return avail;
|
||||
}
|
||||
|
||||
static u32 qm_get_hw_error_status(struct hisi_qm *qm)
|
||||
{
|
||||
return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
|
||||
}
|
||||
|
||||
static u32 qm_get_dev_err_status(struct hisi_qm *qm)
|
||||
{
|
||||
return qm->err_ini->get_dev_hw_err_status(qm);
|
||||
}
|
||||
|
||||
/* Check if the error causes the master ooo block */
|
||||
static int qm_check_dev_error(struct hisi_qm *qm)
|
||||
{
|
||||
u32 val, dev_val;
|
||||
|
||||
if (qm->fun_type == QM_HW_VF)
|
||||
return 0;
|
||||
|
||||
val = qm_get_hw_error_status(qm);
|
||||
dev_val = qm_get_dev_err_status(qm);
|
||||
|
||||
if (qm->ver < QM_HW_V3)
|
||||
return (val & QM_ECC_MBIT) ||
|
||||
(dev_val & qm->err_info.ecc_2bits_mask);
|
||||
|
||||
return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
|
||||
(dev_val & (~qm->err_info.dev_ce_mask));
|
||||
}
|
||||
|
||||
static int qm_wait_reset_finish(struct hisi_qm *qm)
|
||||
{
|
||||
int delay = 0;
|
||||
|
||||
/* All reset requests need to be queued for processing */
|
||||
while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
|
||||
msleep(++delay);
|
||||
if (delay > QM_RESET_WAIT_TIMEOUT)
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int qm_reset_prepare_ready(struct hisi_qm *qm)
|
||||
{
|
||||
struct pci_dev *pdev = qm->pdev;
|
||||
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
|
||||
|
||||
/*
|
||||
* PF and VF on host doesnot support resetting at the
|
||||
* same time on Kunpeng920.
|
||||
*/
|
||||
if (qm->ver < QM_HW_V3)
|
||||
return qm_wait_reset_finish(pf_qm);
|
||||
|
||||
return qm_wait_reset_finish(qm);
|
||||
}
|
||||
|
||||
static void qm_reset_bit_clear(struct hisi_qm *qm)
|
||||
{
|
||||
struct pci_dev *pdev = qm->pdev;
|
||||
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
|
||||
|
||||
if (qm->ver < QM_HW_V3)
|
||||
clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
|
||||
|
||||
clear_bit(QM_RESETTING, &qm->misc_ctl);
|
||||
}
|
||||
|
||||
static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd,
|
||||
u64 base, u16 queue, bool op)
|
||||
{
|
||||
|
@ -707,6 +802,19 @@ static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority)
|
|||
qm->ops->qm_db(qm, qn, cmd, index, priority);
|
||||
}
|
||||
|
||||
static void qm_disable_clock_gate(struct hisi_qm *qm)
|
||||
{
|
||||
u32 val;
|
||||
|
||||
/* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */
|
||||
if (qm->ver < QM_HW_V3)
|
||||
return;
|
||||
|
||||
val = readl(qm->io_base + QM_PM_CTRL);
|
||||
val |= QM_IDLE_DISABLE;
|
||||
writel(val, qm->io_base + QM_PM_CTRL);
|
||||
}
|
||||
|
||||
static int qm_dev_mem_reset(struct hisi_qm *qm)
|
||||
{
|
||||
u32 val;
|
||||
|
@ -899,24 +1007,71 @@ static void qm_set_qp_disable(struct hisi_qp *qp, int offset)
|
|||
mb();
|
||||
}
|
||||
|
||||
static irqreturn_t qm_aeq_irq(int irq, void *data)
|
||||
static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id)
|
||||
{
|
||||
struct hisi_qp *qp = &qm->qp_array[qp_id];
|
||||
|
||||
qm_set_qp_disable(qp, QM_RESET_STOP_TX_OFFSET);
|
||||
hisi_qm_stop_qp(qp);
|
||||
qm_set_qp_disable(qp, QM_RESET_STOP_RX_OFFSET);
|
||||
}
|
||||
|
||||
static void qm_reset_function(struct hisi_qm *qm)
|
||||
{
|
||||
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev));
|
||||
struct device *dev = &qm->pdev->dev;
|
||||
int ret;
|
||||
|
||||
if (qm_check_dev_error(pf_qm))
|
||||
return;
|
||||
|
||||
ret = qm_reset_prepare_ready(qm);
|
||||
if (ret) {
|
||||
dev_err(dev, "reset function not ready\n");
|
||||
return;
|
||||
}
|
||||
|
||||
ret = hisi_qm_stop(qm, QM_FLR);
|
||||
if (ret) {
|
||||
dev_err(dev, "failed to stop qm when reset function\n");
|
||||
goto clear_bit;
|
||||
}
|
||||
|
||||
ret = hisi_qm_start(qm);
|
||||
if (ret)
|
||||
dev_err(dev, "failed to start qm when reset function\n");
|
||||
|
||||
clear_bit:
|
||||
qm_reset_bit_clear(qm);
|
||||
}
|
||||
|
||||
static irqreturn_t qm_aeq_thread(int irq, void *data)
|
||||
{
|
||||
struct hisi_qm *qm = data;
|
||||
struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head;
|
||||
u32 type;
|
||||
|
||||
atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
|
||||
if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
|
||||
return IRQ_NONE;
|
||||
u32 type, qp_id;
|
||||
|
||||
while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) {
|
||||
type = le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT;
|
||||
if (type < ARRAY_SIZE(qm_fifo_overflow))
|
||||
dev_err(&qm->pdev->dev, "%s overflow\n",
|
||||
qm_fifo_overflow[type]);
|
||||
else
|
||||
qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK;
|
||||
|
||||
switch (type) {
|
||||
case QM_EQ_OVERFLOW:
|
||||
dev_err(&qm->pdev->dev, "eq overflow, reset function\n");
|
||||
qm_reset_function(qm);
|
||||
return IRQ_HANDLED;
|
||||
case QM_CQ_OVERFLOW:
|
||||
dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n",
|
||||
qp_id);
|
||||
fallthrough;
|
||||
case QM_CQE_ERROR:
|
||||
qm_disable_qp(qm, qp_id);
|
||||
break;
|
||||
default:
|
||||
dev_err(&qm->pdev->dev, "unknown error type %u\n",
|
||||
type);
|
||||
break;
|
||||
}
|
||||
|
||||
if (qm->status.aeq_head == QM_Q_DEPTH - 1) {
|
||||
qm->status.aeqc_phase = !qm->status.aeqc_phase;
|
||||
|
@ -926,13 +1081,24 @@ static irqreturn_t qm_aeq_irq(int irq, void *data)
|
|||
aeqe++;
|
||||
qm->status.aeq_head++;
|
||||
}
|
||||
|
||||
qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
|
||||
}
|
||||
|
||||
qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
|
||||
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
static irqreturn_t qm_aeq_irq(int irq, void *data)
|
||||
{
|
||||
struct hisi_qm *qm = data;
|
||||
|
||||
atomic64_inc(&qm->debug.dfx.aeq_irq_cnt);
|
||||
if (!readl(qm->io_base + QM_VF_AEQ_INT_SOURCE))
|
||||
return IRQ_NONE;
|
||||
|
||||
return IRQ_WAKE_THREAD;
|
||||
}
|
||||
|
||||
static void qm_irq_unregister(struct hisi_qm *qm)
|
||||
{
|
||||
struct pci_dev *pdev = qm->pdev;
|
||||
|
@ -988,12 +1154,14 @@ static void qm_init_prefetch(struct hisi_qm *qm)
|
|||
}
|
||||
|
||||
/*
|
||||
* acc_shaper_para_calc() Get the IR value by the qos formula, the return value
|
||||
* is the expected qos calculated.
|
||||
* the formula:
|
||||
* IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps
|
||||
*
|
||||
* IR_b * (2 ^ IR_u) * 8
|
||||
* IR(Mbps) * 10 ^ -3 = -------------------------
|
||||
* Tick * (2 ^ IR_s)
|
||||
* IR_b * (2 ^ IR_u) * 8000
|
||||
* IR(Mbps) = -------------------------
|
||||
* Tick * (2 ^ IR_s)
|
||||
*/
|
||||
static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
|
||||
{
|
||||
|
@ -1003,17 +1171,28 @@ static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s)
|
|||
|
||||
static u32 acc_shaper_calc_cbs_s(u32 ir)
|
||||
{
|
||||
int table_size = ARRAY_SIZE(shaper_cbs_s);
|
||||
int i;
|
||||
|
||||
if (ir < typical_qos_val[0])
|
||||
return QM_SHAPER_MIN_CBS_S;
|
||||
|
||||
for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) {
|
||||
if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i])
|
||||
return typical_qos_cbs_s[i - 1];
|
||||
for (i = 0; i < table_size; i++) {
|
||||
if (ir >= shaper_cbs_s[i].start && ir <= shaper_cbs_s[i].end)
|
||||
return shaper_cbs_s[i].val;
|
||||
}
|
||||
|
||||
return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1];
|
||||
return QM_SHAPER_MIN_CBS_S;
|
||||
}
|
||||
|
||||
static u32 acc_shaper_calc_cir_s(u32 ir)
|
||||
{
|
||||
int table_size = ARRAY_SIZE(shaper_cir_s);
|
||||
int i;
|
||||
|
||||
for (i = 0; i < table_size; i++) {
|
||||
if (ir >= shaper_cir_s[i].start && ir <= shaper_cir_s[i].end)
|
||||
return shaper_cir_s[i].val;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
|
||||
|
@ -1022,25 +1201,18 @@ static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor)
|
|||
u32 error_rate;
|
||||
|
||||
factor->cbs_s = acc_shaper_calc_cbs_s(ir);
|
||||
cir_s = acc_shaper_calc_cir_s(ir);
|
||||
|
||||
for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) {
|
||||
for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) {
|
||||
for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) {
|
||||
/** the formula is changed to:
|
||||
* IR_b * (2 ^ IR_u) * DIVISOR_CLK
|
||||
* IR(Mbps) = -------------------------
|
||||
* 768 * (2 ^ IR_s)
|
||||
*/
|
||||
ir_calc = acc_shaper_para_calc(cir_b, cir_u,
|
||||
cir_s);
|
||||
error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
|
||||
if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
|
||||
factor->cir_b = cir_b;
|
||||
factor->cir_u = cir_u;
|
||||
factor->cir_s = cir_s;
|
||||
ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s);
|
||||
|
||||
return 0;
|
||||
}
|
||||
error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir;
|
||||
if (error_rate <= QM_QOS_MIN_ERROR_RATE) {
|
||||
factor->cir_b = cir_b;
|
||||
factor->cir_u = cir_u;
|
||||
factor->cir_s = cir_s;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1126,10 +1298,10 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type,
|
|||
|
||||
static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num)
|
||||
{
|
||||
u32 qos = qm->factor[fun_num].func_qos;
|
||||
int ret, i;
|
||||
|
||||
qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL;
|
||||
ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]);
|
||||
ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]);
|
||||
if (ret) {
|
||||
dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n");
|
||||
return ret;
|
||||
|
@ -2082,35 +2254,6 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm)
|
|||
return ACC_ERR_RECOVERED;
|
||||
}
|
||||
|
||||
static u32 qm_get_hw_error_status(struct hisi_qm *qm)
|
||||
{
|
||||
return readl(qm->io_base + QM_ABNORMAL_INT_STATUS);
|
||||
}
|
||||
|
||||
static u32 qm_get_dev_err_status(struct hisi_qm *qm)
|
||||
{
|
||||
return qm->err_ini->get_dev_hw_err_status(qm);
|
||||
}
|
||||
|
||||
/* Check if the error causes the master ooo block */
|
||||
static int qm_check_dev_error(struct hisi_qm *qm)
|
||||
{
|
||||
u32 val, dev_val;
|
||||
|
||||
if (qm->fun_type == QM_HW_VF)
|
||||
return 0;
|
||||
|
||||
val = qm_get_hw_error_status(qm);
|
||||
dev_val = qm_get_dev_err_status(qm);
|
||||
|
||||
if (qm->ver < QM_HW_V3)
|
||||
return (val & QM_ECC_MBIT) ||
|
||||
(dev_val & qm->err_info.ecc_2bits_mask);
|
||||
|
||||
return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) ||
|
||||
(dev_val & (~qm->err_info.dev_ce_mask));
|
||||
}
|
||||
|
||||
static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num)
|
||||
{
|
||||
struct qm_mailbox mailbox;
|
||||
|
@ -3399,6 +3542,7 @@ void hisi_qm_uninit(struct hisi_qm *qm)
|
|||
dma_free_coherent(dev, qm->qdma.size,
|
||||
qm->qdma.va, qm->qdma.dma);
|
||||
}
|
||||
up_write(&qm->qps_lock);
|
||||
|
||||
qm_irq_unregister(qm);
|
||||
hisi_qm_pci_uninit(qm);
|
||||
|
@ -3406,8 +3550,6 @@ void hisi_qm_uninit(struct hisi_qm *qm)
|
|||
uacce_remove(qm->uacce);
|
||||
qm->uacce = NULL;
|
||||
}
|
||||
|
||||
up_write(&qm->qps_lock);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(hisi_qm_uninit);
|
||||
|
||||
|
@ -3473,6 +3615,22 @@ static void qm_init_eq_aeq_status(struct hisi_qm *qm)
|
|||
status->aeqc_phase = true;
|
||||
}
|
||||
|
||||
static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm)
|
||||
{
|
||||
/* Clear eq/aeq interrupt source */
|
||||
qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0);
|
||||
qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0);
|
||||
|
||||
writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
|
||||
writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
|
||||
}
|
||||
|
||||
static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm)
|
||||
{
|
||||
writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
|
||||
writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
|
||||
}
|
||||
|
||||
static int qm_eq_ctx_cfg(struct hisi_qm *qm)
|
||||
{
|
||||
struct device *dev = &qm->pdev->dev;
|
||||
|
@ -3556,10 +3714,6 @@ static int __hisi_qm_start(struct hisi_qm *qm)
|
|||
WARN_ON(!qm->qdma.va);
|
||||
|
||||
if (qm->fun_type == QM_HW_PF) {
|
||||
ret = qm_dev_mem_reset(qm);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
@ -3578,9 +3732,7 @@ static int __hisi_qm_start(struct hisi_qm *qm)
|
|||
return ret;
|
||||
|
||||
qm_init_prefetch(qm);
|
||||
|
||||
writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK);
|
||||
writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK);
|
||||
qm_enable_eq_aeq_interrupts(qm);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
@ -3728,10 +3880,7 @@ int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r)
|
|||
hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET);
|
||||
}
|
||||
|
||||
/* Mask eq and aeq irq */
|
||||
writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK);
|
||||
writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK);
|
||||
|
||||
qm_disable_eq_aeq_interrupts(qm);
|
||||
if (qm->fun_type == QM_HW_PF) {
|
||||
ret = hisi_qm_set_vft(qm, 0, 0, 0);
|
||||
if (ret < 0) {
|
||||
|
@ -4231,66 +4380,69 @@ static ssize_t qm_qos_value_init(const char *buf, unsigned long *val)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf,
|
||||
unsigned long *val,
|
||||
unsigned int *fun_index)
|
||||
{
|
||||
char tbuf_bdf[QM_DBG_READ_LEN] = {0};
|
||||
char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
|
||||
u32 tmp1, device, function;
|
||||
int ret, bus;
|
||||
|
||||
ret = sscanf(buf, "%s %s", tbuf_bdf, val_buf);
|
||||
if (ret != QM_QOS_PARAM_NUM)
|
||||
return -EINVAL;
|
||||
|
||||
ret = qm_qos_value_init(val_buf, val);
|
||||
if (ret || *val == 0 || *val > QM_QOS_MAX_VAL) {
|
||||
pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
ret = sscanf(tbuf_bdf, "%u:%x:%u.%u", &tmp1, &bus, &device, &function);
|
||||
if (ret != QM_QOS_BDF_PARAM_NUM) {
|
||||
pci_err(qm->pdev, "input pci bdf value is error!\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
*fun_index = PCI_DEVFN(device, function);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
|
||||
size_t count, loff_t *pos)
|
||||
{
|
||||
struct hisi_qm *qm = filp->private_data;
|
||||
char tbuf[QM_DBG_READ_LEN];
|
||||
int tmp1, bus, device, function;
|
||||
char tbuf_bdf[QM_DBG_READ_LEN] = {0};
|
||||
char val_buf[QM_QOS_VAL_MAX_LEN] = {0};
|
||||
unsigned int fun_index;
|
||||
unsigned long val = 0;
|
||||
unsigned long val;
|
||||
int len, ret;
|
||||
|
||||
if (qm->fun_type == QM_HW_VF)
|
||||
return -EINVAL;
|
||||
|
||||
if (*pos != 0)
|
||||
return 0;
|
||||
|
||||
if (count >= QM_DBG_READ_LEN)
|
||||
return -ENOSPC;
|
||||
|
||||
len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
|
||||
if (len < 0)
|
||||
return len;
|
||||
|
||||
tbuf[len] = '\0';
|
||||
ret = qm_get_qos_value(qm, tbuf, &val, &fun_index);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/* Mailbox and reset cannot be operated at the same time */
|
||||
if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
|
||||
pci_err(qm->pdev, "dev resetting, write alg qos failed!\n");
|
||||
return -EAGAIN;
|
||||
}
|
||||
|
||||
if (*pos != 0) {
|
||||
ret = 0;
|
||||
goto err_get_status;
|
||||
}
|
||||
|
||||
if (count >= QM_DBG_READ_LEN) {
|
||||
ret = -ENOSPC;
|
||||
goto err_get_status;
|
||||
}
|
||||
|
||||
len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count);
|
||||
if (len < 0) {
|
||||
ret = len;
|
||||
goto err_get_status;
|
||||
}
|
||||
|
||||
tbuf[len] = '\0';
|
||||
ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf);
|
||||
if (ret != QM_QOS_PARAM_NUM) {
|
||||
ret = -EINVAL;
|
||||
goto err_get_status;
|
||||
}
|
||||
|
||||
ret = qm_qos_value_init(val_buf, &val);
|
||||
if (val == 0 || val > QM_QOS_MAX_VAL || ret) {
|
||||
pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n");
|
||||
ret = -EINVAL;
|
||||
goto err_get_status;
|
||||
}
|
||||
|
||||
ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function);
|
||||
if (ret != QM_QOS_BDF_PARAM_NUM) {
|
||||
pci_err(qm->pdev, "input pci bdf value is error!\n");
|
||||
ret = -EINVAL;
|
||||
goto err_get_status;
|
||||
}
|
||||
|
||||
fun_index = device * 8 + function;
|
||||
|
||||
ret = qm_pm_get_sync(qm);
|
||||
if (ret) {
|
||||
ret = -EINVAL;
|
||||
|
@ -4304,6 +4456,8 @@ static ssize_t qm_algqos_write(struct file *filp, const char __user *buf,
|
|||
goto err_put_sync;
|
||||
}
|
||||
|
||||
pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n",
|
||||
fun_index, val);
|
||||
ret = count;
|
||||
|
||||
err_put_sync:
|
||||
|
@ -4728,46 +4882,6 @@ static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd,
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int qm_wait_reset_finish(struct hisi_qm *qm)
|
||||
{
|
||||
int delay = 0;
|
||||
|
||||
/* All reset requests need to be queued for processing */
|
||||
while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) {
|
||||
msleep(++delay);
|
||||
if (delay > QM_RESET_WAIT_TIMEOUT)
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int qm_reset_prepare_ready(struct hisi_qm *qm)
|
||||
{
|
||||
struct pci_dev *pdev = qm->pdev;
|
||||
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
|
||||
|
||||
/*
|
||||
* PF and VF on host doesnot support resetting at the
|
||||
* same time on Kunpeng920.
|
||||
*/
|
||||
if (qm->ver < QM_HW_V3)
|
||||
return qm_wait_reset_finish(pf_qm);
|
||||
|
||||
return qm_wait_reset_finish(qm);
|
||||
}
|
||||
|
||||
static void qm_reset_bit_clear(struct hisi_qm *qm)
|
||||
{
|
||||
struct pci_dev *pdev = qm->pdev;
|
||||
struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev));
|
||||
|
||||
if (qm->ver < QM_HW_V3)
|
||||
clear_bit(QM_RESETTING, &pf_qm->misc_ctl);
|
||||
|
||||
clear_bit(QM_RESETTING, &qm->misc_ctl);
|
||||
}
|
||||
|
||||
static int qm_controller_reset_prepare(struct hisi_qm *qm)
|
||||
{
|
||||
struct pci_dev *pdev = qm->pdev;
|
||||
|
@ -5053,6 +5167,12 @@ static int qm_controller_reset_done(struct hisi_qm *qm)
|
|||
if (qm->err_ini->open_axi_master_ooo)
|
||||
qm->err_ini->open_axi_master_ooo(qm);
|
||||
|
||||
ret = qm_dev_mem_reset(qm);
|
||||
if (ret) {
|
||||
pci_err(pdev, "failed to reset device memory\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = qm_restart(qm);
|
||||
if (ret) {
|
||||
pci_err(pdev, "Failed to start QM!\n");
|
||||
|
@ -5267,8 +5387,10 @@ static int qm_irq_register(struct hisi_qm *qm)
|
|||
return ret;
|
||||
|
||||
if (qm->ver > QM_HW_V1) {
|
||||
ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR),
|
||||
qm_aeq_irq, 0, qm->dev_name, qm);
|
||||
ret = request_threaded_irq(pci_irq_vector(pdev,
|
||||
QM_AEQ_EVENT_IRQ_VECTOR),
|
||||
qm_aeq_irq, qm_aeq_thread,
|
||||
0, qm->dev_name, qm);
|
||||
if (ret)
|
||||
goto err_aeq_irq;
|
||||
|
||||
|
@ -5750,13 +5872,15 @@ err_init_qp_mem:
|
|||
static int hisi_qm_memory_init(struct hisi_qm *qm)
|
||||
{
|
||||
struct device *dev = &qm->pdev->dev;
|
||||
int ret, total_vfs;
|
||||
int ret, total_func, i;
|
||||
size_t off = 0;
|
||||
|
||||
total_vfs = pci_sriov_get_totalvfs(qm->pdev);
|
||||
qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL);
|
||||
total_func = pci_sriov_get_totalvfs(qm->pdev) + 1;
|
||||
qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL);
|
||||
if (!qm->factor)
|
||||
return -ENOMEM;
|
||||
for (i = 0; i < total_func; i++)
|
||||
qm->factor[i].func_qos = QM_QOS_MAX_VAL;
|
||||
|
||||
#define QM_INIT_BUF(qm, type, num) do { \
|
||||
(qm)->type = ((qm)->qdma.va + (off)); \
|
||||
|
@ -5825,6 +5949,15 @@ int hisi_qm_init(struct hisi_qm *qm)
|
|||
goto err_irq_register;
|
||||
}
|
||||
|
||||
if (qm->fun_type == QM_HW_PF) {
|
||||
qm_disable_clock_gate(qm);
|
||||
ret = qm_dev_mem_reset(qm);
|
||||
if (ret) {
|
||||
dev_err(dev, "failed to reset device memory\n");
|
||||
goto err_irq_register;
|
||||
}
|
||||
}
|
||||
|
||||
if (qm->mode == UACCE_MODE_SVA) {
|
||||
ret = qm_alloc_uacce(qm);
|
||||
if (ret < 0)
|
||||
|
@ -5982,8 +6115,12 @@ static int qm_rebuild_for_resume(struct hisi_qm *qm)
|
|||
|
||||
qm_cmd_init(qm);
|
||||
hisi_qm_dev_err_init(qm);
|
||||
qm_disable_clock_gate(qm);
|
||||
ret = qm_dev_mem_reset(qm);
|
||||
if (ret)
|
||||
pci_err(pdev, "failed to reset device memory\n");
|
||||
|
||||
return 0;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -6038,7 +6175,7 @@ int hisi_qm_resume(struct device *dev)
|
|||
if (ret)
|
||||
pci_err(pdev, "failed to start qm(%d)\n", ret);
|
||||
|
||||
return 0;
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(hisi_qm_resume);
|
||||
|
||||
|
|
|
@ -105,7 +105,7 @@
|
|||
|
||||
#define SEC_SQE_MASK_OFFSET 64
|
||||
#define SEC_SQE_MASK_LEN 48
|
||||
#define SEC_SHAPER_TYPE_RATE 128
|
||||
#define SEC_SHAPER_TYPE_RATE 400
|
||||
|
||||
struct sec_hw_error {
|
||||
u32 int_msk;
|
||||
|
|
|
@ -103,8 +103,8 @@
|
|||
#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0)))
|
||||
#define HZIP_SVA_PREFETCH_DISABLE BIT(26)
|
||||
#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30))
|
||||
#define HZIP_SHAPER_RATE_COMPRESS 252
|
||||
#define HZIP_SHAPER_RATE_DECOMPRESS 229
|
||||
#define HZIP_SHAPER_RATE_COMPRESS 750
|
||||
#define HZIP_SHAPER_RATE_DECOMPRESS 140
|
||||
#define HZIP_DELAY_1_US 1
|
||||
#define HZIP_POLL_TIMEOUT_US 1000
|
||||
|
||||
|
@ -364,15 +364,16 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm)
|
|||
|
||||
/* user domain configurations */
|
||||
writel(AXUSER_BASE, base + HZIP_BD_RUSER_32_63);
|
||||
writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
|
||||
writel(AXUSER_BASE, base + HZIP_BD_WUSER_32_63);
|
||||
|
||||
if (qm->use_sva && qm->ver == QM_HW_V2) {
|
||||
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_RUSER_32_63);
|
||||
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_DATA_WUSER_32_63);
|
||||
writel(AXUSER_BASE | AXUSER_SSV, base + HZIP_SGL_RUSER_32_63);
|
||||
} else {
|
||||
writel(AXUSER_BASE, base + HZIP_DATA_RUSER_32_63);
|
||||
writel(AXUSER_BASE, base + HZIP_DATA_WUSER_32_63);
|
||||
writel(AXUSER_BASE, base + HZIP_SGL_RUSER_32_63);
|
||||
}
|
||||
|
||||
/* let's open all compression/decompression cores */
|
||||
|
@ -829,7 +830,10 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev)
|
|||
|
||||
qm->pdev = pdev;
|
||||
qm->ver = pdev->revision;
|
||||
qm->algs = "zlib\ngzip";
|
||||
if (pdev->revision >= QM_HW_V3)
|
||||
qm->algs = "zlib\ngzip\ndeflate\nlz77_zstd";
|
||||
else
|
||||
qm->algs = "zlib\ngzip";
|
||||
qm->mode = uacce_mode;
|
||||
qm->sqe_size = HZIP_SQE_SIZE;
|
||||
qm->dev_name = hisi_zip_name;
|
||||
|
|
|
@ -930,6 +930,7 @@ static int kmb_ocs_ecc_probe(struct platform_device *pdev)
|
|||
ecc_dev->engine = crypto_engine_alloc_init(dev, 1);
|
||||
if (!ecc_dev->engine) {
|
||||
dev_err(dev, "Could not allocate crypto engine\n");
|
||||
rc = -ENOMEM;
|
||||
goto list_del;
|
||||
}
|
||||
|
||||
|
|
|
@ -94,15 +94,13 @@ static int alloc_pending_queues(struct otx_cpt_pending_qinfo *pqinfo, u32 qlen,
|
|||
u32 num_queues)
|
||||
{
|
||||
struct otx_cpt_pending_queue *queue = NULL;
|
||||
size_t size;
|
||||
int ret;
|
||||
u32 i;
|
||||
|
||||
pqinfo->num_queues = num_queues;
|
||||
size = (qlen * sizeof(struct otx_cpt_pending_entry));
|
||||
|
||||
for_each_pending_queue(pqinfo, queue, i) {
|
||||
queue->head = kzalloc((size), GFP_KERNEL);
|
||||
queue->head = kcalloc(qlen, sizeof(*queue->head), GFP_KERNEL);
|
||||
if (!queue->head) {
|
||||
ret = -ENOMEM;
|
||||
goto pending_qfail;
|
||||
|
|
|
@ -3,7 +3,7 @@ obj-$(CONFIG_CRYPTO_DEV_OCTEONTX2_CPT) += rvu_cptpf.o rvu_cptvf.o
|
|||
|
||||
rvu_cptpf-objs := otx2_cptpf_main.o otx2_cptpf_mbox.o \
|
||||
otx2_cpt_mbox_common.o otx2_cptpf_ucode.o otx2_cptlf.o \
|
||||
cn10k_cpt.o
|
||||
cn10k_cpt.o otx2_cpt_devlink.o
|
||||
rvu_cptvf-objs := otx2_cptvf_main.o otx2_cptvf_mbox.o otx2_cptlf.o \
|
||||
otx2_cpt_mbox_common.o otx2_cptvf_reqmgr.o \
|
||||
otx2_cptvf_algs.o cn10k_cpt.o
|
||||
|
|
|
@ -10,6 +10,7 @@
|
|||
#include <linux/module.h>
|
||||
#include <linux/delay.h>
|
||||
#include <linux/crypto.h>
|
||||
#include <net/devlink.h>
|
||||
#include "otx2_cpt_hw_types.h"
|
||||
#include "rvu.h"
|
||||
#include "mbox.h"
|
||||
|
|
108
drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c
Normal file
108
drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.c
Normal file
|
@ -0,0 +1,108 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-only
|
||||
/* Copyright (C) 2021 Marvell. */
|
||||
|
||||
#include "otx2_cpt_devlink.h"
|
||||
|
||||
static int otx2_cpt_dl_egrp_create(struct devlink *dl, u32 id,
|
||||
struct devlink_param_gset_ctx *ctx)
|
||||
{
|
||||
struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
|
||||
struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
|
||||
|
||||
return otx2_cpt_dl_custom_egrp_create(cptpf, ctx);
|
||||
}
|
||||
|
||||
static int otx2_cpt_dl_egrp_delete(struct devlink *dl, u32 id,
|
||||
struct devlink_param_gset_ctx *ctx)
|
||||
{
|
||||
struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
|
||||
struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
|
||||
|
||||
return otx2_cpt_dl_custom_egrp_delete(cptpf, ctx);
|
||||
}
|
||||
|
||||
static int otx2_cpt_dl_uc_info(struct devlink *dl, u32 id,
|
||||
struct devlink_param_gset_ctx *ctx)
|
||||
{
|
||||
struct otx2_cpt_devlink *cpt_dl = devlink_priv(dl);
|
||||
struct otx2_cptpf_dev *cptpf = cpt_dl->cptpf;
|
||||
|
||||
otx2_cpt_print_uc_dbg_info(cptpf);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
enum otx2_cpt_dl_param_id {
|
||||
OTX2_CPT_DEVLINK_PARAM_ID_BASE = DEVLINK_PARAM_GENERIC_ID_MAX,
|
||||
OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE,
|
||||
OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE,
|
||||
};
|
||||
|
||||
static const struct devlink_param otx2_cpt_dl_params[] = {
|
||||
DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_CREATE,
|
||||
"egrp_create", DEVLINK_PARAM_TYPE_STRING,
|
||||
BIT(DEVLINK_PARAM_CMODE_RUNTIME),
|
||||
otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_create,
|
||||
NULL),
|
||||
DEVLINK_PARAM_DRIVER(OTX2_CPT_DEVLINK_PARAM_ID_EGRP_DELETE,
|
||||
"egrp_delete", DEVLINK_PARAM_TYPE_STRING,
|
||||
BIT(DEVLINK_PARAM_CMODE_RUNTIME),
|
||||
otx2_cpt_dl_uc_info, otx2_cpt_dl_egrp_delete,
|
||||
NULL),
|
||||
};
|
||||
|
||||
static int otx2_cpt_devlink_info_get(struct devlink *devlink,
|
||||
struct devlink_info_req *req,
|
||||
struct netlink_ext_ack *extack)
|
||||
{
|
||||
return devlink_info_driver_name_put(req, "rvu_cptpf");
|
||||
}
|
||||
|
||||
static const struct devlink_ops otx2_cpt_devlink_ops = {
|
||||
.info_get = otx2_cpt_devlink_info_get,
|
||||
};
|
||||
|
||||
int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf)
|
||||
{
|
||||
struct device *dev = &cptpf->pdev->dev;
|
||||
struct otx2_cpt_devlink *cpt_dl;
|
||||
struct devlink *dl;
|
||||
int ret;
|
||||
|
||||
dl = devlink_alloc(&otx2_cpt_devlink_ops,
|
||||
sizeof(struct otx2_cpt_devlink), dev);
|
||||
if (!dl) {
|
||||
dev_warn(dev, "devlink_alloc failed\n");
|
||||
return -ENOMEM;
|
||||
}
|
||||
|
||||
cpt_dl = devlink_priv(dl);
|
||||
cpt_dl->dl = dl;
|
||||
cpt_dl->cptpf = cptpf;
|
||||
cptpf->dl = dl;
|
||||
ret = devlink_params_register(dl, otx2_cpt_dl_params,
|
||||
ARRAY_SIZE(otx2_cpt_dl_params));
|
||||
if (ret) {
|
||||
dev_err(dev, "devlink params register failed with error %d",
|
||||
ret);
|
||||
devlink_free(dl);
|
||||
return ret;
|
||||
}
|
||||
|
||||
devlink_register(dl);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf)
|
||||
{
|
||||
struct devlink *dl = cptpf->dl;
|
||||
|
||||
if (!dl)
|
||||
return;
|
||||
|
||||
devlink_unregister(dl);
|
||||
devlink_params_unregister(dl, otx2_cpt_dl_params,
|
||||
ARRAY_SIZE(otx2_cpt_dl_params));
|
||||
devlink_free(dl);
|
||||
}
|
20
drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h
Normal file
20
drivers/crypto/marvell/octeontx2/otx2_cpt_devlink.h
Normal file
|
@ -0,0 +1,20 @@
|
|||
/* SPDX-License-Identifier: GPL-2.0-only
|
||||
* Copyright (C) 2021 Marvell.
|
||||
*/
|
||||
|
||||
#ifndef __OTX2_CPT_DEVLINK_H
|
||||
#define __OTX2_CPT_DEVLINK_H
|
||||
|
||||
#include "otx2_cpt_common.h"
|
||||
#include "otx2_cptpf.h"
|
||||
|
||||
struct otx2_cpt_devlink {
|
||||
struct devlink *dl;
|
||||
struct otx2_cptpf_dev *cptpf;
|
||||
};
|
||||
|
||||
/* Devlink APIs */
|
||||
int otx2_cpt_register_dl(struct otx2_cptpf_dev *cptpf);
|
||||
void otx2_cpt_unregister_dl(struct otx2_cptpf_dev *cptpf);
|
||||
|
||||
#endif /* __OTX2_CPT_DEVLINK_H */
|
|
@ -53,6 +53,9 @@ struct otx2_cptpf_dev {
|
|||
u8 enabled_vfs; /* Number of enabled VFs */
|
||||
u8 kvf_limits; /* Kernel crypto limits */
|
||||
bool has_cpt1;
|
||||
|
||||
/* Devlink */
|
||||
struct devlink *dl;
|
||||
};
|
||||
|
||||
irqreturn_t otx2_cptpf_afpf_mbox_intr(int irq, void *arg);
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
#include <linux/firmware.h>
|
||||
#include "otx2_cpt_hw_types.h"
|
||||
#include "otx2_cpt_common.h"
|
||||
#include "otx2_cpt_devlink.h"
|
||||
#include "otx2_cptpf_ucode.h"
|
||||
#include "otx2_cptpf.h"
|
||||
#include "cn10k_cpt.h"
|
||||
|
@ -494,12 +495,11 @@ static ssize_t kvf_limits_store(struct device *dev,
|
|||
{
|
||||
struct otx2_cptpf_dev *cptpf = dev_get_drvdata(dev);
|
||||
int lfs_num;
|
||||
int ret;
|
||||
|
||||
if (kstrtoint(buf, 0, &lfs_num)) {
|
||||
dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
|
||||
lfs_num, num_online_cpus());
|
||||
return -EINVAL;
|
||||
}
|
||||
ret = kstrtoint(buf, 0, &lfs_num);
|
||||
if (ret)
|
||||
return ret;
|
||||
if (lfs_num < 1 || lfs_num > num_online_cpus()) {
|
||||
dev_err(dev, "lfs count %d must be in range [1 - %d]\n",
|
||||
lfs_num, num_online_cpus());
|
||||
|
@ -767,8 +767,15 @@ static int otx2_cptpf_probe(struct pci_dev *pdev,
|
|||
err = sysfs_create_group(&dev->kobj, &cptpf_sysfs_group);
|
||||
if (err)
|
||||
goto cleanup_eng_grps;
|
||||
|
||||
err = otx2_cpt_register_dl(cptpf);
|
||||
if (err)
|
||||
goto sysfs_grp_del;
|
||||
|
||||
return 0;
|
||||
|
||||
sysfs_grp_del:
|
||||
sysfs_remove_group(&dev->kobj, &cptpf_sysfs_group);
|
||||
cleanup_eng_grps:
|
||||
otx2_cpt_cleanup_eng_grps(pdev, &cptpf->eng_grps);
|
||||
unregister_intr:
|
||||
|
@ -788,6 +795,7 @@ static void otx2_cptpf_remove(struct pci_dev *pdev)
|
|||
return;
|
||||
|
||||
cptpf_sriov_disable(pdev);
|
||||
otx2_cpt_unregister_dl(cptpf);
|
||||
/* Delete sysfs entry created for kernel VF limits */
|
||||
sysfs_remove_group(&pdev->dev.kobj, &cptpf_sysfs_group);
|
||||
/* Cleanup engine groups */
|
||||
|
|
|
@ -29,7 +29,8 @@ static struct otx2_cpt_bitmap get_cores_bmap(struct device *dev,
|
|||
bool found = false;
|
||||
int i;
|
||||
|
||||
if (eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
|
||||
if (eng_grp->g->engs_num < 0 ||
|
||||
eng_grp->g->engs_num > OTX2_CPT_MAX_ENGINES) {
|
||||
dev_err(dev, "unsupported number of engines %d on octeontx2\n",
|
||||
eng_grp->g->engs_num);
|
||||
return bmap;
|
||||
|
@ -1110,18 +1111,19 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
|
|||
struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { {0} };
|
||||
struct pci_dev *pdev = cptpf->pdev;
|
||||
struct fw_info_t fw_info;
|
||||
int ret;
|
||||
int ret = 0;
|
||||
|
||||
mutex_lock(&eng_grps->lock);
|
||||
/*
|
||||
* We don't create engine groups if it was already
|
||||
* made (when user enabled VFs for the first time)
|
||||
*/
|
||||
if (eng_grps->is_grps_created)
|
||||
return 0;
|
||||
goto unlock;
|
||||
|
||||
ret = cpt_ucode_load_fw(pdev, &fw_info);
|
||||
if (ret)
|
||||
return ret;
|
||||
goto unlock;
|
||||
|
||||
/*
|
||||
* Create engine group with SE engines for kernel
|
||||
|
@ -1186,7 +1188,7 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
|
|||
cpt_ucode_release_fw(&fw_info);
|
||||
|
||||
if (is_dev_otx2(pdev))
|
||||
return 0;
|
||||
goto unlock;
|
||||
/*
|
||||
* Configure engine group mask to allow context prefetching
|
||||
* for the groups.
|
||||
|
@ -1201,12 +1203,15 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
|
|||
*/
|
||||
otx2_cpt_write_af_reg(&cptpf->afpf_mbox, pdev, CPT_AF_CTX_FLUSH_TIMER,
|
||||
CTX_FLUSH_TIMER_CNT, BLKADDR_CPT0);
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
return 0;
|
||||
|
||||
delete_eng_grp:
|
||||
delete_engine_grps(pdev, eng_grps);
|
||||
release_fw:
|
||||
cpt_ucode_release_fw(&fw_info);
|
||||
unlock:
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
@ -1286,6 +1291,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
|
|||
struct otx2_cpt_eng_grp_info *grp;
|
||||
int i, j;
|
||||
|
||||
mutex_lock(&eng_grps->lock);
|
||||
delete_engine_grps(pdev, eng_grps);
|
||||
/* Release memory */
|
||||
for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
|
||||
|
@ -1295,6 +1301,7 @@ void otx2_cpt_cleanup_eng_grps(struct pci_dev *pdev,
|
|||
grp->engs[j].bmap = NULL;
|
||||
}
|
||||
}
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
}
|
||||
|
||||
int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
|
||||
|
@ -1303,6 +1310,7 @@ int otx2_cpt_init_eng_grps(struct pci_dev *pdev,
|
|||
struct otx2_cpt_eng_grp_info *grp;
|
||||
int i, j, ret;
|
||||
|
||||
mutex_init(&eng_grps->lock);
|
||||
eng_grps->obj = pci_get_drvdata(pdev);
|
||||
eng_grps->avail.se_cnt = eng_grps->avail.max_se_cnt;
|
||||
eng_grps->avail.ie_cnt = eng_grps->avail.max_ie_cnt;
|
||||
|
@ -1349,11 +1357,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
|
|||
struct fw_info_t fw_info;
|
||||
int ret;
|
||||
|
||||
mutex_lock(&eng_grps->lock);
|
||||
ret = cpt_ucode_load_fw(pdev, &fw_info);
|
||||
if (ret)
|
||||
if (ret) {
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
uc_info[0] = get_ucode(&fw_info, OTX2_CPT_SE_TYPES);
|
||||
uc_info[0] = get_ucode(&fw_info, OTX2_CPT_AE_TYPES);
|
||||
if (uc_info[0] == NULL) {
|
||||
dev_err(&pdev->dev, "Unable to find firmware for AE\n");
|
||||
ret = -EINVAL;
|
||||
|
@ -1396,12 +1407,14 @@ static int create_eng_caps_discovery_grps(struct pci_dev *pdev,
|
|||
goto delete_eng_grp;
|
||||
|
||||
cpt_ucode_release_fw(&fw_info);
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
return 0;
|
||||
|
||||
delete_eng_grp:
|
||||
delete_engine_grps(pdev, eng_grps);
|
||||
release_fw:
|
||||
cpt_ucode_release_fw(&fw_info);
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
@ -1501,3 +1514,291 @@ delete_grps:
|
|||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
|
||||
struct devlink_param_gset_ctx *ctx)
|
||||
{
|
||||
struct otx2_cpt_engines engs[OTX2_CPT_MAX_ETYPES_PER_GRP] = { { 0 } };
|
||||
struct otx2_cpt_uc_info_t *uc_info[OTX2_CPT_MAX_ETYPES_PER_GRP] = {};
|
||||
struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
|
||||
char *ucode_filename[OTX2_CPT_MAX_ETYPES_PER_GRP];
|
||||
char tmp_buf[OTX2_CPT_NAME_LENGTH] = { 0 };
|
||||
struct device *dev = &cptpf->pdev->dev;
|
||||
char *start, *val, *err_msg, *tmp;
|
||||
int grp_idx = 0, ret = -EINVAL;
|
||||
bool has_se, has_ie, has_ae;
|
||||
struct fw_info_t fw_info;
|
||||
int ucode_idx = 0;
|
||||
|
||||
if (!eng_grps->is_grps_created) {
|
||||
dev_err(dev, "Not allowed before creating the default groups\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
err_msg = "Invalid engine group format";
|
||||
strscpy(tmp_buf, ctx->val.vstr, strlen(ctx->val.vstr) + 1);
|
||||
start = tmp_buf;
|
||||
|
||||
has_se = has_ie = has_ae = false;
|
||||
|
||||
for (;;) {
|
||||
val = strsep(&start, ";");
|
||||
if (!val)
|
||||
break;
|
||||
val = strim(val);
|
||||
if (!*val)
|
||||
continue;
|
||||
|
||||
if (!strncasecmp(val, "se", 2) && strchr(val, ':')) {
|
||||
if (has_se || ucode_idx)
|
||||
goto err_print;
|
||||
tmp = strim(strsep(&val, ":"));
|
||||
if (!val)
|
||||
goto err_print;
|
||||
if (strlen(tmp) != 2)
|
||||
goto err_print;
|
||||
if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
|
||||
goto err_print;
|
||||
engs[grp_idx++].type = OTX2_CPT_SE_TYPES;
|
||||
has_se = true;
|
||||
} else if (!strncasecmp(val, "ae", 2) && strchr(val, ':')) {
|
||||
if (has_ae || ucode_idx)
|
||||
goto err_print;
|
||||
tmp = strim(strsep(&val, ":"));
|
||||
if (!val)
|
||||
goto err_print;
|
||||
if (strlen(tmp) != 2)
|
||||
goto err_print;
|
||||
if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
|
||||
goto err_print;
|
||||
engs[grp_idx++].type = OTX2_CPT_AE_TYPES;
|
||||
has_ae = true;
|
||||
} else if (!strncasecmp(val, "ie", 2) && strchr(val, ':')) {
|
||||
if (has_ie || ucode_idx)
|
||||
goto err_print;
|
||||
tmp = strim(strsep(&val, ":"));
|
||||
if (!val)
|
||||
goto err_print;
|
||||
if (strlen(tmp) != 2)
|
||||
goto err_print;
|
||||
if (kstrtoint(strim(val), 10, &engs[grp_idx].count))
|
||||
goto err_print;
|
||||
engs[grp_idx++].type = OTX2_CPT_IE_TYPES;
|
||||
has_ie = true;
|
||||
} else {
|
||||
if (ucode_idx > 1)
|
||||
goto err_print;
|
||||
if (!strlen(val))
|
||||
goto err_print;
|
||||
if (strnstr(val, " ", strlen(val)))
|
||||
goto err_print;
|
||||
ucode_filename[ucode_idx++] = val;
|
||||
}
|
||||
}
|
||||
|
||||
/* Validate input parameters */
|
||||
if (!(grp_idx && ucode_idx))
|
||||
goto err_print;
|
||||
|
||||
if (ucode_idx > 1 && grp_idx < 2)
|
||||
goto err_print;
|
||||
|
||||
if (grp_idx > OTX2_CPT_MAX_ETYPES_PER_GRP) {
|
||||
err_msg = "Error max 2 engine types can be attached";
|
||||
goto err_print;
|
||||
}
|
||||
|
||||
if (grp_idx > 1) {
|
||||
if ((engs[0].type + engs[1].type) !=
|
||||
(OTX2_CPT_SE_TYPES + OTX2_CPT_IE_TYPES)) {
|
||||
err_msg = "Only combination of SE+IE engines is allowed";
|
||||
goto err_print;
|
||||
}
|
||||
/* Keep SE engines at zero index */
|
||||
if (engs[1].type == OTX2_CPT_SE_TYPES)
|
||||
swap(engs[0], engs[1]);
|
||||
}
|
||||
mutex_lock(&eng_grps->lock);
|
||||
|
||||
if (cptpf->enabled_vfs) {
|
||||
dev_err(dev, "Disable VFs before modifying engine groups\n");
|
||||
ret = -EACCES;
|
||||
goto err_unlock;
|
||||
}
|
||||
INIT_LIST_HEAD(&fw_info.ucodes);
|
||||
ret = load_fw(dev, &fw_info, ucode_filename[0]);
|
||||
if (ret) {
|
||||
dev_err(dev, "Unable to load firmware %s\n", ucode_filename[0]);
|
||||
goto err_unlock;
|
||||
}
|
||||
if (ucode_idx > 1) {
|
||||
ret = load_fw(dev, &fw_info, ucode_filename[1]);
|
||||
if (ret) {
|
||||
dev_err(dev, "Unable to load firmware %s\n",
|
||||
ucode_filename[1]);
|
||||
goto release_fw;
|
||||
}
|
||||
}
|
||||
uc_info[0] = get_ucode(&fw_info, engs[0].type);
|
||||
if (uc_info[0] == NULL) {
|
||||
dev_err(dev, "Unable to find firmware for %s\n",
|
||||
get_eng_type_str(engs[0].type));
|
||||
ret = -EINVAL;
|
||||
goto release_fw;
|
||||
}
|
||||
if (ucode_idx > 1) {
|
||||
uc_info[1] = get_ucode(&fw_info, engs[1].type);
|
||||
if (uc_info[1] == NULL) {
|
||||
dev_err(dev, "Unable to find firmware for %s\n",
|
||||
get_eng_type_str(engs[1].type));
|
||||
ret = -EINVAL;
|
||||
goto release_fw;
|
||||
}
|
||||
}
|
||||
ret = create_engine_group(dev, eng_grps, engs, grp_idx,
|
||||
(void **)uc_info, 1);
|
||||
|
||||
release_fw:
|
||||
cpt_ucode_release_fw(&fw_info);
|
||||
err_unlock:
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
return ret;
|
||||
err_print:
|
||||
dev_err(dev, "%s\n", err_msg);
|
||||
return ret;
|
||||
}
|
||||
|
||||
int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
|
||||
struct devlink_param_gset_ctx *ctx)
|
||||
{
|
||||
struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
|
||||
struct device *dev = &cptpf->pdev->dev;
|
||||
char *tmp, *err_msg;
|
||||
int egrp;
|
||||
int ret;
|
||||
|
||||
err_msg = "Invalid input string format(ex: egrp:0)";
|
||||
if (strncasecmp(ctx->val.vstr, "egrp", 4))
|
||||
goto err_print;
|
||||
tmp = ctx->val.vstr;
|
||||
strsep(&tmp, ":");
|
||||
if (!tmp)
|
||||
goto err_print;
|
||||
if (kstrtoint(tmp, 10, &egrp))
|
||||
goto err_print;
|
||||
|
||||
if (egrp < 0 || egrp >= OTX2_CPT_MAX_ENGINE_GROUPS) {
|
||||
dev_err(dev, "Invalid engine group %d", egrp);
|
||||
return -EINVAL;
|
||||
}
|
||||
if (!eng_grps->grp[egrp].is_enabled) {
|
||||
dev_err(dev, "Error engine_group%d is not configured", egrp);
|
||||
return -EINVAL;
|
||||
}
|
||||
mutex_lock(&eng_grps->lock);
|
||||
ret = delete_engine_group(dev, &eng_grps->grp[egrp]);
|
||||
mutex_unlock(&eng_grps->lock);
|
||||
|
||||
return ret;
|
||||
|
||||
err_print:
|
||||
dev_err(dev, "%s\n", err_msg);
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
static void get_engs_info(struct otx2_cpt_eng_grp_info *eng_grp, char *buf,
|
||||
int size, int idx)
|
||||
{
|
||||
struct otx2_cpt_engs_rsvd *mirrored_engs = NULL;
|
||||
struct otx2_cpt_engs_rsvd *engs;
|
||||
int len, i;
|
||||
|
||||
buf[0] = '\0';
|
||||
for (i = 0; i < OTX2_CPT_MAX_ETYPES_PER_GRP; i++) {
|
||||
engs = &eng_grp->engs[i];
|
||||
if (!engs->type)
|
||||
continue;
|
||||
if (idx != -1 && idx != i)
|
||||
continue;
|
||||
|
||||
if (eng_grp->mirror.is_ena)
|
||||
mirrored_engs = find_engines_by_type(
|
||||
&eng_grp->g->grp[eng_grp->mirror.idx],
|
||||
engs->type);
|
||||
if (i > 0 && idx == -1) {
|
||||
len = strlen(buf);
|
||||
scnprintf(buf + len, size - len, ", ");
|
||||
}
|
||||
|
||||
len = strlen(buf);
|
||||
scnprintf(buf + len, size - len, "%d %s ",
|
||||
mirrored_engs ? engs->count + mirrored_engs->count :
|
||||
engs->count,
|
||||
get_eng_type_str(engs->type));
|
||||
if (mirrored_engs) {
|
||||
len = strlen(buf);
|
||||
scnprintf(buf + len, size - len,
|
||||
"(%d shared with engine_group%d) ",
|
||||
engs->count <= 0 ?
|
||||
engs->count + mirrored_engs->count :
|
||||
mirrored_engs->count,
|
||||
eng_grp->mirror.idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf)
|
||||
{
|
||||
struct otx2_cpt_eng_grps *eng_grps = &cptpf->eng_grps;
|
||||
struct otx2_cpt_eng_grp_info *mirrored_grp;
|
||||
char engs_info[2 * OTX2_CPT_NAME_LENGTH];
|
||||
struct otx2_cpt_eng_grp_info *grp;
|
||||
struct otx2_cpt_engs_rsvd *engs;
|
||||
u32 mask[4];
|
||||
int i, j;
|
||||
|
||||
pr_debug("Engine groups global info");
|
||||
pr_debug("max SE %d, max IE %d, max AE %d", eng_grps->avail.max_se_cnt,
|
||||
eng_grps->avail.max_ie_cnt, eng_grps->avail.max_ae_cnt);
|
||||
pr_debug("free SE %d", eng_grps->avail.se_cnt);
|
||||
pr_debug("free IE %d", eng_grps->avail.ie_cnt);
|
||||
pr_debug("free AE %d", eng_grps->avail.ae_cnt);
|
||||
|
||||
for (i = 0; i < OTX2_CPT_MAX_ENGINE_GROUPS; i++) {
|
||||
grp = &eng_grps->grp[i];
|
||||
pr_debug("engine_group%d, state %s", i,
|
||||
grp->is_enabled ? "enabled" : "disabled");
|
||||
if (grp->is_enabled) {
|
||||
mirrored_grp = &eng_grps->grp[grp->mirror.idx];
|
||||
pr_debug("Ucode0 filename %s, version %s",
|
||||
grp->mirror.is_ena ?
|
||||
mirrored_grp->ucode[0].filename :
|
||||
grp->ucode[0].filename,
|
||||
grp->mirror.is_ena ?
|
||||
mirrored_grp->ucode[0].ver_str :
|
||||
grp->ucode[0].ver_str);
|
||||
if (is_2nd_ucode_used(grp))
|
||||
pr_debug("Ucode1 filename %s, version %s",
|
||||
grp->ucode[1].filename,
|
||||
grp->ucode[1].ver_str);
|
||||
}
|
||||
|
||||
for (j = 0; j < OTX2_CPT_MAX_ETYPES_PER_GRP; j++) {
|
||||
engs = &grp->engs[j];
|
||||
if (engs->type) {
|
||||
get_engs_info(grp, engs_info,
|
||||
2 * OTX2_CPT_NAME_LENGTH, j);
|
||||
pr_debug("Slot%d: %s", j, engs_info);
|
||||
bitmap_to_arr32(mask, engs->bmap,
|
||||
eng_grps->engs_num);
|
||||
if (is_dev_otx2(cptpf->pdev))
|
||||
pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x",
|
||||
mask[3], mask[2], mask[1],
|
||||
mask[0]);
|
||||
else
|
||||
pr_debug("Mask: %8.8x %8.8x %8.8x %8.8x %8.8x",
|
||||
mask[4], mask[3], mask[2], mask[1],
|
||||
mask[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -143,6 +143,7 @@ struct otx2_cpt_eng_grp_info {
|
|||
};
|
||||
|
||||
struct otx2_cpt_eng_grps {
|
||||
struct mutex lock;
|
||||
struct otx2_cpt_eng_grp_info grp[OTX2_CPT_MAX_ENGINE_GROUPS];
|
||||
struct otx2_cpt_engs_available avail;
|
||||
void *obj; /* device specific data */
|
||||
|
@ -160,5 +161,9 @@ int otx2_cpt_create_eng_grps(struct otx2_cptpf_dev *cptpf,
|
|||
int otx2_cpt_disable_all_cores(struct otx2_cptpf_dev *cptpf);
|
||||
int otx2_cpt_get_eng_grp(struct otx2_cpt_eng_grps *eng_grps, int eng_type);
|
||||
int otx2_cpt_discover_eng_capabilities(struct otx2_cptpf_dev *cptpf);
|
||||
|
||||
int otx2_cpt_dl_custom_egrp_create(struct otx2_cptpf_dev *cptpf,
|
||||
struct devlink_param_gset_ctx *ctx);
|
||||
int otx2_cpt_dl_custom_egrp_delete(struct otx2_cptpf_dev *cptpf,
|
||||
struct devlink_param_gset_ctx *ctx);
|
||||
void otx2_cpt_print_uc_dbg_info(struct otx2_cptpf_dev *cptpf);
|
||||
#endif /* __OTX2_CPTPF_UCODE_H */
|
||||
|
|
|
@ -1682,11 +1682,8 @@ static void swap_func(void *lptr, void *rptr, int size)
|
|||
{
|
||||
struct cpt_device_desc *ldesc = lptr;
|
||||
struct cpt_device_desc *rdesc = rptr;
|
||||
struct cpt_device_desc desc;
|
||||
|
||||
desc = *ldesc;
|
||||
*ldesc = *rdesc;
|
||||
*rdesc = desc;
|
||||
swap(*ldesc, *rdesc);
|
||||
}
|
||||
|
||||
int otx2_cpt_crypto_init(struct pci_dev *pdev, struct module *mod,
|
||||
|
|
|
@ -1302,7 +1302,7 @@ static int omap_aes_suspend(struct device *dev)
|
|||
|
||||
static int omap_aes_resume(struct device *dev)
|
||||
{
|
||||
pm_runtime_resume_and_get(dev);
|
||||
pm_runtime_get_sync(dev);
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
|
|
@ -735,7 +735,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
|
|||
{
|
||||
.base.cra_name = "ecb(des)",
|
||||
.base.cra_driver_name = "ecb-des-omap",
|
||||
.base.cra_priority = 100,
|
||||
.base.cra_priority = 300,
|
||||
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
|
||||
CRYPTO_ALG_ASYNC,
|
||||
.base.cra_blocksize = DES_BLOCK_SIZE,
|
||||
|
@ -752,7 +752,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
|
|||
{
|
||||
.base.cra_name = "cbc(des)",
|
||||
.base.cra_driver_name = "cbc-des-omap",
|
||||
.base.cra_priority = 100,
|
||||
.base.cra_priority = 300,
|
||||
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
|
||||
CRYPTO_ALG_ASYNC,
|
||||
.base.cra_blocksize = DES_BLOCK_SIZE,
|
||||
|
@ -770,7 +770,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
|
|||
{
|
||||
.base.cra_name = "ecb(des3_ede)",
|
||||
.base.cra_driver_name = "ecb-des3-omap",
|
||||
.base.cra_priority = 100,
|
||||
.base.cra_priority = 300,
|
||||
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
|
||||
CRYPTO_ALG_ASYNC,
|
||||
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
|
||||
|
@ -787,7 +787,7 @@ static struct skcipher_alg algs_ecb_cbc[] = {
|
|||
{
|
||||
.base.cra_name = "cbc(des3_ede)",
|
||||
.base.cra_driver_name = "cbc-des3-omap",
|
||||
.base.cra_priority = 100,
|
||||
.base.cra_priority = 300,
|
||||
.base.cra_flags = CRYPTO_ALG_KERN_DRIVER_ONLY |
|
||||
CRYPTO_ALG_ASYNC,
|
||||
.base.cra_blocksize = DES3_EDE_BLOCK_SIZE,
|
||||
|
|
|
@ -13,6 +13,7 @@ config CRYPTO_DEV_QAT
|
|||
select CRYPTO_SHA512
|
||||
select CRYPTO_LIB_AES
|
||||
select FW_LOADER
|
||||
select CRC8
|
||||
|
||||
config CRYPTO_DEV_QAT_DH895xCC
|
||||
tristate "Support for Intel(R) DH895xCC"
|
||||
|
|
|
@ -1,10 +1,11 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2020 Intel Corporation */
|
||||
/* Copyright(c) 2020 - 2021 Intel Corporation */
|
||||
#include <linux/iopoll.h>
|
||||
#include <adf_accel_devices.h>
|
||||
#include <adf_cfg.h>
|
||||
#include <adf_common_drv.h>
|
||||
#include <adf_pf2vf_msg.h>
|
||||
#include <adf_gen4_hw_data.h>
|
||||
#include <adf_gen4_pfvf.h>
|
||||
#include "adf_4xxx_hw_data.h"
|
||||
#include "icp_qat_hw.h"
|
||||
|
||||
|
@ -13,12 +14,18 @@ struct adf_fw_config {
|
|||
char *obj_name;
|
||||
};
|
||||
|
||||
static struct adf_fw_config adf_4xxx_fw_config[] = {
|
||||
static struct adf_fw_config adf_4xxx_fw_cy_config[] = {
|
||||
{0xF0, ADF_4XXX_SYM_OBJ},
|
||||
{0xF, ADF_4XXX_ASYM_OBJ},
|
||||
{0x100, ADF_4XXX_ADMIN_OBJ},
|
||||
};
|
||||
|
||||
static struct adf_fw_config adf_4xxx_fw_dc_config[] = {
|
||||
{0xF0, ADF_4XXX_DC_OBJ},
|
||||
{0xF, ADF_4XXX_DC_OBJ},
|
||||
{0x100, ADF_4XXX_ADMIN_OBJ},
|
||||
};
|
||||
|
||||
/* Worker thread to service arbiter mappings */
|
||||
static const u32 thrd_to_arb_map[ADF_4XXX_MAX_ACCELENGINES] = {
|
||||
0x5555555, 0x5555555, 0x5555555, 0x5555555,
|
||||
|
@ -32,6 +39,39 @@ static struct adf_hw_device_class adf_4xxx_class = {
|
|||
.instances = 0,
|
||||
};
|
||||
|
||||
enum dev_services {
|
||||
SVC_CY = 0,
|
||||
SVC_DC,
|
||||
};
|
||||
|
||||
static const char *const dev_cfg_services[] = {
|
||||
[SVC_CY] = ADF_CFG_CY,
|
||||
[SVC_DC] = ADF_CFG_DC,
|
||||
};
|
||||
|
||||
static int get_service_enabled(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0};
|
||||
u32 ret;
|
||||
|
||||
ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC,
|
||||
ADF_SERVICES_ENABLED, services);
|
||||
if (ret) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
ADF_SERVICES_ENABLED " param not found\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
ret = match_string(dev_cfg_services, ARRAY_SIZE(dev_cfg_services),
|
||||
services);
|
||||
if (ret < 0)
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Invalid value of " ADF_SERVICES_ENABLED " param: %s\n",
|
||||
services);
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
static u32 get_accel_mask(struct adf_hw_device_data *self)
|
||||
{
|
||||
return ADF_4XXX_ACCELERATORS_MASK;
|
||||
|
@ -96,23 +136,67 @@ static void set_msix_default_rttable(struct adf_accel_dev *accel_dev)
|
|||
static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev;
|
||||
u32 capabilities_cy, capabilities_dc;
|
||||
u32 fusectl1;
|
||||
u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
|
||||
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
|
||||
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
|
||||
ICP_ACCEL_CAPABILITIES_AES_V2;
|
||||
|
||||
/* Read accelerator capabilities mask */
|
||||
pci_read_config_dword(pdev, ADF_4XXX_FUSECTL1_OFFSET, &fusectl1);
|
||||
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
|
||||
capabilities_cy = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
|
||||
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
|
||||
ICP_ACCEL_CAPABILITIES_CIPHER |
|
||||
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
|
||||
ICP_ACCEL_CAPABILITIES_SHA3 |
|
||||
ICP_ACCEL_CAPABILITIES_SHA3_EXT |
|
||||
ICP_ACCEL_CAPABILITIES_HKDF |
|
||||
ICP_ACCEL_CAPABILITIES_ECEDMONT |
|
||||
ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
|
||||
ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
|
||||
ICP_ACCEL_CAPABILITIES_AES_V2;
|
||||
|
||||
return capabilities;
|
||||
/* A set bit in fusectl1 means the feature is OFF in this SKU */
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_CIPHER_SLICE) {
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_HKDF;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
|
||||
}
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_UCS_SLICE) {
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
|
||||
}
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_AUTH_SLICE) {
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
|
||||
}
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_PKE_SLICE) {
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
|
||||
capabilities_cy &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT;
|
||||
}
|
||||
|
||||
capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
|
||||
ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
|
||||
ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
|
||||
ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
|
||||
|
||||
if (fusectl1 & ICP_ACCEL_4XXX_MASK_COMPRESS_SLICE) {
|
||||
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
|
||||
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
|
||||
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
|
||||
capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
|
||||
}
|
||||
|
||||
switch (get_service_enabled(accel_dev)) {
|
||||
case SVC_CY:
|
||||
return capabilities_cy;
|
||||
case SVC_DC:
|
||||
return capabilities_dc;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
|
||||
|
@ -191,29 +275,36 @@ static int adf_init_device(struct adf_accel_dev *accel_dev)
|
|||
return ret;
|
||||
}
|
||||
|
||||
static int pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static u32 uof_get_num_objs(void)
|
||||
{
|
||||
return ARRAY_SIZE(adf_4xxx_fw_config);
|
||||
BUILD_BUG_ON_MSG(ARRAY_SIZE(adf_4xxx_fw_cy_config) !=
|
||||
ARRAY_SIZE(adf_4xxx_fw_dc_config),
|
||||
"Size mismatch between adf_4xxx_fw_*_config arrays");
|
||||
|
||||
return ARRAY_SIZE(adf_4xxx_fw_cy_config);
|
||||
}
|
||||
|
||||
static char *uof_get_name(u32 obj_num)
|
||||
static char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num)
|
||||
{
|
||||
return adf_4xxx_fw_config[obj_num].obj_name;
|
||||
switch (get_service_enabled(accel_dev)) {
|
||||
case SVC_CY:
|
||||
return adf_4xxx_fw_cy_config[obj_num].obj_name;
|
||||
case SVC_DC:
|
||||
return adf_4xxx_fw_dc_config[obj_num].obj_name;
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static u32 uof_get_ae_mask(u32 obj_num)
|
||||
static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
|
||||
{
|
||||
return adf_4xxx_fw_config[obj_num].ae_mask;
|
||||
}
|
||||
switch (get_service_enabled(accel_dev)) {
|
||||
case SVC_CY:
|
||||
return adf_4xxx_fw_cy_config[obj_num].ae_mask;
|
||||
case SVC_DC:
|
||||
return adf_4xxx_fw_dc_config[obj_num].ae_mask;
|
||||
}
|
||||
|
||||
static u32 get_vf2pf_sources(void __iomem *pmisc_addr)
|
||||
{
|
||||
/* For the moment do not report vf2pf sources */
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
@ -222,12 +313,14 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
|
|||
hw_data->dev_class = &adf_4xxx_class;
|
||||
hw_data->instance_id = adf_4xxx_class.instances++;
|
||||
hw_data->num_banks = ADF_4XXX_ETR_MAX_BANKS;
|
||||
hw_data->num_banks_per_vf = ADF_4XXX_NUM_BANKS_PER_VF;
|
||||
hw_data->num_rings_per_bank = ADF_4XXX_NUM_RINGS_PER_BANK;
|
||||
hw_data->num_accel = ADF_4XXX_MAX_ACCELERATORS;
|
||||
hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES;
|
||||
hw_data->num_logical_accel = 1;
|
||||
hw_data->tx_rx_gap = ADF_4XXX_RX_RINGS_OFFSET;
|
||||
hw_data->tx_rings_mask = ADF_4XXX_TX_RINGS_MASK;
|
||||
hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP;
|
||||
hw_data->alloc_irq = adf_isr_resource_alloc;
|
||||
hw_data->free_irq = adf_isr_resource_free;
|
||||
hw_data->enable_error_correction = adf_enable_error_correction;
|
||||
|
@ -259,12 +352,11 @@ void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data)
|
|||
hw_data->uof_get_ae_mask = uof_get_ae_mask;
|
||||
hw_data->set_msix_rttable = set_msix_default_rttable;
|
||||
hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
|
||||
hw_data->enable_pfvf_comms = pfvf_comms_disabled;
|
||||
hw_data->get_vf2pf_sources = get_vf2pf_sources;
|
||||
hw_data->disable_iov = adf_disable_sriov;
|
||||
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
|
||||
hw_data->ring_pair_reset = adf_gen4_ring_pair_reset;
|
||||
|
||||
adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);
|
||||
adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops);
|
||||
}
|
||||
|
||||
void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data)
|
||||
|
|
|
@ -37,6 +37,7 @@
|
|||
|
||||
/* Bank and ring configuration */
|
||||
#define ADF_4XXX_NUM_RINGS_PER_BANK 2
|
||||
#define ADF_4XXX_NUM_BANKS_PER_VF 4
|
||||
|
||||
/* Error source registers */
|
||||
#define ADF_4XXX_ERRSOU0 (0x41A200)
|
||||
|
@ -76,6 +77,7 @@
|
|||
#define ADF_4XXX_FW "qat_4xxx.bin"
|
||||
#define ADF_4XXX_MMP "qat_4xxx_mmp.bin"
|
||||
#define ADF_4XXX_SYM_OBJ "qat_4xxx_sym.bin"
|
||||
#define ADF_4XXX_DC_OBJ "qat_4xxx_dc.bin"
|
||||
#define ADF_4XXX_ASYM_OBJ "qat_4xxx_asym.bin"
|
||||
#define ADF_4XXX_ADMIN_OBJ "qat_4xxx_admin.bin"
|
||||
|
||||
|
|
|
@ -29,6 +29,29 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev)
|
|||
adf_devmgr_rm_dev(accel_dev, NULL);
|
||||
}
|
||||
|
||||
static int adf_cfg_dev_init(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
const char *config;
|
||||
int ret;
|
||||
|
||||
config = accel_dev->accel_id % 2 ? ADF_CFG_DC : ADF_CFG_CY;
|
||||
|
||||
ret = adf_cfg_section_add(accel_dev, ADF_GENERAL_SEC);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
/* Default configuration is crypto only for even devices
|
||||
* and compression for odd devices
|
||||
*/
|
||||
ret = adf_cfg_add_key_value_param(accel_dev, ADF_GENERAL_SEC,
|
||||
ADF_SERVICES_ENABLED, config,
|
||||
ADF_STR);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int adf_crypto_dev_config(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES];
|
||||
|
@ -227,8 +250,18 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|||
goto out_err;
|
||||
}
|
||||
|
||||
ret = adf_cfg_dev_init(accel_dev);
|
||||
if (ret) {
|
||||
dev_err(&pdev->dev, "Failed to initialize configuration.\n");
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
/* Get accelerator capabilities mask */
|
||||
hw_data->accel_capabilities_mask = hw_data->get_accel_cap(accel_dev);
|
||||
if (!hw_data->accel_capabilities_mask) {
|
||||
dev_err(&pdev->dev, "Failed to get capabilities mask.\n");
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
/* Find and map all the device's BARS */
|
||||
bar_mask = pci_select_bars(pdev, IORESOURCE_MEM) & ADF_4XXX_BAR_MASK;
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2014 - 2020 Intel Corporation */
|
||||
/* Copyright(c) 2014 - 2021 Intel Corporation */
|
||||
#include <adf_accel_devices.h>
|
||||
#include <adf_common_drv.h>
|
||||
#include <adf_pf2vf_msg.h>
|
||||
#include <adf_gen2_hw_data.h>
|
||||
#include <adf_gen2_pfvf.h>
|
||||
#include "adf_c3xxx_hw_data.h"
|
||||
#include "icp_qat_hw.h"
|
||||
|
||||
|
@ -109,6 +109,7 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data)
|
|||
hw_data->num_engines = ADF_C3XXX_MAX_ACCELENGINES;
|
||||
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
|
||||
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
|
||||
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
|
||||
hw_data->alloc_irq = adf_isr_resource_alloc;
|
||||
hw_data->free_irq = adf_isr_resource_free;
|
||||
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
|
||||
|
@ -135,14 +136,9 @@ void adf_init_hw_data_c3xxx(struct adf_hw_device_data *hw_data)
|
|||
hw_data->enable_ints = adf_enable_ints;
|
||||
hw_data->reset_device = adf_reset_flr;
|
||||
hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer;
|
||||
hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
|
||||
hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
|
||||
hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
|
||||
hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
|
||||
hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
|
||||
hw_data->disable_iov = adf_disable_sriov;
|
||||
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
|
||||
|
||||
adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
|
||||
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2015 - 2020 Intel Corporation */
|
||||
/* Copyright(c) 2015 - 2021 Intel Corporation */
|
||||
#include <adf_accel_devices.h>
|
||||
#include <adf_pf2vf_msg.h>
|
||||
#include <adf_common_drv.h>
|
||||
#include <adf_gen2_hw_data.h>
|
||||
#include <adf_gen2_pfvf.h>
|
||||
#include <adf_pfvf_vf_msg.h>
|
||||
#include "adf_c3xxxvf_hw_data.h"
|
||||
|
||||
static struct adf_hw_device_class c3xxxiov_class = {
|
||||
|
@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
|
|||
return DEV_SKU_VF;
|
||||
}
|
||||
|
||||
static u32 get_pf2vf_offset(u32 i)
|
||||
{
|
||||
return ADF_C3XXXIOV_PF2VF_OFFSET;
|
||||
}
|
||||
|
||||
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
return 0;
|
||||
|
@ -71,6 +67,7 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data)
|
|||
hw_data->num_engines = ADF_C3XXXIOV_MAX_ACCELENGINES;
|
||||
hw_data->tx_rx_gap = ADF_C3XXXIOV_RX_RINGS_OFFSET;
|
||||
hw_data->tx_rings_mask = ADF_C3XXXIOV_TX_RINGS_MASK;
|
||||
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
|
||||
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
|
||||
hw_data->free_irq = adf_vf_isr_resource_free;
|
||||
hw_data->enable_error_correction = adf_vf_void_noop;
|
||||
|
@ -86,13 +83,11 @@ void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data)
|
|||
hw_data->get_num_aes = get_num_aes;
|
||||
hw_data->get_etr_bar_id = get_etr_bar_id;
|
||||
hw_data->get_misc_bar_id = get_misc_bar_id;
|
||||
hw_data->get_pf2vf_offset = get_pf2vf_offset;
|
||||
hw_data->get_sku = get_sku;
|
||||
hw_data->enable_ints = adf_vf_void_noop;
|
||||
hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
|
||||
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
|
||||
hw_data->dev_class->instances++;
|
||||
adf_devmgr_update_class_index(hw_data);
|
||||
adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
|
||||
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
|
||||
}
|
||||
|
||||
|
|
|
@ -12,7 +12,6 @@
|
|||
#define ADF_C3XXXIOV_TX_RINGS_MASK 0xFF
|
||||
#define ADF_C3XXXIOV_ETR_BAR 0
|
||||
#define ADF_C3XXXIOV_ETR_MAX_BANKS 1
|
||||
#define ADF_C3XXXIOV_PF2VF_OFFSET 0x200
|
||||
|
||||
void adf_init_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data);
|
||||
void adf_clean_hw_data_c3xxxiov(struct adf_hw_device_data *hw_data);
|
||||
|
|
|
@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|||
}
|
||||
pci_set_master(pdev);
|
||||
/* Completion for VF2PF request/response message exchange */
|
||||
init_completion(&accel_dev->vf.iov_msg_completion);
|
||||
|
||||
ret = qat_crypto_dev_config(accel_dev);
|
||||
if (ret)
|
||||
goto out_err_free_reg;
|
||||
init_completion(&accel_dev->vf.msg_received);
|
||||
|
||||
ret = adf_dev_init(accel_dev);
|
||||
if (ret)
|
||||
|
|
|
@ -1,9 +1,9 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2014 - 2020 Intel Corporation */
|
||||
/* Copyright(c) 2014 - 2021 Intel Corporation */
|
||||
#include <adf_accel_devices.h>
|
||||
#include <adf_common_drv.h>
|
||||
#include <adf_pf2vf_msg.h>
|
||||
#include <adf_gen2_hw_data.h>
|
||||
#include <adf_gen2_pfvf.h>
|
||||
#include "adf_c62x_hw_data.h"
|
||||
#include "icp_qat_hw.h"
|
||||
|
||||
|
@ -111,6 +111,7 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data)
|
|||
hw_data->num_engines = ADF_C62X_MAX_ACCELENGINES;
|
||||
hw_data->tx_rx_gap = ADF_GEN2_RX_RINGS_OFFSET;
|
||||
hw_data->tx_rings_mask = ADF_GEN2_TX_RINGS_MASK;
|
||||
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
|
||||
hw_data->alloc_irq = adf_isr_resource_alloc;
|
||||
hw_data->free_irq = adf_isr_resource_free;
|
||||
hw_data->enable_error_correction = adf_gen2_enable_error_correction;
|
||||
|
@ -137,14 +138,9 @@ void adf_init_hw_data_c62x(struct adf_hw_device_data *hw_data)
|
|||
hw_data->enable_ints = adf_enable_ints;
|
||||
hw_data->reset_device = adf_reset_flr;
|
||||
hw_data->set_ssm_wdtimer = adf_gen2_set_ssm_wdtimer;
|
||||
hw_data->get_pf2vf_offset = adf_gen2_get_pf2vf_offset;
|
||||
hw_data->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
|
||||
hw_data->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
|
||||
hw_data->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
|
||||
hw_data->enable_pfvf_comms = adf_enable_pf2vf_comms;
|
||||
hw_data->disable_iov = adf_disable_sriov;
|
||||
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
|
||||
|
||||
adf_gen2_init_pf_pfvf_ops(&hw_data->pfvf_ops);
|
||||
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2015 - 2020 Intel Corporation */
|
||||
/* Copyright(c) 2015 - 2021 Intel Corporation */
|
||||
#include <adf_accel_devices.h>
|
||||
#include <adf_pf2vf_msg.h>
|
||||
#include <adf_common_drv.h>
|
||||
#include <adf_gen2_hw_data.h>
|
||||
#include <adf_gen2_pfvf.h>
|
||||
#include <adf_pfvf_vf_msg.h>
|
||||
#include "adf_c62xvf_hw_data.h"
|
||||
|
||||
static struct adf_hw_device_class c62xiov_class = {
|
||||
|
@ -47,11 +48,6 @@ static enum dev_sku_info get_sku(struct adf_hw_device_data *self)
|
|||
return DEV_SKU_VF;
|
||||
}
|
||||
|
||||
static u32 get_pf2vf_offset(u32 i)
|
||||
{
|
||||
return ADF_C62XIOV_PF2VF_OFFSET;
|
||||
}
|
||||
|
||||
static int adf_vf_int_noop(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
return 0;
|
||||
|
@ -71,6 +67,7 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data)
|
|||
hw_data->num_engines = ADF_C62XIOV_MAX_ACCELENGINES;
|
||||
hw_data->tx_rx_gap = ADF_C62XIOV_RX_RINGS_OFFSET;
|
||||
hw_data->tx_rings_mask = ADF_C62XIOV_TX_RINGS_MASK;
|
||||
hw_data->ring_to_svc_map = ADF_GEN2_DEFAULT_RING_TO_SRV_MAP;
|
||||
hw_data->alloc_irq = adf_vf_isr_resource_alloc;
|
||||
hw_data->free_irq = adf_vf_isr_resource_free;
|
||||
hw_data->enable_error_correction = adf_vf_void_noop;
|
||||
|
@ -86,13 +83,11 @@ void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data)
|
|||
hw_data->get_num_aes = get_num_aes;
|
||||
hw_data->get_etr_bar_id = get_etr_bar_id;
|
||||
hw_data->get_misc_bar_id = get_misc_bar_id;
|
||||
hw_data->get_pf2vf_offset = get_pf2vf_offset;
|
||||
hw_data->get_sku = get_sku;
|
||||
hw_data->enable_ints = adf_vf_void_noop;
|
||||
hw_data->enable_pfvf_comms = adf_enable_vf2pf_comms;
|
||||
hw_data->min_iov_compat_ver = ADF_PFVF_COMPAT_THIS_VERSION;
|
||||
hw_data->dev_class->instances++;
|
||||
adf_devmgr_update_class_index(hw_data);
|
||||
adf_gen2_init_vf_pfvf_ops(&hw_data->pfvf_ops);
|
||||
adf_gen2_init_hw_csr_ops(&hw_data->csr_ops);
|
||||
}
|
||||
|
||||
|
|
|
@ -12,7 +12,6 @@
|
|||
#define ADF_C62XIOV_TX_RINGS_MASK 0xFF
|
||||
#define ADF_C62XIOV_ETR_BAR 0
|
||||
#define ADF_C62XIOV_ETR_MAX_BANKS 1
|
||||
#define ADF_C62XIOV_PF2VF_OFFSET 0x200
|
||||
|
||||
void adf_init_hw_data_c62xiov(struct adf_hw_device_data *hw_data);
|
||||
void adf_clean_hw_data_c62xiov(struct adf_hw_device_data *hw_data);
|
||||
|
|
|
@ -171,11 +171,7 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
|
|||
}
|
||||
pci_set_master(pdev);
|
||||
/* Completion for VF2PF request/response message exchange */
|
||||
init_completion(&accel_dev->vf.iov_msg_completion);
|
||||
|
||||
ret = qat_crypto_dev_config(accel_dev);
|
||||
if (ret)
|
||||
goto out_err_free_reg;
|
||||
init_completion(&accel_dev->vf.msg_received);
|
||||
|
||||
ret = adf_dev_init(accel_dev);
|
||||
if (ret)
|
||||
|
|
|
@ -19,5 +19,7 @@ intel_qat-objs := adf_cfg.o \
|
|||
qat_hal.o
|
||||
|
||||
intel_qat-$(CONFIG_DEBUG_FS) += adf_transport_debug.o
|
||||
intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_pf2vf_msg.o \
|
||||
adf_vf2pf_msg.o adf_vf_isr.o
|
||||
intel_qat-$(CONFIG_PCI_IOV) += adf_sriov.o adf_vf_isr.o adf_pfvf_utils.o \
|
||||
adf_pfvf_pf_msg.o adf_pfvf_pf_proto.o \
|
||||
adf_pfvf_vf_msg.o adf_pfvf_vf_proto.o \
|
||||
adf_gen2_pfvf.o adf_gen4_pfvf.o
|
||||
|
|
|
@ -8,6 +8,7 @@
|
|||
#include <linux/io.h>
|
||||
#include <linux/ratelimit.h>
|
||||
#include "adf_cfg_common.h"
|
||||
#include "adf_pfvf_msg.h"
|
||||
|
||||
#define ADF_DH895XCC_DEVICE_NAME "dh895xcc"
|
||||
#define ADF_DH895XCCVF_DEVICE_NAME "dh895xccvf"
|
||||
|
@ -147,6 +148,19 @@ struct adf_accel_dev;
|
|||
struct adf_etr_data;
|
||||
struct adf_etr_ring_data;
|
||||
|
||||
struct adf_pfvf_ops {
|
||||
int (*enable_comms)(struct adf_accel_dev *accel_dev);
|
||||
u32 (*get_pf2vf_offset)(u32 i);
|
||||
u32 (*get_vf2pf_offset)(u32 i);
|
||||
u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr);
|
||||
void (*enable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);
|
||||
void (*disable_vf2pf_interrupts)(void __iomem *pmisc_addr, u32 vf_mask);
|
||||
int (*send_msg)(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
|
||||
u32 pfvf_offset, struct mutex *csr_lock);
|
||||
struct pfvf_message (*recv_msg)(struct adf_accel_dev *accel_dev,
|
||||
u32 pfvf_offset, u8 compat_ver);
|
||||
};
|
||||
|
||||
struct adf_hw_device_data {
|
||||
struct adf_hw_device_class *dev_class;
|
||||
u32 (*get_accel_mask)(struct adf_hw_device_data *self);
|
||||
|
@ -157,7 +171,6 @@ struct adf_hw_device_data {
|
|||
u32 (*get_etr_bar_id)(struct adf_hw_device_data *self);
|
||||
u32 (*get_num_aes)(struct adf_hw_device_data *self);
|
||||
u32 (*get_num_accels)(struct adf_hw_device_data *self);
|
||||
u32 (*get_pf2vf_offset)(u32 i);
|
||||
void (*get_arb_info)(struct arb_info *arb_csrs_info);
|
||||
void (*get_admin_info)(struct admin_info *admin_csrs_info);
|
||||
enum dev_sku_info (*get_sku)(struct adf_hw_device_data *self);
|
||||
|
@ -176,35 +189,34 @@ struct adf_hw_device_data {
|
|||
bool enable);
|
||||
void (*enable_ints)(struct adf_accel_dev *accel_dev);
|
||||
void (*set_ssm_wdtimer)(struct adf_accel_dev *accel_dev);
|
||||
int (*enable_pfvf_comms)(struct adf_accel_dev *accel_dev);
|
||||
u32 (*get_vf2pf_sources)(void __iomem *pmisc_addr);
|
||||
void (*enable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr,
|
||||
u32 vf_mask);
|
||||
void (*disable_vf2pf_interrupts)(void __iomem *pmisc_bar_addr,
|
||||
u32 vf_mask);
|
||||
int (*ring_pair_reset)(struct adf_accel_dev *accel_dev, u32 bank_nr);
|
||||
void (*reset_device)(struct adf_accel_dev *accel_dev);
|
||||
void (*set_msix_rttable)(struct adf_accel_dev *accel_dev);
|
||||
char *(*uof_get_name)(u32 obj_num);
|
||||
char *(*uof_get_name)(struct adf_accel_dev *accel_dev, u32 obj_num);
|
||||
u32 (*uof_get_num_objs)(void);
|
||||
u32 (*uof_get_ae_mask)(u32 obj_num);
|
||||
u32 (*uof_get_ae_mask)(struct adf_accel_dev *accel_dev, u32 obj_num);
|
||||
struct adf_pfvf_ops pfvf_ops;
|
||||
struct adf_hw_csr_ops csr_ops;
|
||||
const char *fw_name;
|
||||
const char *fw_mmp_name;
|
||||
u32 fuses;
|
||||
u32 straps;
|
||||
u32 accel_capabilities_mask;
|
||||
u32 extended_dc_capabilities;
|
||||
u32 clock_frequency;
|
||||
u32 instance_id;
|
||||
u16 accel_mask;
|
||||
u32 ae_mask;
|
||||
u32 admin_ae_mask;
|
||||
u16 tx_rings_mask;
|
||||
u16 ring_to_svc_map;
|
||||
u8 tx_rx_gap;
|
||||
u8 num_banks;
|
||||
u16 num_banks_per_vf;
|
||||
u8 num_rings_per_bank;
|
||||
u8 num_accel;
|
||||
u8 num_logical_accel;
|
||||
u8 num_engines;
|
||||
u8 min_iov_compat_ver;
|
||||
};
|
||||
|
||||
/* CSR write macro */
|
||||
|
@ -214,14 +226,22 @@ struct adf_hw_device_data {
|
|||
/* CSR read macro */
|
||||
#define ADF_CSR_RD(csr_base, csr_offset) __raw_readl(csr_base + csr_offset)
|
||||
|
||||
#define ADF_CFG_NUM_SERVICES 4
|
||||
#define ADF_SRV_TYPE_BIT_LEN 3
|
||||
#define ADF_SRV_TYPE_MASK 0x7
|
||||
|
||||
#define GET_DEV(accel_dev) ((accel_dev)->accel_pci_dev.pci_dev->dev)
|
||||
#define GET_BARS(accel_dev) ((accel_dev)->accel_pci_dev.pci_bars)
|
||||
#define GET_HW_DATA(accel_dev) (accel_dev->hw_device)
|
||||
#define GET_MAX_BANKS(accel_dev) (GET_HW_DATA(accel_dev)->num_banks)
|
||||
#define GET_NUM_RINGS_PER_BANK(accel_dev) \
|
||||
GET_HW_DATA(accel_dev)->num_rings_per_bank
|
||||
#define GET_SRV_TYPE(accel_dev, idx) \
|
||||
(((GET_HW_DATA(accel_dev)->ring_to_svc_map) >> (ADF_SRV_TYPE_BIT_LEN * (idx))) \
|
||||
& ADF_SRV_TYPE_MASK)
|
||||
#define GET_MAX_ACCELENGINES(accel_dev) (GET_HW_DATA(accel_dev)->num_engines)
|
||||
#define GET_CSR_OPS(accel_dev) (&(accel_dev)->hw_device->csr_ops)
|
||||
#define GET_PFVF_OPS(accel_dev) (&(accel_dev)->hw_device->pfvf_ops)
|
||||
#define accel_to_pci_dev(accel_ptr) accel_ptr->accel_pci_dev.pci_dev
|
||||
|
||||
struct adf_admin_comms;
|
||||
|
@ -238,6 +258,7 @@ struct adf_accel_vf_info {
|
|||
struct ratelimit_state vf2pf_ratelimit;
|
||||
u32 vf_nr;
|
||||
bool init;
|
||||
u8 vf_compat_ver;
|
||||
};
|
||||
|
||||
struct adf_accel_dev {
|
||||
|
@ -265,9 +286,9 @@ struct adf_accel_dev {
|
|||
char irq_name[ADF_MAX_MSIX_VECTOR_NAME];
|
||||
struct tasklet_struct pf2vf_bh_tasklet;
|
||||
struct mutex vf2pf_lock; /* protect CSR access */
|
||||
struct completion iov_msg_completion;
|
||||
u8 compatible;
|
||||
u8 pf_version;
|
||||
struct completion msg_received;
|
||||
struct pfvf_message response; /* temp field holding pf2vf response */
|
||||
u8 pf_compat_ver;
|
||||
} vf;
|
||||
};
|
||||
bool is_vf;
|
||||
|
|
|
@ -22,8 +22,12 @@ static int adf_ae_fw_load_images(struct adf_accel_dev *accel_dev, void *fw_addr,
|
|||
num_objs = hw_device->uof_get_num_objs();
|
||||
|
||||
for (i = 0; i < num_objs; i++) {
|
||||
obj_name = hw_device->uof_get_name(i);
|
||||
ae_mask = hw_device->uof_get_ae_mask(i);
|
||||
obj_name = hw_device->uof_get_name(accel_dev, i);
|
||||
ae_mask = hw_device->uof_get_ae_mask(accel_dev, i);
|
||||
if (!obj_name || !ae_mask) {
|
||||
dev_err(&GET_DEV(accel_dev), "Invalid UOF image\n");
|
||||
goto out_err;
|
||||
}
|
||||
|
||||
if (qat_uclo_set_cfg_ae_mask(loader, ae_mask)) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
|
|
|
@ -194,6 +194,35 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
|
|||
return adf_send_admin(accel_dev, &req, &resp, ae_mask);
|
||||
}
|
||||
|
||||
static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev,
|
||||
u32 *capabilities)
|
||||
{
|
||||
struct adf_hw_device_data *hw_device = accel_dev->hw_device;
|
||||
struct icp_qat_fw_init_admin_resp resp;
|
||||
struct icp_qat_fw_init_admin_req req;
|
||||
unsigned long ae_mask;
|
||||
unsigned long ae;
|
||||
int ret;
|
||||
|
||||
/* Target only service accelerator engines */
|
||||
ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask;
|
||||
|
||||
memset(&req, 0, sizeof(req));
|
||||
memset(&resp, 0, sizeof(resp));
|
||||
req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET;
|
||||
|
||||
*capabilities = 0;
|
||||
for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) {
|
||||
ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
*capabilities |= resp.extended_features;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_send_admin_init() - Function sends init message to FW
|
||||
* @accel_dev: Pointer to acceleration device.
|
||||
|
@ -204,8 +233,16 @@ static int adf_set_fw_constants(struct adf_accel_dev *accel_dev)
|
|||
*/
|
||||
int adf_send_admin_init(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
u32 dc_capabilities = 0;
|
||||
int ret;
|
||||
|
||||
ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities);
|
||||
if (ret) {
|
||||
dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n");
|
||||
return ret;
|
||||
}
|
||||
accel_dev->hw_device->extended_dc_capabilities = dc_capabilities;
|
||||
|
||||
ret = adf_set_fw_constants(accel_dev);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
@ -218,9 +255,7 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
|
|||
{
|
||||
struct adf_admin_comms *admin;
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
struct adf_bar *pmisc =
|
||||
&GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
|
||||
void __iomem *csr = pmisc->virt_addr;
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
struct admin_info admin_csrs_info;
|
||||
u32 mailbox_offset, adminmsg_u, adminmsg_l;
|
||||
void __iomem *mailbox;
|
||||
|
@ -254,13 +289,13 @@ int adf_init_admin_comms(struct adf_accel_dev *accel_dev)
|
|||
hw_data->get_admin_info(&admin_csrs_info);
|
||||
|
||||
mailbox_offset = admin_csrs_info.mailbox_offset;
|
||||
mailbox = csr + mailbox_offset;
|
||||
mailbox = pmisc_addr + mailbox_offset;
|
||||
adminmsg_u = admin_csrs_info.admin_msg_ur;
|
||||
adminmsg_l = admin_csrs_info.admin_msg_lr;
|
||||
|
||||
reg_val = (u64)admin->phy_addr;
|
||||
ADF_CSR_WR(csr, adminmsg_u, upper_32_bits(reg_val));
|
||||
ADF_CSR_WR(csr, adminmsg_l, lower_32_bits(reg_val));
|
||||
ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val));
|
||||
ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val));
|
||||
|
||||
mutex_init(&admin->lock);
|
||||
admin->mailbox_addr = mailbox;
|
||||
|
|
|
@ -297,3 +297,4 @@ int adf_cfg_get_param_value(struct adf_accel_dev *accel_dev,
|
|||
up_read(&cfg->lock);
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_cfg_get_param_value);
|
||||
|
|
|
@ -19,6 +19,19 @@
|
|||
#define ADF_MAX_DEVICES (32 * 32)
|
||||
#define ADF_DEVS_ARRAY_SIZE BITS_TO_LONGS(ADF_MAX_DEVICES)
|
||||
|
||||
#define ADF_CFG_SERV_RING_PAIR_0_SHIFT 0
|
||||
#define ADF_CFG_SERV_RING_PAIR_1_SHIFT 3
|
||||
#define ADF_CFG_SERV_RING_PAIR_2_SHIFT 6
|
||||
#define ADF_CFG_SERV_RING_PAIR_3_SHIFT 9
|
||||
enum adf_cfg_service_type {
|
||||
UNUSED = 0,
|
||||
CRYPTO,
|
||||
COMP,
|
||||
SYM,
|
||||
ASYM,
|
||||
USED
|
||||
};
|
||||
|
||||
enum adf_cfg_val_type {
|
||||
ADF_DEC,
|
||||
ADF_HEX,
|
||||
|
|
|
@ -22,6 +22,9 @@
|
|||
#define ADF_RING_ASYM_BANK_NUM "BankAsymNumber"
|
||||
#define ADF_CY "Cy"
|
||||
#define ADF_DC "Dc"
|
||||
#define ADF_CFG_DC "dc"
|
||||
#define ADF_CFG_CY "sym;asym"
|
||||
#define ADF_SERVICES_ENABLED "ServicesEnabled"
|
||||
#define ADF_ETRMGR_COALESCING_ENABLED "InterruptCoalescingEnabled"
|
||||
#define ADF_ETRMGR_COALESCING_ENABLED_FORMAT \
|
||||
ADF_ETRMGR_BANK "%d" ADF_ETRMGR_COALESCING_ENABLED
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2014 - 2020 Intel Corporation */
|
||||
/* Copyright(c) 2014 - 2021 Intel Corporation */
|
||||
#ifndef ADF_DRV_H
|
||||
#define ADF_DRV_H
|
||||
|
||||
|
@ -62,9 +62,6 @@ int adf_dev_start(struct adf_accel_dev *accel_dev);
|
|||
void adf_dev_stop(struct adf_accel_dev *accel_dev);
|
||||
void adf_dev_shutdown(struct adf_accel_dev *accel_dev);
|
||||
|
||||
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev);
|
||||
int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev);
|
||||
void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info);
|
||||
void adf_devmgr_update_class_index(struct adf_hw_device_data *hw_data);
|
||||
void adf_clean_vf_map(bool);
|
||||
|
||||
|
@ -117,6 +114,7 @@ void adf_cleanup_etr_data(struct adf_accel_dev *accel_dev);
|
|||
int qat_crypto_register(void);
|
||||
int qat_crypto_unregister(void);
|
||||
int qat_crypto_dev_config(struct adf_accel_dev *accel_dev);
|
||||
int qat_crypto_vf_dev_config(struct adf_accel_dev *accel_dev);
|
||||
struct qat_crypto_instance *qat_crypto_get_instance_node(int node);
|
||||
void qat_crypto_put_instance(struct qat_crypto_instance *inst);
|
||||
void qat_alg_callback(void *resp);
|
||||
|
@ -131,6 +129,8 @@ void adf_isr_resource_free(struct adf_accel_dev *accel_dev);
|
|||
int adf_vf_isr_resource_alloc(struct adf_accel_dev *accel_dev);
|
||||
void adf_vf_isr_resource_free(struct adf_accel_dev *accel_dev);
|
||||
|
||||
int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev);
|
||||
|
||||
int qat_hal_init(struct adf_accel_dev *accel_dev);
|
||||
void qat_hal_deinit(struct icp_qat_fw_loader_handle *handle);
|
||||
int qat_hal_start(struct icp_qat_fw_loader_handle *handle);
|
||||
|
@ -193,17 +193,14 @@ int adf_sriov_configure(struct pci_dev *pdev, int numvfs);
|
|||
void adf_disable_sriov(struct adf_accel_dev *accel_dev);
|
||||
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev,
|
||||
u32 vf_mask);
|
||||
void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
|
||||
u32 vf_mask);
|
||||
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev,
|
||||
u32 vf_mask);
|
||||
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev);
|
||||
bool adf_recv_and_handle_pf2vf_msg(struct adf_accel_dev *accel_dev);
|
||||
bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr);
|
||||
int adf_pf2vf_handle_pf_restarting(struct adf_accel_dev *accel_dev);
|
||||
void adf_enable_pf2vf_interrupts(struct adf_accel_dev *accel_dev);
|
||||
void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev);
|
||||
void adf_schedule_vf2pf_handler(struct adf_accel_vf_info *vf_info);
|
||||
int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg);
|
||||
int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev);
|
||||
void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev);
|
||||
int adf_init_pf_wq(void);
|
||||
void adf_exit_pf_wq(void);
|
||||
int adf_init_vf_wq(void);
|
||||
|
@ -212,11 +209,6 @@ void adf_flush_vf_wq(struct adf_accel_dev *accel_dev);
|
|||
#else
|
||||
#define adf_sriov_configure NULL
|
||||
|
||||
static inline int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void adf_disable_sriov(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
}
|
||||
|
@ -229,15 +221,6 @@ static inline void adf_disable_pf2vf_interrupts(struct adf_accel_dev *accel_dev)
|
|||
{
|
||||
}
|
||||
|
||||
static inline int adf_vf2pf_notify_init(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void adf_vf2pf_notify_shutdown(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int adf_init_pf_wq(void)
|
||||
{
|
||||
return 0;
|
||||
|
@ -261,4 +244,15 @@ static inline void adf_flush_vf_wq(struct adf_accel_dev *accel_dev)
|
|||
}
|
||||
|
||||
#endif
|
||||
|
||||
static inline void __iomem *adf_get_pmisc_base(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
struct adf_bar *pmisc;
|
||||
|
||||
pmisc = &GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
|
||||
|
||||
return pmisc->virt_addr;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
@ -1,57 +1,10 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2020 Intel Corporation */
|
||||
#include "adf_common_drv.h"
|
||||
#include "adf_gen2_hw_data.h"
|
||||
#include "icp_qat_hw.h"
|
||||
#include <linux/pci.h>
|
||||
|
||||
#define ADF_GEN2_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
|
||||
|
||||
u32 adf_gen2_get_pf2vf_offset(u32 i)
|
||||
{
|
||||
return ADF_GEN2_PF2VF_OFFSET(i);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_get_pf2vf_offset);
|
||||
|
||||
u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr)
|
||||
{
|
||||
u32 errsou3, errmsk3, vf_int_mask;
|
||||
|
||||
/* Get the interrupt sources triggered by VFs */
|
||||
errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
|
||||
vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3);
|
||||
|
||||
/* To avoid adding duplicate entries to work queue, clear
|
||||
* vf_int_mask_sets bits that are already masked in ERRMSK register.
|
||||
*/
|
||||
errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
|
||||
vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3);
|
||||
|
||||
return vf_int_mask;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_get_vf2pf_sources);
|
||||
|
||||
void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
|
||||
{
|
||||
/* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
|
||||
if (vf_mask & 0xFFFF) {
|
||||
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
|
||||
& ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_enable_vf2pf_interrupts);
|
||||
|
||||
void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask)
|
||||
{
|
||||
/* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
|
||||
if (vf_mask & 0xFFFF) {
|
||||
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
|
||||
| ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_disable_vf2pf_interrupts);
|
||||
|
||||
u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self)
|
||||
{
|
||||
if (!self || !self->accel_mask)
|
||||
|
@ -73,31 +26,29 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_num_aes);
|
|||
void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
struct adf_bar *misc_bar = &GET_BARS(accel_dev)
|
||||
[hw_data->get_misc_bar_id(hw_data)];
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
unsigned long accel_mask = hw_data->accel_mask;
|
||||
unsigned long ae_mask = hw_data->ae_mask;
|
||||
void __iomem *csr = misc_bar->virt_addr;
|
||||
unsigned int val, i;
|
||||
|
||||
/* Enable Accel Engine error detection & correction */
|
||||
for_each_set_bit(i, &ae_mask, hw_data->num_engines) {
|
||||
val = ADF_CSR_RD(csr, ADF_GEN2_AE_CTX_ENABLES(i));
|
||||
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i));
|
||||
val |= ADF_GEN2_ENABLE_AE_ECC_ERR;
|
||||
ADF_CSR_WR(csr, ADF_GEN2_AE_CTX_ENABLES(i), val);
|
||||
val = ADF_CSR_RD(csr, ADF_GEN2_AE_MISC_CONTROL(i));
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_CTX_ENABLES(i), val);
|
||||
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i));
|
||||
val |= ADF_GEN2_ENABLE_AE_ECC_PARITY_CORR;
|
||||
ADF_CSR_WR(csr, ADF_GEN2_AE_MISC_CONTROL(i), val);
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_AE_MISC_CONTROL(i), val);
|
||||
}
|
||||
|
||||
/* Enable shared memory error detection & correction */
|
||||
for_each_set_bit(i, &accel_mask, hw_data->num_accel) {
|
||||
val = ADF_CSR_RD(csr, ADF_GEN2_UERRSSMSH(i));
|
||||
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_UERRSSMSH(i));
|
||||
val |= ADF_GEN2_ERRSSMSH_EN;
|
||||
ADF_CSR_WR(csr, ADF_GEN2_UERRSSMSH(i), val);
|
||||
val = ADF_CSR_RD(csr, ADF_GEN2_CERRSSMSH(i));
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_UERRSSMSH(i), val);
|
||||
val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_CERRSSMSH(i));
|
||||
val |= ADF_GEN2_ERRSSMSH_EN;
|
||||
ADF_CSR_WR(csr, ADF_GEN2_CERRSSMSH(i), val);
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_CERRSSMSH(i), val);
|
||||
}
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction);
|
||||
|
@ -105,15 +56,9 @@ EXPORT_SYMBOL_GPL(adf_gen2_enable_error_correction);
|
|||
void adf_gen2_cfg_iov_thds(struct adf_accel_dev *accel_dev, bool enable,
|
||||
int num_a_regs, int num_b_regs)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
void __iomem *pmisc_addr;
|
||||
struct adf_bar *pmisc;
|
||||
int pmisc_id, i;
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
u32 reg;
|
||||
|
||||
pmisc_id = hw_data->get_misc_bar_id(hw_data);
|
||||
pmisc = &GET_BARS(accel_dev)[pmisc_id];
|
||||
pmisc_addr = pmisc->virt_addr;
|
||||
int i;
|
||||
|
||||
/* Set/Unset Valid bit in AE Thread to PCIe Function Mapping Group A */
|
||||
for (i = 0; i < num_a_regs; i++) {
|
||||
|
@ -259,21 +204,33 @@ u32 adf_gen2_get_accel_cap(struct adf_accel_dev *accel_dev)
|
|||
u32 legfuses;
|
||||
u32 capabilities = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
|
||||
ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
|
||||
ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
|
||||
ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
|
||||
ICP_ACCEL_CAPABILITIES_CIPHER |
|
||||
ICP_ACCEL_CAPABILITIES_COMPRESSION;
|
||||
|
||||
/* Read accelerator capabilities mask */
|
||||
pci_read_config_dword(pdev, ADF_DEVICE_LEGFUSE_OFFSET, &legfuses);
|
||||
|
||||
if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE)
|
||||
/* A set bit in legfuses means the feature is OFF in this SKU */
|
||||
if (legfuses & ICP_ACCEL_MASK_CIPHER_SLICE) {
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
|
||||
}
|
||||
if (legfuses & ICP_ACCEL_MASK_PKE_SLICE)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
|
||||
if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE)
|
||||
if (legfuses & ICP_ACCEL_MASK_AUTH_SLICE) {
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
|
||||
}
|
||||
if (legfuses & ICP_ACCEL_MASK_COMPRESS_SLICE)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
|
||||
|
||||
if ((straps | fuses) & ADF_POWERGATE_PKE)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
|
||||
|
||||
if ((straps | fuses) & ADF_POWERGATE_DC)
|
||||
capabilities &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
|
||||
|
||||
return capabilities;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap);
|
||||
|
@ -281,18 +238,12 @@ EXPORT_SYMBOL_GPL(adf_gen2_get_accel_cap);
|
|||
void adf_gen2_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
u32 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
|
||||
u32 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
|
||||
unsigned long accel_mask = hw_data->accel_mask;
|
||||
void __iomem *pmisc_addr;
|
||||
struct adf_bar *pmisc;
|
||||
int pmisc_id;
|
||||
u32 i = 0;
|
||||
|
||||
pmisc_id = hw_data->get_misc_bar_id(hw_data);
|
||||
pmisc = &GET_BARS(accel_dev)[pmisc_id];
|
||||
pmisc_addr = pmisc->virt_addr;
|
||||
|
||||
/* Configures WDT timers */
|
||||
for_each_set_bit(i, &accel_mask, hw_data->num_accel) {
|
||||
/* Enable WDT for sym and dc */
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
#define ADF_GEN2_HW_DATA_H_
|
||||
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_cfg_common.h"
|
||||
|
||||
/* Transport access */
|
||||
#define ADF_BANK_INT_SRC_SEL_MASK_0 0x4444444CUL
|
||||
|
@ -113,8 +114,16 @@ do { \
|
|||
(ADF_ARB_REG_SLOT * (index)), value)
|
||||
|
||||
/* Power gating */
|
||||
#define ADF_POWERGATE_DC BIT(23)
|
||||
#define ADF_POWERGATE_PKE BIT(24)
|
||||
|
||||
/* Default ring mapping */
|
||||
#define ADF_GEN2_DEFAULT_RING_TO_SRV_MAP \
|
||||
(CRYPTO << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
|
||||
CRYPTO << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
|
||||
UNUSED << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
|
||||
COMP << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
|
||||
|
||||
/* WDT timers
|
||||
*
|
||||
* Timeout is in cycles. Clock speed may vary across products but this
|
||||
|
@ -136,19 +145,6 @@ do { \
|
|||
#define ADF_GEN2_CERRSSMSH(i) ((i) * 0x4000 + 0x10)
|
||||
#define ADF_GEN2_ERRSSMSH_EN BIT(3)
|
||||
|
||||
/* VF2PF interrupts */
|
||||
#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C)
|
||||
#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8)
|
||||
#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C)
|
||||
#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC)
|
||||
#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
|
||||
#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9)
|
||||
|
||||
u32 adf_gen2_get_pf2vf_offset(u32 i);
|
||||
u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_bar);
|
||||
void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask);
|
||||
void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask);
|
||||
|
||||
u32 adf_gen2_get_num_accels(struct adf_hw_device_data *self);
|
||||
u32 adf_gen2_get_num_aes(struct adf_hw_device_data *self);
|
||||
void adf_gen2_enable_error_correction(struct adf_accel_dev *accel_dev);
|
||||
|
|
381
drivers/crypto/qat/qat_common/adf_gen2_pfvf.c
Normal file
381
drivers/crypto/qat/qat_common/adf_gen2_pfvf.c
Normal file
|
@ -0,0 +1,381 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#include <linux/delay.h>
|
||||
#include <linux/iopoll.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/types.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_common_drv.h"
|
||||
#include "adf_gen2_pfvf.h"
|
||||
#include "adf_pfvf_msg.h"
|
||||
#include "adf_pfvf_pf_proto.h"
|
||||
#include "adf_pfvf_vf_proto.h"
|
||||
#include "adf_pfvf_utils.h"
|
||||
|
||||
/* VF2PF interrupts */
|
||||
#define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9)
|
||||
#define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & 0xFFFF) << 9)
|
||||
|
||||
#define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04))
|
||||
#define ADF_GEN2_VF_PF2VF_OFFSET 0x200
|
||||
|
||||
#define ADF_GEN2_CSR_IN_USE 0x6AC2
|
||||
#define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE
|
||||
|
||||
enum gen2_csr_pos {
|
||||
ADF_GEN2_CSR_PF2VF_OFFSET = 0,
|
||||
ADF_GEN2_CSR_VF2PF_OFFSET = 16,
|
||||
};
|
||||
|
||||
#define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2
|
||||
#define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F
|
||||
#define ADF_PFVF_GEN2_MSGDATA_SHIFT 6
|
||||
#define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF
|
||||
|
||||
static const struct pfvf_csr_format csr_gen2_fmt = {
|
||||
{ ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK },
|
||||
{ ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK },
|
||||
};
|
||||
|
||||
#define ADF_PFVF_MSG_RETRY_DELAY 5
|
||||
#define ADF_PFVF_MSG_MAX_RETRIES 3
|
||||
|
||||
static u32 adf_gen2_pf_get_pfvf_offset(u32 i)
|
||||
{
|
||||
return ADF_GEN2_PF_PF2VF_OFFSET(i);
|
||||
}
|
||||
|
||||
static u32 adf_gen2_vf_get_pfvf_offset(u32 i)
|
||||
{
|
||||
return ADF_GEN2_VF_PF2VF_OFFSET;
|
||||
}
|
||||
|
||||
static u32 adf_gen2_get_vf2pf_sources(void __iomem *pmisc_addr)
|
||||
{
|
||||
u32 errsou3, errmsk3, vf_int_mask;
|
||||
|
||||
/* Get the interrupt sources triggered by VFs */
|
||||
errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3);
|
||||
vf_int_mask = ADF_GEN2_ERR_REG_VF2PF(errsou3);
|
||||
|
||||
/* To avoid adding duplicate entries to work queue, clear
|
||||
* vf_int_mask_sets bits that are already masked in ERRMSK register.
|
||||
*/
|
||||
errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3);
|
||||
vf_int_mask &= ~ADF_GEN2_ERR_REG_VF2PF(errmsk3);
|
||||
|
||||
return vf_int_mask;
|
||||
}
|
||||
|
||||
static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr,
|
||||
u32 vf_mask)
|
||||
{
|
||||
/* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */
|
||||
if (vf_mask & 0xFFFF) {
|
||||
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
|
||||
& ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
|
||||
}
|
||||
}
|
||||
|
||||
static void adf_gen2_disable_vf2pf_interrupts(void __iomem *pmisc_addr,
|
||||
u32 vf_mask)
|
||||
{
|
||||
/* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */
|
||||
if (vf_mask & 0xFFFF) {
|
||||
u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3)
|
||||
| ADF_GEN2_ERR_MSK_VF2PF(vf_mask);
|
||||
ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val);
|
||||
}
|
||||
}
|
||||
|
||||
static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset)
|
||||
{
|
||||
return ADF_PFVF_INT << offset;
|
||||
}
|
||||
|
||||
static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset)
|
||||
{
|
||||
return (csr_msg & 0xFFFF) << offset;
|
||||
}
|
||||
|
||||
static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset)
|
||||
{
|
||||
return (csr_val >> offset) & 0xFFFF;
|
||||
}
|
||||
|
||||
static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset)
|
||||
{
|
||||
return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE;
|
||||
}
|
||||
|
||||
static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset)
|
||||
{
|
||||
*msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset);
|
||||
}
|
||||
|
||||
static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset)
|
||||
{
|
||||
*msg |= (ADF_GEN2_CSR_IN_USE << offset);
|
||||
}
|
||||
|
||||
static bool is_legacy_user_pfvf_message(u32 msg)
|
||||
{
|
||||
return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM);
|
||||
}
|
||||
|
||||
static bool is_pf2vf_notification(u8 msg_type)
|
||||
{
|
||||
switch (msg_type) {
|
||||
case ADF_PF2VF_MSGTYPE_RESTARTING:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
static bool is_vf2pf_notification(u8 msg_type)
|
||||
{
|
||||
switch (msg_type) {
|
||||
case ADF_VF2PF_MSGTYPE_INIT:
|
||||
case ADF_VF2PF_MSGTYPE_SHUTDOWN:
|
||||
return true;
|
||||
default:
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
struct pfvf_gen2_params {
|
||||
u32 pfvf_offset;
|
||||
struct mutex *csr_lock; /* lock preventing concurrent access of CSR */
|
||||
enum gen2_csr_pos local_offset;
|
||||
enum gen2_csr_pos remote_offset;
|
||||
bool (*is_notification_message)(u8 msg_type);
|
||||
u8 compat_ver;
|
||||
};
|
||||
|
||||
static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev,
|
||||
struct pfvf_message msg,
|
||||
struct pfvf_gen2_params *params)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
enum gen2_csr_pos remote_offset = params->remote_offset;
|
||||
enum gen2_csr_pos local_offset = params->local_offset;
|
||||
unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES;
|
||||
struct mutex *lock = params->csr_lock;
|
||||
u32 pfvf_offset = params->pfvf_offset;
|
||||
u32 int_bit;
|
||||
u32 csr_val;
|
||||
u32 csr_msg;
|
||||
int ret;
|
||||
|
||||
/* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This
|
||||
* allows us to build and read messages as if they where all 0 based.
|
||||
* However, send and receive are in a single shared 32 bits register,
|
||||
* so we need to shift and/or mask the message half before decoding
|
||||
* it and after encoding it. Which one to shift depends on the
|
||||
* direction.
|
||||
*/
|
||||
|
||||
int_bit = gen2_csr_get_int_bit(local_offset);
|
||||
|
||||
csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt);
|
||||
if (unlikely(!csr_msg))
|
||||
return -EINVAL;
|
||||
|
||||
/* Prepare for CSR format, shifting the wire message in place and
|
||||
* setting the in use pattern
|
||||
*/
|
||||
csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset);
|
||||
gen2_csr_set_in_use(&csr_msg, remote_offset);
|
||||
|
||||
mutex_lock(lock);
|
||||
|
||||
start:
|
||||
/* Check if the PFVF CSR is in use by remote function */
|
||||
csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
|
||||
if (gen2_csr_is_in_use(csr_val, local_offset)) {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"PFVF CSR in use by remote function\n");
|
||||
goto retry;
|
||||
}
|
||||
|
||||
/* Attempt to get ownership of the PFVF CSR */
|
||||
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit);
|
||||
|
||||
/* Wait for confirmation from remote func it received the message */
|
||||
ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit),
|
||||
ADF_PFVF_MSG_ACK_DELAY_US,
|
||||
ADF_PFVF_MSG_ACK_MAX_DELAY_US,
|
||||
true, pmisc_addr, pfvf_offset);
|
||||
if (unlikely(ret < 0)) {
|
||||
dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
|
||||
csr_val &= ~int_bit;
|
||||
}
|
||||
|
||||
/* For fire-and-forget notifications, the receiver does not clear
|
||||
* the in-use pattern. This is used to detect collisions.
|
||||
*/
|
||||
if (params->is_notification_message(msg.type) && csr_val != csr_msg) {
|
||||
/* Collision must have overwritten the message */
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Collision on notification - PFVF CSR overwritten by remote function\n");
|
||||
goto retry;
|
||||
}
|
||||
|
||||
/* If the far side did not clear the in-use pattern it is either
|
||||
* 1) Notification - message left intact to detect collision
|
||||
* 2) Older protocol (compatibility version < 3) on the far side
|
||||
* where the sender is responsible for clearing the in-use
|
||||
* pattern after the received has acknowledged receipt.
|
||||
* In either case, clear the in-use pattern now.
|
||||
*/
|
||||
if (gen2_csr_is_in_use(csr_val, remote_offset)) {
|
||||
gen2_csr_clear_in_use(&csr_val, remote_offset);
|
||||
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
|
||||
}
|
||||
|
||||
out:
|
||||
mutex_unlock(lock);
|
||||
return ret;
|
||||
|
||||
retry:
|
||||
if (--retries) {
|
||||
msleep(ADF_PFVF_MSG_RETRY_DELAY);
|
||||
goto start;
|
||||
} else {
|
||||
ret = -EBUSY;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev,
|
||||
struct pfvf_gen2_params *params)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
enum gen2_csr_pos remote_offset = params->remote_offset;
|
||||
enum gen2_csr_pos local_offset = params->local_offset;
|
||||
u32 pfvf_offset = params->pfvf_offset;
|
||||
struct pfvf_message msg = { 0 };
|
||||
u32 int_bit;
|
||||
u32 csr_val;
|
||||
u16 csr_msg;
|
||||
|
||||
int_bit = gen2_csr_get_int_bit(local_offset);
|
||||
|
||||
/* Read message */
|
||||
csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
|
||||
if (!(csr_val & int_bit)) {
|
||||
dev_info(&GET_DEV(accel_dev),
|
||||
"Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val);
|
||||
return msg;
|
||||
}
|
||||
|
||||
/* Extract the message from the CSR */
|
||||
csr_msg = gen2_csr_msg_from_position(csr_val, local_offset);
|
||||
|
||||
/* Ignore legacy non-system (non-kernel) messages */
|
||||
if (unlikely(is_legacy_user_pfvf_message(csr_msg))) {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Ignored non-system message (0x%.8x);\n", csr_val);
|
||||
/* Because this must be a legacy message, the far side
|
||||
* must clear the in-use pattern, so don't do it.
|
||||
*/
|
||||
return msg;
|
||||
}
|
||||
|
||||
/* Return the pfvf_message format */
|
||||
msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt);
|
||||
|
||||
/* The in-use pattern is not cleared for notifications (so that
|
||||
* it can be used for collision detection) or older implementations
|
||||
*/
|
||||
if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK &&
|
||||
!params->is_notification_message(msg.type))
|
||||
gen2_csr_clear_in_use(&csr_val, remote_offset);
|
||||
|
||||
/* To ACK, clear the INT bit */
|
||||
csr_val &= ~int_bit;
|
||||
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val);
|
||||
|
||||
return msg;
|
||||
}
|
||||
|
||||
static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
|
||||
u32 pfvf_offset, struct mutex *csr_lock)
|
||||
{
|
||||
struct pfvf_gen2_params params = {
|
||||
.csr_lock = csr_lock,
|
||||
.pfvf_offset = pfvf_offset,
|
||||
.local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
|
||||
.remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
|
||||
.is_notification_message = is_pf2vf_notification,
|
||||
};
|
||||
|
||||
return adf_gen2_pfvf_send(accel_dev, msg, ¶ms);
|
||||
}
|
||||
|
||||
static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg,
|
||||
u32 pfvf_offset, struct mutex *csr_lock)
|
||||
{
|
||||
struct pfvf_gen2_params params = {
|
||||
.csr_lock = csr_lock,
|
||||
.pfvf_offset = pfvf_offset,
|
||||
.local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
|
||||
.remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
|
||||
.is_notification_message = is_vf2pf_notification,
|
||||
};
|
||||
|
||||
return adf_gen2_pfvf_send(accel_dev, msg, ¶ms);
|
||||
}
|
||||
|
||||
static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev,
|
||||
u32 pfvf_offset, u8 compat_ver)
|
||||
{
|
||||
struct pfvf_gen2_params params = {
|
||||
.pfvf_offset = pfvf_offset,
|
||||
.local_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
|
||||
.remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
|
||||
.is_notification_message = is_pf2vf_notification,
|
||||
.compat_ver = compat_ver,
|
||||
};
|
||||
|
||||
return adf_gen2_pfvf_recv(accel_dev, ¶ms);
|
||||
}
|
||||
|
||||
static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev,
|
||||
u32 pfvf_offset, u8 compat_ver)
|
||||
{
|
||||
struct pfvf_gen2_params params = {
|
||||
.pfvf_offset = pfvf_offset,
|
||||
.local_offset = ADF_GEN2_CSR_VF2PF_OFFSET,
|
||||
.remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET,
|
||||
.is_notification_message = is_vf2pf_notification,
|
||||
.compat_ver = compat_ver,
|
||||
};
|
||||
|
||||
return adf_gen2_pfvf_recv(accel_dev, ¶ms);
|
||||
}
|
||||
|
||||
void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
|
||||
{
|
||||
pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
|
||||
pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset;
|
||||
pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset;
|
||||
pfvf_ops->get_vf2pf_sources = adf_gen2_get_vf2pf_sources;
|
||||
pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts;
|
||||
pfvf_ops->disable_vf2pf_interrupts = adf_gen2_disable_vf2pf_interrupts;
|
||||
pfvf_ops->send_msg = adf_gen2_pf2vf_send;
|
||||
pfvf_ops->recv_msg = adf_gen2_vf2pf_recv;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops);
|
||||
|
||||
void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
|
||||
{
|
||||
pfvf_ops->enable_comms = adf_enable_vf2pf_comms;
|
||||
pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset;
|
||||
pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset;
|
||||
pfvf_ops->send_msg = adf_gen2_vf2pf_send;
|
||||
pfvf_ops->recv_msg = adf_gen2_pf2vf_recv;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops);
|
29
drivers/crypto/qat/qat_common/adf_gen2_pfvf.h
Normal file
29
drivers/crypto/qat/qat_common/adf_gen2_pfvf.h
Normal file
|
@ -0,0 +1,29 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#ifndef ADF_GEN2_PFVF_H
|
||||
#define ADF_GEN2_PFVF_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include "adf_accel_devices.h"
|
||||
|
||||
#define ADF_GEN2_ERRSOU3 (0x3A000 + 0x0C)
|
||||
#define ADF_GEN2_ERRSOU5 (0x3A000 + 0xD8)
|
||||
#define ADF_GEN2_ERRMSK3 (0x3A000 + 0x1C)
|
||||
#define ADF_GEN2_ERRMSK5 (0x3A000 + 0xDC)
|
||||
|
||||
#if defined(CONFIG_PCI_IOV)
|
||||
void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
|
||||
void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
|
||||
#else
|
||||
static inline void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
|
||||
{
|
||||
pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
|
||||
}
|
||||
|
||||
static inline void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
|
||||
{
|
||||
pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ADF_GEN2_PFVF_H */
|
|
@ -1,6 +1,8 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2020 Intel Corporation */
|
||||
#include <linux/iopoll.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_common_drv.h"
|
||||
#include "adf_gen4_hw_data.h"
|
||||
|
||||
static u64 build_csr_ring_base_addr(dma_addr_t addr, u32 size)
|
||||
|
@ -109,20 +111,13 @@ static inline void adf_gen4_unpack_ssm_wdtimer(u64 value, u32 *upper,
|
|||
|
||||
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
u64 timer_val_pke = ADF_SSM_WDT_PKE_DEFAULT_VALUE;
|
||||
u64 timer_val = ADF_SSM_WDT_DEFAULT_VALUE;
|
||||
u32 ssm_wdt_pke_high = 0;
|
||||
u32 ssm_wdt_pke_low = 0;
|
||||
u32 ssm_wdt_high = 0;
|
||||
u32 ssm_wdt_low = 0;
|
||||
void __iomem *pmisc_addr;
|
||||
struct adf_bar *pmisc;
|
||||
int pmisc_id;
|
||||
|
||||
pmisc_id = hw_data->get_misc_bar_id(hw_data);
|
||||
pmisc = &GET_BARS(accel_dev)[pmisc_id];
|
||||
pmisc_addr = pmisc->virt_addr;
|
||||
|
||||
/* Convert 64bit WDT timer value into 32bit values for
|
||||
* mmio write to 32bit CSRs.
|
||||
|
@ -139,3 +134,61 @@ void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev)
|
|||
ADF_CSR_WR(pmisc_addr, ADF_SSMWDTPKEH_OFFSET, ssm_wdt_pke_high);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen4_set_ssm_wdtimer);
|
||||
|
||||
int adf_pfvf_comms_disabled(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_pfvf_comms_disabled);
|
||||
|
||||
static int reset_ring_pair(void __iomem *csr, u32 bank_number)
|
||||
{
|
||||
u32 status;
|
||||
int ret;
|
||||
|
||||
/* Write rpresetctl register BIT(0) as 1
|
||||
* Since rpresetctl registers have no RW fields, no need to preserve
|
||||
* values for other bits. Just write directly.
|
||||
*/
|
||||
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETCTL(bank_number),
|
||||
ADF_WQM_CSR_RPRESETCTL_RESET);
|
||||
|
||||
/* Read rpresetsts register and wait for rp reset to complete */
|
||||
ret = read_poll_timeout(ADF_CSR_RD, status,
|
||||
status & ADF_WQM_CSR_RPRESETSTS_STATUS,
|
||||
ADF_RPRESET_POLL_DELAY_US,
|
||||
ADF_RPRESET_POLL_TIMEOUT_US, true,
|
||||
csr, ADF_WQM_CSR_RPRESETSTS(bank_number));
|
||||
if (!ret) {
|
||||
/* When rp reset is done, clear rpresetsts */
|
||||
ADF_CSR_WR(csr, ADF_WQM_CSR_RPRESETSTS(bank_number),
|
||||
ADF_WQM_CSR_RPRESETSTS_STATUS);
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
u32 etr_bar_id = hw_data->get_etr_bar_id(hw_data);
|
||||
void __iomem *csr;
|
||||
int ret;
|
||||
|
||||
if (bank_number >= hw_data->num_banks)
|
||||
return -EINVAL;
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"ring pair reset for bank:%d\n", bank_number);
|
||||
|
||||
csr = (&GET_BARS(accel_dev)[etr_bar_id])->virt_addr;
|
||||
ret = reset_ring_pair(csr, bank_number);
|
||||
if (ret)
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"ring pair reset failed (timeout)\n");
|
||||
else
|
||||
dev_dbg(&GET_DEV(accel_dev), "ring pair reset successful\n");
|
||||
|
||||
return ret;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen4_ring_pair_reset);
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
#define ADF_GEN4_HW_CSR_DATA_H_
|
||||
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_cfg_common.h"
|
||||
|
||||
/* Transport access */
|
||||
#define ADF_BANK_INT_SRC_SEL_MASK 0x44UL
|
||||
|
@ -94,6 +95,13 @@ do { \
|
|||
ADF_RING_BUNDLE_SIZE * (bank) + \
|
||||
ADF_RING_CSR_RING_SRV_ARB_EN, (value))
|
||||
|
||||
/* Default ring mapping */
|
||||
#define ADF_GEN4_DEFAULT_RING_TO_SRV_MAP \
|
||||
(ASYM << ADF_CFG_SERV_RING_PAIR_0_SHIFT | \
|
||||
SYM << ADF_CFG_SERV_RING_PAIR_1_SHIFT | \
|
||||
ASYM << ADF_CFG_SERV_RING_PAIR_2_SHIFT | \
|
||||
SYM << ADF_CFG_SERV_RING_PAIR_3_SHIFT)
|
||||
|
||||
/* WDT timers
|
||||
*
|
||||
* Timeout is in cycles. Clock speed may vary across products but this
|
||||
|
@ -106,6 +114,15 @@ do { \
|
|||
#define ADF_SSMWDTPKEL_OFFSET 0x58
|
||||
#define ADF_SSMWDTPKEH_OFFSET 0x60
|
||||
|
||||
/* Ring reset */
|
||||
#define ADF_RPRESET_POLL_TIMEOUT_US (5 * USEC_PER_SEC)
|
||||
#define ADF_RPRESET_POLL_DELAY_US 20
|
||||
#define ADF_WQM_CSR_RPRESETCTL_RESET BIT(0)
|
||||
#define ADF_WQM_CSR_RPRESETCTL(bank) (0x6000 + ((bank) << 3))
|
||||
#define ADF_WQM_CSR_RPRESETSTS_STATUS BIT(0)
|
||||
#define ADF_WQM_CSR_RPRESETSTS(bank) (ADF_WQM_CSR_RPRESETCTL(bank) + 4)
|
||||
|
||||
void adf_gen4_set_ssm_wdtimer(struct adf_accel_dev *accel_dev);
|
||||
void adf_gen4_init_hw_csr_ops(struct adf_hw_csr_ops *csr_ops);
|
||||
int adf_gen4_ring_pair_reset(struct adf_accel_dev *accel_dev, u32 bank_number);
|
||||
#endif
|
||||
|
|
148
drivers/crypto/qat/qat_common/adf_gen4_pfvf.c
Normal file
148
drivers/crypto/qat/qat_common/adf_gen4_pfvf.c
Normal file
|
@ -0,0 +1,148 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#include <linux/iopoll.h>
|
||||
#include <linux/mutex.h>
|
||||
#include <linux/types.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_common_drv.h"
|
||||
#include "adf_gen4_pfvf.h"
|
||||
#include "adf_pfvf_pf_proto.h"
|
||||
#include "adf_pfvf_utils.h"
|
||||
|
||||
#define ADF_4XXX_MAX_NUM_VFS 16
|
||||
|
||||
#define ADF_4XXX_PF2VM_OFFSET(i) (0x40B010 + ((i) * 0x20))
|
||||
#define ADF_4XXX_VM2PF_OFFSET(i) (0x40B014 + ((i) * 0x20))
|
||||
|
||||
/* VF2PF interrupt source registers */
|
||||
#define ADF_4XXX_VM2PF_SOU(i) (0x41A180 + ((i) * 4))
|
||||
#define ADF_4XXX_VM2PF_MSK(i) (0x41A1C0 + ((i) * 4))
|
||||
#define ADF_4XXX_VM2PF_INT_EN_MSK BIT(0)
|
||||
|
||||
#define ADF_PFVF_GEN4_MSGTYPE_SHIFT 2
|
||||
#define ADF_PFVF_GEN4_MSGTYPE_MASK 0x3F
|
||||
#define ADF_PFVF_GEN4_MSGDATA_SHIFT 8
|
||||
#define ADF_PFVF_GEN4_MSGDATA_MASK 0xFFFFFF
|
||||
|
||||
static const struct pfvf_csr_format csr_gen4_fmt = {
|
||||
{ ADF_PFVF_GEN4_MSGTYPE_SHIFT, ADF_PFVF_GEN4_MSGTYPE_MASK },
|
||||
{ ADF_PFVF_GEN4_MSGDATA_SHIFT, ADF_PFVF_GEN4_MSGDATA_MASK },
|
||||
};
|
||||
|
||||
static u32 adf_gen4_pf_get_pf2vf_offset(u32 i)
|
||||
{
|
||||
return ADF_4XXX_PF2VM_OFFSET(i);
|
||||
}
|
||||
|
||||
static u32 adf_gen4_pf_get_vf2pf_offset(u32 i)
|
||||
{
|
||||
return ADF_4XXX_VM2PF_OFFSET(i);
|
||||
}
|
||||
|
||||
static u32 adf_gen4_get_vf2pf_sources(void __iomem *pmisc_addr)
|
||||
{
|
||||
int i;
|
||||
u32 sou, mask;
|
||||
int num_csrs = ADF_4XXX_MAX_NUM_VFS;
|
||||
u32 vf_mask = 0;
|
||||
|
||||
for (i = 0; i < num_csrs; i++) {
|
||||
sou = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_SOU(i));
|
||||
mask = ADF_CSR_RD(pmisc_addr, ADF_4XXX_VM2PF_MSK(i));
|
||||
sou &= ~mask;
|
||||
vf_mask |= sou << i;
|
||||
}
|
||||
|
||||
return vf_mask;
|
||||
}
|
||||
|
||||
static void adf_gen4_enable_vf2pf_interrupts(void __iomem *pmisc_addr,
|
||||
u32 vf_mask)
|
||||
{
|
||||
int num_csrs = ADF_4XXX_MAX_NUM_VFS;
|
||||
unsigned long mask = vf_mask;
|
||||
unsigned int val;
|
||||
int i;
|
||||
|
||||
for_each_set_bit(i, &mask, num_csrs) {
|
||||
unsigned int offset = ADF_4XXX_VM2PF_MSK(i);
|
||||
|
||||
val = ADF_CSR_RD(pmisc_addr, offset) & ~ADF_4XXX_VM2PF_INT_EN_MSK;
|
||||
ADF_CSR_WR(pmisc_addr, offset, val);
|
||||
}
|
||||
}
|
||||
|
||||
static void adf_gen4_disable_vf2pf_interrupts(void __iomem *pmisc_addr,
|
||||
u32 vf_mask)
|
||||
{
|
||||
int num_csrs = ADF_4XXX_MAX_NUM_VFS;
|
||||
unsigned long mask = vf_mask;
|
||||
unsigned int val;
|
||||
int i;
|
||||
|
||||
for_each_set_bit(i, &mask, num_csrs) {
|
||||
unsigned int offset = ADF_4XXX_VM2PF_MSK(i);
|
||||
|
||||
val = ADF_CSR_RD(pmisc_addr, offset) | ADF_4XXX_VM2PF_INT_EN_MSK;
|
||||
ADF_CSR_WR(pmisc_addr, offset, val);
|
||||
}
|
||||
}
|
||||
|
||||
static int adf_gen4_pfvf_send(struct adf_accel_dev *accel_dev,
|
||||
struct pfvf_message msg, u32 pfvf_offset,
|
||||
struct mutex *csr_lock)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
u32 csr_val;
|
||||
int ret;
|
||||
|
||||
csr_val = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen4_fmt);
|
||||
if (unlikely(!csr_val))
|
||||
return -EINVAL;
|
||||
|
||||
mutex_lock(csr_lock);
|
||||
|
||||
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val | ADF_PFVF_INT);
|
||||
|
||||
/* Wait for confirmation from remote that it received the message */
|
||||
ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & ADF_PFVF_INT),
|
||||
ADF_PFVF_MSG_ACK_DELAY_US,
|
||||
ADF_PFVF_MSG_ACK_MAX_DELAY_US,
|
||||
true, pmisc_addr, pfvf_offset);
|
||||
if (ret < 0)
|
||||
dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
|
||||
|
||||
mutex_unlock(csr_lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static struct pfvf_message adf_gen4_pfvf_recv(struct adf_accel_dev *accel_dev,
|
||||
u32 pfvf_offset, u8 compat_ver)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
u32 csr_val;
|
||||
|
||||
/* Read message from the CSR */
|
||||
csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset);
|
||||
|
||||
/* We can now acknowledge the message reception by clearing the
|
||||
* interrupt bit
|
||||
*/
|
||||
ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val & ~ADF_PFVF_INT);
|
||||
|
||||
/* Return the pfvf_message format */
|
||||
return adf_pfvf_message_of(accel_dev, csr_val, &csr_gen4_fmt);
|
||||
}
|
||||
|
||||
void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
|
||||
{
|
||||
pfvf_ops->enable_comms = adf_enable_pf2vf_comms;
|
||||
pfvf_ops->get_pf2vf_offset = adf_gen4_pf_get_pf2vf_offset;
|
||||
pfvf_ops->get_vf2pf_offset = adf_gen4_pf_get_vf2pf_offset;
|
||||
pfvf_ops->get_vf2pf_sources = adf_gen4_get_vf2pf_sources;
|
||||
pfvf_ops->enable_vf2pf_interrupts = adf_gen4_enable_vf2pf_interrupts;
|
||||
pfvf_ops->disable_vf2pf_interrupts = adf_gen4_disable_vf2pf_interrupts;
|
||||
pfvf_ops->send_msg = adf_gen4_pfvf_send;
|
||||
pfvf_ops->recv_msg = adf_gen4_pfvf_recv;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_gen4_init_pf_pfvf_ops);
|
17
drivers/crypto/qat/qat_common/adf_gen4_pfvf.h
Normal file
17
drivers/crypto/qat/qat_common/adf_gen4_pfvf.h
Normal file
|
@ -0,0 +1,17 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#ifndef ADF_GEN4_PFVF_H
|
||||
#define ADF_GEN4_PFVF_H
|
||||
|
||||
#include "adf_accel_devices.h"
|
||||
|
||||
#ifdef CONFIG_PCI_IOV
|
||||
void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops);
|
||||
#else
|
||||
static inline void adf_gen4_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops)
|
||||
{
|
||||
pfvf_ops->enable_comms = adf_pfvf_comms_disabled;
|
||||
}
|
||||
#endif
|
||||
|
||||
#endif /* ADF_GEN4_PFVF_H */
|
|
@ -69,7 +69,8 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
|
|||
return -EFAULT;
|
||||
}
|
||||
|
||||
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) {
|
||||
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
|
||||
!accel_dev->is_vf) {
|
||||
dev_err(&GET_DEV(accel_dev), "Device not configured\n");
|
||||
return -EFAULT;
|
||||
}
|
||||
|
@ -117,10 +118,16 @@ int adf_dev_init(struct adf_accel_dev *accel_dev)
|
|||
hw_data->enable_ints(accel_dev);
|
||||
hw_data->enable_error_correction(accel_dev);
|
||||
|
||||
ret = hw_data->enable_pfvf_comms(accel_dev);
|
||||
ret = hw_data->pfvf_ops.enable_comms(accel_dev);
|
||||
if (ret)
|
||||
return ret;
|
||||
|
||||
if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status) &&
|
||||
accel_dev->is_vf) {
|
||||
if (qat_crypto_vf_dev_config(accel_dev))
|
||||
return -EFAULT;
|
||||
}
|
||||
|
||||
/*
|
||||
* Subservice initialisation is divided into two stages: init and start.
|
||||
* This is to facilitate any ordering dependencies between services
|
||||
|
|
|
@ -54,52 +54,83 @@ static irqreturn_t adf_msix_isr_bundle(int irq, void *bank_ptr)
|
|||
return IRQ_HANDLED;
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PCI_IOV
|
||||
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
GET_PFVF_OPS(accel_dev)->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
|
||||
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
}
|
||||
|
||||
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
|
||||
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
}
|
||||
|
||||
static void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
|
||||
u32 vf_mask)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
|
||||
spin_lock(&accel_dev->pf.vf2pf_ints_lock);
|
||||
GET_PFVF_OPS(accel_dev)->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
|
||||
spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
|
||||
}
|
||||
|
||||
static bool adf_handle_vf2pf_int(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev);
|
||||
bool irq_handled = false;
|
||||
unsigned long vf_mask;
|
||||
|
||||
/* Get the interrupt sources triggered by VFs */
|
||||
vf_mask = GET_PFVF_OPS(accel_dev)->get_vf2pf_sources(pmisc_addr);
|
||||
|
||||
if (vf_mask) {
|
||||
struct adf_accel_vf_info *vf_info;
|
||||
int i;
|
||||
|
||||
/* Disable VF2PF interrupts for VFs with pending ints */
|
||||
adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
|
||||
|
||||
/*
|
||||
* Handle VF2PF interrupt unless the VF is malicious and
|
||||
* is attempting to flood the host OS with VF2PF interrupts.
|
||||
*/
|
||||
for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
|
||||
vf_info = accel_dev->pf.vf_info + i;
|
||||
|
||||
if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
|
||||
dev_info(&GET_DEV(accel_dev),
|
||||
"Too many ints from VF%d\n",
|
||||
vf_info->vf_nr);
|
||||
continue;
|
||||
}
|
||||
|
||||
adf_schedule_vf2pf_handler(vf_info);
|
||||
irq_handled = true;
|
||||
}
|
||||
}
|
||||
return irq_handled;
|
||||
}
|
||||
#endif /* CONFIG_PCI_IOV */
|
||||
|
||||
static irqreturn_t adf_msix_isr_ae(int irq, void *dev_ptr)
|
||||
{
|
||||
struct adf_accel_dev *accel_dev = dev_ptr;
|
||||
|
||||
#ifdef CONFIG_PCI_IOV
|
||||
/* If SR-IOV is enabled (vf_info is non-NULL), check for VF->PF ints */
|
||||
if (accel_dev->pf.vf_info) {
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
struct adf_bar *pmisc =
|
||||
&GET_BARS(accel_dev)[hw_data->get_misc_bar_id(hw_data)];
|
||||
void __iomem *pmisc_addr = pmisc->virt_addr;
|
||||
unsigned long vf_mask;
|
||||
|
||||
/* Get the interrupt sources triggered by VFs */
|
||||
vf_mask = hw_data->get_vf2pf_sources(pmisc_addr);
|
||||
|
||||
if (vf_mask) {
|
||||
struct adf_accel_vf_info *vf_info;
|
||||
bool irq_handled = false;
|
||||
int i;
|
||||
|
||||
/* Disable VF2PF interrupts for VFs with pending ints */
|
||||
adf_disable_vf2pf_interrupts_irq(accel_dev, vf_mask);
|
||||
|
||||
/*
|
||||
* Handle VF2PF interrupt unless the VF is malicious and
|
||||
* is attempting to flood the host OS with VF2PF interrupts.
|
||||
*/
|
||||
for_each_set_bit(i, &vf_mask, ADF_MAX_NUM_VFS) {
|
||||
vf_info = accel_dev->pf.vf_info + i;
|
||||
|
||||
if (!__ratelimit(&vf_info->vf2pf_ratelimit)) {
|
||||
dev_info(&GET_DEV(accel_dev),
|
||||
"Too many ints from VF%d\n",
|
||||
vf_info->vf_nr + 1);
|
||||
continue;
|
||||
}
|
||||
|
||||
adf_schedule_vf2pf_handler(vf_info);
|
||||
irq_handled = true;
|
||||
}
|
||||
|
||||
if (irq_handled)
|
||||
return IRQ_HANDLED;
|
||||
}
|
||||
}
|
||||
if (accel_dev->pf.vf_info && adf_handle_vf2pf_int(accel_dev))
|
||||
return IRQ_HANDLED;
|
||||
#endif /* CONFIG_PCI_IOV */
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev), "qat_dev%d spurious AE interrupt\n",
|
||||
|
|
|
@ -1,416 +0,0 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2015 - 2020 Intel Corporation */
|
||||
#include <linux/delay.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_common_drv.h"
|
||||
#include "adf_pf2vf_msg.h"
|
||||
|
||||
#define ADF_PFVF_MSG_COLLISION_DETECT_DELAY 10
|
||||
#define ADF_PFVF_MSG_ACK_DELAY 2
|
||||
#define ADF_PFVF_MSG_ACK_MAX_RETRY 100
|
||||
#define ADF_PFVF_MSG_RETRY_DELAY 5
|
||||
#define ADF_PFVF_MSG_MAX_RETRIES 3
|
||||
#define ADF_PFVF_MSG_RESP_TIMEOUT (ADF_PFVF_MSG_ACK_DELAY * \
|
||||
ADF_PFVF_MSG_ACK_MAX_RETRY + \
|
||||
ADF_PFVF_MSG_COLLISION_DETECT_DELAY)
|
||||
|
||||
void adf_enable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
|
||||
struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
|
||||
void __iomem *pmisc_addr = pmisc->virt_addr;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
hw_data->enable_vf2pf_interrupts(pmisc_addr, vf_mask);
|
||||
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
}
|
||||
|
||||
void adf_disable_vf2pf_interrupts(struct adf_accel_dev *accel_dev, u32 vf_mask)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
|
||||
struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
|
||||
void __iomem *pmisc_addr = pmisc->virt_addr;
|
||||
unsigned long flags;
|
||||
|
||||
spin_lock_irqsave(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
|
||||
spin_unlock_irqrestore(&accel_dev->pf.vf2pf_ints_lock, flags);
|
||||
}
|
||||
|
||||
void adf_disable_vf2pf_interrupts_irq(struct adf_accel_dev *accel_dev,
|
||||
u32 vf_mask)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
u32 misc_bar_id = hw_data->get_misc_bar_id(hw_data);
|
||||
struct adf_bar *pmisc = &GET_BARS(accel_dev)[misc_bar_id];
|
||||
void __iomem *pmisc_addr = pmisc->virt_addr;
|
||||
|
||||
spin_lock(&accel_dev->pf.vf2pf_ints_lock);
|
||||
hw_data->disable_vf2pf_interrupts(pmisc_addr, vf_mask);
|
||||
spin_unlock(&accel_dev->pf.vf2pf_ints_lock);
|
||||
}
|
||||
|
||||
static int __adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
|
||||
{
|
||||
struct adf_accel_pci *pci_info = &accel_dev->accel_pci_dev;
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
void __iomem *pmisc_bar_addr =
|
||||
pci_info->pci_bars[hw_data->get_misc_bar_id(hw_data)].virt_addr;
|
||||
u32 val, pf2vf_offset, count = 0;
|
||||
u32 local_in_use_mask, local_in_use_pattern;
|
||||
u32 remote_in_use_mask, remote_in_use_pattern;
|
||||
struct mutex *lock; /* lock preventing concurrent acces of CSR */
|
||||
u32 int_bit;
|
||||
int ret = 0;
|
||||
|
||||
if (accel_dev->is_vf) {
|
||||
pf2vf_offset = hw_data->get_pf2vf_offset(0);
|
||||
lock = &accel_dev->vf.vf2pf_lock;
|
||||
local_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK;
|
||||
local_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF;
|
||||
remote_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK;
|
||||
remote_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF;
|
||||
int_bit = ADF_VF2PF_INT;
|
||||
} else {
|
||||
pf2vf_offset = hw_data->get_pf2vf_offset(vf_nr);
|
||||
lock = &accel_dev->pf.vf_info[vf_nr].pf2vf_lock;
|
||||
local_in_use_mask = ADF_PF2VF_IN_USE_BY_PF_MASK;
|
||||
local_in_use_pattern = ADF_PF2VF_IN_USE_BY_PF;
|
||||
remote_in_use_mask = ADF_VF2PF_IN_USE_BY_VF_MASK;
|
||||
remote_in_use_pattern = ADF_VF2PF_IN_USE_BY_VF;
|
||||
int_bit = ADF_PF2VF_INT;
|
||||
}
|
||||
|
||||
mutex_lock(lock);
|
||||
|
||||
/* Check if the PFVF CSR is in use by remote function */
|
||||
val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
|
||||
if ((val & remote_in_use_mask) == remote_in_use_pattern) {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"PFVF CSR in use by remote function\n");
|
||||
ret = -EBUSY;
|
||||
goto out;
|
||||
}
|
||||
|
||||
msg &= ~local_in_use_mask;
|
||||
msg |= local_in_use_pattern;
|
||||
|
||||
/* Attempt to get ownership of the PFVF CSR */
|
||||
ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, msg | int_bit);
|
||||
|
||||
/* Wait for confirmation from remote func it received the message */
|
||||
do {
|
||||
msleep(ADF_PFVF_MSG_ACK_DELAY);
|
||||
val = ADF_CSR_RD(pmisc_bar_addr, pf2vf_offset);
|
||||
} while ((val & int_bit) && (count++ < ADF_PFVF_MSG_ACK_MAX_RETRY));
|
||||
|
||||
if (val != msg) {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Collision - PFVF CSR overwritten by remote function\n");
|
||||
ret = -EIO;
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (val & int_bit) {
|
||||
dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n");
|
||||
val &= ~int_bit;
|
||||
ret = -EIO;
|
||||
}
|
||||
|
||||
/* Finished with the PFVF CSR; relinquish it and leave msg in CSR */
|
||||
ADF_CSR_WR(pmisc_bar_addr, pf2vf_offset, val & ~local_in_use_mask);
|
||||
out:
|
||||
mutex_unlock(lock);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_iov_putmsg() - send PFVF message
|
||||
* @accel_dev: Pointer to acceleration device.
|
||||
* @msg: Message to send
|
||||
* @vf_nr: VF number to which the message will be sent if on PF, ignored
|
||||
* otherwise
|
||||
*
|
||||
* Function sends a message through the PFVF channel
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
static int adf_iov_putmsg(struct adf_accel_dev *accel_dev, u32 msg, u8 vf_nr)
|
||||
{
|
||||
u32 count = 0;
|
||||
int ret;
|
||||
|
||||
do {
|
||||
ret = __adf_iov_putmsg(accel_dev, msg, vf_nr);
|
||||
if (ret)
|
||||
msleep(ADF_PFVF_MSG_RETRY_DELAY);
|
||||
} while (ret && (count++ < ADF_PFVF_MSG_MAX_RETRIES));
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_send_pf2vf_msg() - send PF to VF message
|
||||
* @accel_dev: Pointer to acceleration device
|
||||
* @vf_nr: VF number to which the message will be sent
|
||||
* @msg: Message to send
|
||||
*
|
||||
* This function allows the PF to send a message to a specific VF.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
static int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, u32 msg)
|
||||
{
|
||||
return adf_iov_putmsg(accel_dev, msg, vf_nr);
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_send_vf2pf_msg() - send VF to PF message
|
||||
* @accel_dev: Pointer to acceleration device
|
||||
* @msg: Message to send
|
||||
*
|
||||
* This function allows the VF to send a message to the PF.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
int adf_send_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 msg)
|
||||
{
|
||||
return adf_iov_putmsg(accel_dev, msg, 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_send_vf2pf_req() - send VF2PF request message
|
||||
* @accel_dev: Pointer to acceleration device.
|
||||
* @msg: Request message to send
|
||||
*
|
||||
* This function sends a message that requires a response from the VF to the PF
|
||||
* and waits for a reply.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
static int adf_send_vf2pf_req(struct adf_accel_dev *accel_dev, u32 msg)
|
||||
{
|
||||
unsigned long timeout = msecs_to_jiffies(ADF_PFVF_MSG_RESP_TIMEOUT);
|
||||
int ret;
|
||||
|
||||
reinit_completion(&accel_dev->vf.iov_msg_completion);
|
||||
|
||||
/* Send request from VF to PF */
|
||||
ret = adf_send_vf2pf_msg(accel_dev, msg);
|
||||
if (ret) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Failed to send request msg to PF\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Wait for response */
|
||||
if (!wait_for_completion_timeout(&accel_dev->vf.iov_msg_completion,
|
||||
timeout)) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"PFVF request/response message timeout expired\n");
|
||||
return -EIO;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
void adf_vf2pf_req_hndl(struct adf_accel_vf_info *vf_info)
|
||||
{
|
||||
struct adf_accel_dev *accel_dev = vf_info->accel_dev;
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
int bar_id = hw_data->get_misc_bar_id(hw_data);
|
||||
struct adf_bar *pmisc = &GET_BARS(accel_dev)[bar_id];
|
||||
void __iomem *pmisc_addr = pmisc->virt_addr;
|
||||
u32 msg, resp = 0, vf_nr = vf_info->vf_nr;
|
||||
|
||||
/* Read message from the VF */
|
||||
msg = ADF_CSR_RD(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr));
|
||||
if (!(msg & ADF_VF2PF_INT)) {
|
||||
dev_info(&GET_DEV(accel_dev),
|
||||
"Spurious VF2PF interrupt, msg %X. Ignored\n", msg);
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* To ACK, clear the VF2PFINT bit */
|
||||
msg &= ~ADF_VF2PF_INT;
|
||||
ADF_CSR_WR(pmisc_addr, hw_data->get_pf2vf_offset(vf_nr), msg);
|
||||
|
||||
if (!(msg & ADF_VF2PF_MSGORIGIN_SYSTEM))
|
||||
/* Ignore legacy non-system (non-kernel) VF2PF messages */
|
||||
goto err;
|
||||
|
||||
switch ((msg & ADF_VF2PF_MSGTYPE_MASK) >> ADF_VF2PF_MSGTYPE_SHIFT) {
|
||||
case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
|
||||
{
|
||||
u8 vf_compat_ver = msg >> ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
|
||||
|
||||
resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
|
||||
(ADF_PF2VF_MSGTYPE_VERSION_RESP <<
|
||||
ADF_PF2VF_MSGTYPE_SHIFT) |
|
||||
(ADF_PFVF_COMPAT_THIS_VERSION <<
|
||||
ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Compatibility Version Request from VF%d vers=%u\n",
|
||||
vf_nr + 1, vf_compat_ver);
|
||||
|
||||
if (vf_compat_ver < hw_data->min_iov_compat_ver) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"VF (vers %d) incompatible with PF (vers %d)\n",
|
||||
vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
|
||||
resp |= ADF_PF2VF_VF_INCOMPATIBLE <<
|
||||
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
|
||||
} else if (vf_compat_ver > ADF_PFVF_COMPAT_THIS_VERSION) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"VF (vers %d) compat with PF (vers %d) unkn.\n",
|
||||
vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
|
||||
resp |= ADF_PF2VF_VF_COMPAT_UNKNOWN <<
|
||||
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
|
||||
} else {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"VF (vers %d) compatible with PF (vers %d)\n",
|
||||
vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
|
||||
resp |= ADF_PF2VF_VF_COMPATIBLE <<
|
||||
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
|
||||
}
|
||||
}
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_VERSION_REQ:
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Legacy VersionRequest received from VF%d 0x%x\n",
|
||||
vf_nr + 1, msg);
|
||||
resp = (ADF_PF2VF_MSGORIGIN_SYSTEM |
|
||||
(ADF_PF2VF_MSGTYPE_VERSION_RESP <<
|
||||
ADF_PF2VF_MSGTYPE_SHIFT) |
|
||||
(ADF_PFVF_COMPAT_THIS_VERSION <<
|
||||
ADF_PF2VF_VERSION_RESP_VERS_SHIFT));
|
||||
resp |= ADF_PF2VF_VF_COMPATIBLE <<
|
||||
ADF_PF2VF_VERSION_RESP_RESULT_SHIFT;
|
||||
/* Set legacy major and minor version num */
|
||||
resp |= 1 << ADF_PF2VF_MAJORVERSION_SHIFT |
|
||||
1 << ADF_PF2VF_MINORVERSION_SHIFT;
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_INIT:
|
||||
{
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Init message received from VF%d 0x%x\n",
|
||||
vf_nr + 1, msg);
|
||||
vf_info->init = true;
|
||||
}
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_SHUTDOWN:
|
||||
{
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Shutdown message received from VF%d 0x%x\n",
|
||||
vf_nr + 1, msg);
|
||||
vf_info->init = false;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
goto err;
|
||||
}
|
||||
|
||||
if (resp && adf_send_pf2vf_msg(accel_dev, vf_nr, resp))
|
||||
dev_err(&GET_DEV(accel_dev), "Failed to send response to VF\n");
|
||||
|
||||
out:
|
||||
/* re-enable interrupt on PF from this VF */
|
||||
adf_enable_vf2pf_interrupts(accel_dev, (1 << vf_nr));
|
||||
|
||||
return;
|
||||
err:
|
||||
dev_dbg(&GET_DEV(accel_dev), "Unknown message from VF%d (0x%x);\n",
|
||||
vf_nr + 1, msg);
|
||||
}
|
||||
|
||||
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_accel_vf_info *vf;
|
||||
u32 msg = (ADF_PF2VF_MSGORIGIN_SYSTEM |
|
||||
(ADF_PF2VF_MSGTYPE_RESTARTING << ADF_PF2VF_MSGTYPE_SHIFT));
|
||||
int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev));
|
||||
|
||||
for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) {
|
||||
if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg))
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Failed to send restarting msg to VF%d\n", i);
|
||||
}
|
||||
}
|
||||
|
||||
static int adf_vf2pf_request_version(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
u32 msg = 0;
|
||||
int ret;
|
||||
|
||||
msg = ADF_VF2PF_MSGORIGIN_SYSTEM;
|
||||
msg |= ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ << ADF_VF2PF_MSGTYPE_SHIFT;
|
||||
msg |= ADF_PFVF_COMPAT_THIS_VERSION << ADF_VF2PF_COMPAT_VER_REQ_SHIFT;
|
||||
BUILD_BUG_ON(ADF_PFVF_COMPAT_THIS_VERSION > 255);
|
||||
|
||||
ret = adf_send_vf2pf_req(accel_dev, msg);
|
||||
if (ret) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Failed to send Compatibility Version Request.\n");
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* Response from PF received, check compatibility */
|
||||
switch (accel_dev->vf.compatible) {
|
||||
case ADF_PF2VF_VF_COMPATIBLE:
|
||||
break;
|
||||
case ADF_PF2VF_VF_COMPAT_UNKNOWN:
|
||||
/* VF is newer than PF and decides whether it is compatible */
|
||||
if (accel_dev->vf.pf_version >= hw_data->min_iov_compat_ver) {
|
||||
accel_dev->vf.compatible = ADF_PF2VF_VF_COMPATIBLE;
|
||||
break;
|
||||
}
|
||||
fallthrough;
|
||||
case ADF_PF2VF_VF_INCOMPATIBLE:
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"PF (vers %d) and VF (vers %d) are not compatible\n",
|
||||
accel_dev->vf.pf_version,
|
||||
ADF_PFVF_COMPAT_THIS_VERSION);
|
||||
return -EINVAL;
|
||||
default:
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Invalid response from PF; assume not compatible\n");
|
||||
return -EINVAL;
|
||||
}
|
||||
return ret;
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_enable_vf2pf_comms() - Function enables communication from vf to pf
|
||||
*
|
||||
* @accel_dev: Pointer to acceleration device virtual function.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
int adf_enable_vf2pf_comms(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
adf_enable_pf2vf_interrupts(accel_dev);
|
||||
return adf_vf2pf_request_version(accel_dev);
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_enable_vf2pf_comms);
|
||||
|
||||
/**
|
||||
* adf_enable_pf2vf_comms() - Function enables communication from pf to vf
|
||||
*
|
||||
* @accel_dev: Pointer to acceleration device virtual function.
|
||||
*
|
||||
* This function carries out the necessary steps to setup and start the PFVF
|
||||
* communication channel, if any.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
spin_lock_init(&accel_dev->pf.vf2pf_ints_lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms);
|
|
@ -1,93 +0,0 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2015 - 2020 Intel Corporation */
|
||||
#ifndef ADF_PF2VF_MSG_H
|
||||
#define ADF_PF2VF_MSG_H
|
||||
|
||||
/*
|
||||
* PF<->VF Messaging
|
||||
* The PF has an array of 32-bit PF2VF registers, one for each VF. The
|
||||
* PF can access all these registers; each VF can access only the one
|
||||
* register associated with that particular VF.
|
||||
*
|
||||
* The register functionally is split into two parts:
|
||||
* The bottom half is for PF->VF messages. In particular when the first
|
||||
* bit of this register (bit 0) gets set an interrupt will be triggered
|
||||
* in the respective VF.
|
||||
* The top half is for VF->PF messages. In particular when the first bit
|
||||
* of this half of register (bit 16) gets set an interrupt will be triggered
|
||||
* in the PF.
|
||||
*
|
||||
* The remaining bits within this register are available to encode messages.
|
||||
* and implement a collision control mechanism to prevent concurrent use of
|
||||
* the PF2VF register by both the PF and VF.
|
||||
*
|
||||
* 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
|
||||
* _______________________________________________
|
||||
* | | | | | | | | | | | | | | | | |
|
||||
* +-----------------------------------------------+
|
||||
* \___________________________/ \_________/ ^ ^
|
||||
* ^ ^ | |
|
||||
* | | | VF2PF Int
|
||||
* | | Message Origin
|
||||
* | Message Type
|
||||
* Message-specific Data/Reserved
|
||||
*
|
||||
* 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
|
||||
* _______________________________________________
|
||||
* | | | | | | | | | | | | | | | | |
|
||||
* +-----------------------------------------------+
|
||||
* \___________________________/ \_________/ ^ ^
|
||||
* ^ ^ | |
|
||||
* | | | PF2VF Int
|
||||
* | | Message Origin
|
||||
* | Message Type
|
||||
* Message-specific Data/Reserved
|
||||
*
|
||||
* Message Origin (Should always be 1)
|
||||
* A legacy out-of-tree QAT driver allowed for a set of messages not supported
|
||||
* by this driver; these had a Msg Origin of 0 and are ignored by this driver.
|
||||
*
|
||||
* When a PF or VF attempts to send a message in the lower or upper 16 bits,
|
||||
* respectively, the other 16 bits are written to first with a defined
|
||||
* IN_USE_BY pattern as part of a collision control scheme (see adf_iov_putmsg).
|
||||
*/
|
||||
|
||||
#define ADF_PFVF_COMPAT_THIS_VERSION 0x1 /* PF<->VF compat */
|
||||
|
||||
/* PF->VF messages */
|
||||
#define ADF_PF2VF_INT BIT(0)
|
||||
#define ADF_PF2VF_MSGORIGIN_SYSTEM BIT(1)
|
||||
#define ADF_PF2VF_MSGTYPE_MASK 0x0000003C
|
||||
#define ADF_PF2VF_MSGTYPE_SHIFT 2
|
||||
#define ADF_PF2VF_MSGTYPE_RESTARTING 0x01
|
||||
#define ADF_PF2VF_MSGTYPE_VERSION_RESP 0x02
|
||||
#define ADF_PF2VF_IN_USE_BY_PF 0x6AC20000
|
||||
#define ADF_PF2VF_IN_USE_BY_PF_MASK 0xFFFE0000
|
||||
|
||||
/* PF->VF Version Response */
|
||||
#define ADF_PF2VF_VERSION_RESP_VERS_MASK 0x00003FC0
|
||||
#define ADF_PF2VF_VERSION_RESP_VERS_SHIFT 6
|
||||
#define ADF_PF2VF_VERSION_RESP_RESULT_MASK 0x0000C000
|
||||
#define ADF_PF2VF_VERSION_RESP_RESULT_SHIFT 14
|
||||
#define ADF_PF2VF_MINORVERSION_SHIFT 6
|
||||
#define ADF_PF2VF_MAJORVERSION_SHIFT 10
|
||||
#define ADF_PF2VF_VF_COMPATIBLE 1
|
||||
#define ADF_PF2VF_VF_INCOMPATIBLE 2
|
||||
#define ADF_PF2VF_VF_COMPAT_UNKNOWN 3
|
||||
|
||||
/* VF->PF messages */
|
||||
#define ADF_VF2PF_IN_USE_BY_VF 0x00006AC2
|
||||
#define ADF_VF2PF_IN_USE_BY_VF_MASK 0x0000FFFE
|
||||
#define ADF_VF2PF_INT BIT(16)
|
||||
#define ADF_VF2PF_MSGORIGIN_SYSTEM BIT(17)
|
||||
#define ADF_VF2PF_MSGTYPE_MASK 0x003C0000
|
||||
#define ADF_VF2PF_MSGTYPE_SHIFT 18
|
||||
#define ADF_VF2PF_MSGTYPE_INIT 0x3
|
||||
#define ADF_VF2PF_MSGTYPE_SHUTDOWN 0x4
|
||||
#define ADF_VF2PF_MSGTYPE_VERSION_REQ 0x5
|
||||
#define ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ 0x6
|
||||
|
||||
/* VF->PF Compatible Version Request */
|
||||
#define ADF_VF2PF_COMPAT_VER_REQ_SHIFT 22
|
||||
|
||||
#endif /* ADF_IOV_MSG_H */
|
259
drivers/crypto/qat/qat_common/adf_pfvf_msg.h
Normal file
259
drivers/crypto/qat/qat_common/adf_pfvf_msg.h
Normal file
|
@ -0,0 +1,259 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2015 - 2021 Intel Corporation */
|
||||
#ifndef ADF_PFVF_MSG_H
|
||||
#define ADF_PFVF_MSG_H
|
||||
|
||||
#include <linux/bits.h>
|
||||
|
||||
/*
|
||||
* PF<->VF Gen2 Messaging format
|
||||
*
|
||||
* The PF has an array of 32-bit PF2VF registers, one for each VF. The
|
||||
* PF can access all these registers; each VF can access only the one
|
||||
* register associated with that particular VF.
|
||||
*
|
||||
* The register functionally is split into two parts:
|
||||
* The bottom half is for PF->VF messages. In particular when the first
|
||||
* bit of this register (bit 0) gets set an interrupt will be triggered
|
||||
* in the respective VF.
|
||||
* The top half is for VF->PF messages. In particular when the first bit
|
||||
* of this half of register (bit 16) gets set an interrupt will be triggered
|
||||
* in the PF.
|
||||
*
|
||||
* The remaining bits within this register are available to encode messages.
|
||||
* and implement a collision control mechanism to prevent concurrent use of
|
||||
* the PF2VF register by both the PF and VF.
|
||||
*
|
||||
* 31 30 29 28 27 26 25 24 23 22 21 20 19 18 17 16
|
||||
* _______________________________________________
|
||||
* | | | | | | | | | | | | | | | | |
|
||||
* +-----------------------------------------------+
|
||||
* \___________________________/ \_________/ ^ ^
|
||||
* ^ ^ | |
|
||||
* | | | VF2PF Int
|
||||
* | | Message Origin
|
||||
* | Message Type
|
||||
* Message-specific Data/Reserved
|
||||
*
|
||||
* 15 14 13 12 11 10 9 8 7 6 5 4 3 2 1 0
|
||||
* _______________________________________________
|
||||
* | | | | | | | | | | | | | | | | |
|
||||
* +-----------------------------------------------+
|
||||
* \___________________________/ \_________/ ^ ^
|
||||
* ^ ^ | |
|
||||
* | | | PF2VF Int
|
||||
* | | Message Origin
|
||||
* | Message Type
|
||||
* Message-specific Data/Reserved
|
||||
*
|
||||
* Message Origin (Should always be 1)
|
||||
* A legacy out-of-tree QAT driver allowed for a set of messages not supported
|
||||
* by this driver; these had a Msg Origin of 0 and are ignored by this driver.
|
||||
*
|
||||
* When a PF or VF attempts to send a message in the lower or upper 16 bits,
|
||||
* respectively, the other 16 bits are written to first with a defined
|
||||
* IN_USE_BY pattern as part of a collision control scheme (see function
|
||||
* adf_gen2_pfvf_send() in adf_pf2vf_msg.c).
|
||||
*
|
||||
*
|
||||
* PF<->VF Gen4 Messaging format
|
||||
*
|
||||
* Similarly to the gen2 messaging format, 32-bit long registers are used for
|
||||
* communication between PF and VFs. However, each VF and PF share a pair of
|
||||
* 32-bits register to avoid collisions: one for PV to VF messages and one
|
||||
* for VF to PF messages.
|
||||
*
|
||||
* Both the Interrupt bit and the Message Origin bit retain the same position
|
||||
* and meaning, although non-system messages are now deprecated and not
|
||||
* expected.
|
||||
*
|
||||
* 31 30 9 8 7 6 5 4 3 2 1 0
|
||||
* _______________________________________________
|
||||
* | | | . . . | | | | | | | | | | |
|
||||
* +-----------------------------------------------+
|
||||
* \_____________________/ \_______________/ ^ ^
|
||||
* ^ ^ | |
|
||||
* | | | PF/VF Int
|
||||
* | | Message Origin
|
||||
* | Message Type
|
||||
* Message-specific Data/Reserved
|
||||
*
|
||||
* For both formats, the message reception is acknowledged by lowering the
|
||||
* interrupt bit on the register where the message was sent.
|
||||
*/
|
||||
|
||||
/* PFVF message common bits */
|
||||
#define ADF_PFVF_INT BIT(0)
|
||||
#define ADF_PFVF_MSGORIGIN_SYSTEM BIT(1)
|
||||
|
||||
/* Different generations have different CSR layouts, use this struct
|
||||
* to abstract these differences away
|
||||
*/
|
||||
struct pfvf_message {
|
||||
u8 type;
|
||||
u32 data;
|
||||
};
|
||||
|
||||
/* PF->VF messages */
|
||||
enum pf2vf_msgtype {
|
||||
ADF_PF2VF_MSGTYPE_RESTARTING = 0x01,
|
||||
ADF_PF2VF_MSGTYPE_VERSION_RESP = 0x02,
|
||||
ADF_PF2VF_MSGTYPE_BLKMSG_RESP = 0x03,
|
||||
/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */
|
||||
ADF_PF2VF_MSGTYPE_RP_RESET_RESP = 0x10,
|
||||
};
|
||||
|
||||
/* VF->PF messages */
|
||||
enum vf2pf_msgtype {
|
||||
ADF_VF2PF_MSGTYPE_INIT = 0x03,
|
||||
ADF_VF2PF_MSGTYPE_SHUTDOWN = 0x04,
|
||||
ADF_VF2PF_MSGTYPE_VERSION_REQ = 0x05,
|
||||
ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ = 0x06,
|
||||
ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ = 0x07,
|
||||
ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ = 0x08,
|
||||
ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ = 0x09,
|
||||
/* Values from 0x10 are Gen4 specific, message type is only 4 bits in Gen2 devices. */
|
||||
ADF_VF2PF_MSGTYPE_RP_RESET = 0x10,
|
||||
};
|
||||
|
||||
/* VF/PF compatibility version. */
|
||||
enum pfvf_compatibility_version {
|
||||
/* Support for extended capabilities */
|
||||
ADF_PFVF_COMPAT_CAPABILITIES = 0x02,
|
||||
/* In-use pattern cleared by receiver */
|
||||
ADF_PFVF_COMPAT_FAST_ACK = 0x03,
|
||||
/* Ring to service mapping support for non-standard mappings */
|
||||
ADF_PFVF_COMPAT_RING_TO_SVC_MAP = 0x04,
|
||||
/* Reference to the latest version */
|
||||
ADF_PFVF_COMPAT_THIS_VERSION = 0x04,
|
||||
};
|
||||
|
||||
/* PF->VF Version Response */
|
||||
#define ADF_PF2VF_VERSION_RESP_VERS_MASK GENMASK(7, 0)
|
||||
#define ADF_PF2VF_VERSION_RESP_RESULT_MASK GENMASK(9, 8)
|
||||
|
||||
enum pf2vf_compat_response {
|
||||
ADF_PF2VF_VF_COMPATIBLE = 0x01,
|
||||
ADF_PF2VF_VF_INCOMPATIBLE = 0x02,
|
||||
ADF_PF2VF_VF_COMPAT_UNKNOWN = 0x03,
|
||||
};
|
||||
|
||||
enum ring_reset_result {
|
||||
RPRESET_SUCCESS = 0x00,
|
||||
RPRESET_NOT_SUPPORTED = 0x01,
|
||||
RPRESET_INVAL_BANK = 0x02,
|
||||
RPRESET_TIMEOUT = 0x03,
|
||||
};
|
||||
|
||||
#define ADF_VF2PF_RNG_RESET_RP_MASK GENMASK(1, 0)
|
||||
#define ADF_VF2PF_RNG_RESET_RSVD_MASK GENMASK(25, 2)
|
||||
|
||||
/* PF->VF Block Responses */
|
||||
#define ADF_PF2VF_BLKMSG_RESP_TYPE_MASK GENMASK(1, 0)
|
||||
#define ADF_PF2VF_BLKMSG_RESP_DATA_MASK GENMASK(9, 2)
|
||||
|
||||
enum pf2vf_blkmsg_resp_type {
|
||||
ADF_PF2VF_BLKMSG_RESP_TYPE_DATA = 0x00,
|
||||
ADF_PF2VF_BLKMSG_RESP_TYPE_CRC = 0x01,
|
||||
ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR = 0x02,
|
||||
};
|
||||
|
||||
/* PF->VF Block Error Code */
|
||||
enum pf2vf_blkmsg_error {
|
||||
ADF_PF2VF_INVALID_BLOCK_TYPE = 0x00,
|
||||
ADF_PF2VF_INVALID_BYTE_NUM_REQ = 0x01,
|
||||
ADF_PF2VF_PAYLOAD_TRUNCATED = 0x02,
|
||||
ADF_PF2VF_UNSPECIFIED_ERROR = 0x03,
|
||||
};
|
||||
|
||||
/* VF->PF Block Requests */
|
||||
#define ADF_VF2PF_LARGE_BLOCK_TYPE_MASK GENMASK(1, 0)
|
||||
#define ADF_VF2PF_LARGE_BLOCK_BYTE_MASK GENMASK(8, 2)
|
||||
#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK GENMASK(2, 0)
|
||||
#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK GENMASK(8, 3)
|
||||
#define ADF_VF2PF_SMALL_BLOCK_TYPE_MASK GENMASK(3, 0)
|
||||
#define ADF_VF2PF_SMALL_BLOCK_BYTE_MASK GENMASK(8, 4)
|
||||
#define ADF_VF2PF_BLOCK_CRC_REQ_MASK BIT(9)
|
||||
|
||||
/* PF->VF Block Request Types
|
||||
* 0..15 - 32 byte message
|
||||
* 16..23 - 64 byte message
|
||||
* 24..27 - 128 byte message
|
||||
*/
|
||||
enum vf2pf_blkmsg_req_type {
|
||||
ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY = 0x02,
|
||||
ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP = 0x03,
|
||||
};
|
||||
|
||||
#define ADF_VF2PF_SMALL_BLOCK_TYPE_MAX \
|
||||
(FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK))
|
||||
|
||||
#define ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX \
|
||||
(FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK) + \
|
||||
ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1)
|
||||
|
||||
#define ADF_VF2PF_LARGE_BLOCK_TYPE_MAX \
|
||||
(FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK) + \
|
||||
ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX)
|
||||
|
||||
#define ADF_VF2PF_SMALL_BLOCK_BYTE_MAX \
|
||||
FIELD_MAX(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK)
|
||||
|
||||
#define ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX \
|
||||
FIELD_MAX(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK)
|
||||
|
||||
#define ADF_VF2PF_LARGE_BLOCK_BYTE_MAX \
|
||||
FIELD_MAX(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK)
|
||||
|
||||
struct pfvf_blkmsg_header {
|
||||
u8 version;
|
||||
u8 payload_size;
|
||||
} __packed;
|
||||
|
||||
#define ADF_PFVF_BLKMSG_HEADER_SIZE (sizeof(struct pfvf_blkmsg_header))
|
||||
#define ADF_PFVF_BLKMSG_PAYLOAD_SIZE(blkmsg) (sizeof(blkmsg) - \
|
||||
ADF_PFVF_BLKMSG_HEADER_SIZE)
|
||||
#define ADF_PFVF_BLKMSG_MSG_SIZE(blkmsg) (ADF_PFVF_BLKMSG_HEADER_SIZE + \
|
||||
(blkmsg)->hdr.payload_size)
|
||||
#define ADF_PFVF_BLKMSG_MSG_MAX_SIZE 128
|
||||
|
||||
/* PF->VF Block message header bytes */
|
||||
#define ADF_PFVF_BLKMSG_VER_BYTE 0
|
||||
#define ADF_PFVF_BLKMSG_LEN_BYTE 1
|
||||
|
||||
/* PF/VF Capabilities message values */
|
||||
enum blkmsg_capabilities_versions {
|
||||
ADF_PFVF_CAPABILITIES_V1_VERSION = 0x01,
|
||||
ADF_PFVF_CAPABILITIES_V2_VERSION = 0x02,
|
||||
ADF_PFVF_CAPABILITIES_V3_VERSION = 0x03,
|
||||
};
|
||||
|
||||
struct capabilities_v1 {
|
||||
struct pfvf_blkmsg_header hdr;
|
||||
u32 ext_dc_caps;
|
||||
} __packed;
|
||||
|
||||
struct capabilities_v2 {
|
||||
struct pfvf_blkmsg_header hdr;
|
||||
u32 ext_dc_caps;
|
||||
u32 capabilities;
|
||||
} __packed;
|
||||
|
||||
struct capabilities_v3 {
|
||||
struct pfvf_blkmsg_header hdr;
|
||||
u32 ext_dc_caps;
|
||||
u32 capabilities;
|
||||
u32 frequency;
|
||||
} __packed;
|
||||
|
||||
/* PF/VF Ring to service mapping values */
|
||||
enum blkmsg_ring_to_svc_versions {
|
||||
ADF_PFVF_RING_TO_SVC_VERSION = 0x01,
|
||||
};
|
||||
|
||||
struct ring_to_svc_map_v1 {
|
||||
struct pfvf_blkmsg_header hdr;
|
||||
u16 map;
|
||||
} __packed;
|
||||
|
||||
#endif /* ADF_PFVF_MSG_H */
|
52
drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c
Normal file
52
drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.c
Normal file
|
@ -0,0 +1,52 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2015 - 2021 Intel Corporation */
|
||||
#include <linux/pci.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_pfvf_msg.h"
|
||||
#include "adf_pfvf_pf_msg.h"
|
||||
#include "adf_pfvf_pf_proto.h"
|
||||
|
||||
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
struct adf_accel_vf_info *vf;
|
||||
struct pfvf_message msg = { .type = ADF_PF2VF_MSGTYPE_RESTARTING };
|
||||
int i, num_vfs = pci_num_vf(accel_to_pci_dev(accel_dev));
|
||||
|
||||
for (i = 0, vf = accel_dev->pf.vf_info; i < num_vfs; i++, vf++) {
|
||||
if (vf->init && adf_send_pf2vf_msg(accel_dev, i, msg))
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Failed to send restarting msg to VF%d\n", i);
|
||||
}
|
||||
}
|
||||
|
||||
int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev,
|
||||
u8 *buffer, u8 compat)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
struct capabilities_v2 caps_msg;
|
||||
|
||||
caps_msg.ext_dc_caps = hw_data->extended_dc_capabilities;
|
||||
caps_msg.capabilities = hw_data->accel_capabilities_mask;
|
||||
|
||||
caps_msg.hdr.version = ADF_PFVF_CAPABILITIES_V2_VERSION;
|
||||
caps_msg.hdr.payload_size =
|
||||
ADF_PFVF_BLKMSG_PAYLOAD_SIZE(struct capabilities_v2);
|
||||
|
||||
memcpy(buffer, &caps_msg, sizeof(caps_msg));
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev,
|
||||
u8 *buffer, u8 compat)
|
||||
{
|
||||
struct ring_to_svc_map_v1 rts_map_msg;
|
||||
|
||||
rts_map_msg.map = accel_dev->hw_device->ring_to_svc_map;
|
||||
rts_map_msg.hdr.version = ADF_PFVF_RING_TO_SVC_VERSION;
|
||||
rts_map_msg.hdr.payload_size = ADF_PFVF_BLKMSG_PAYLOAD_SIZE(rts_map_msg);
|
||||
|
||||
memcpy(buffer, &rts_map_msg, sizeof(rts_map_msg));
|
||||
|
||||
return 0;
|
||||
}
|
18
drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h
Normal file
18
drivers/crypto/qat/qat_common/adf_pfvf_pf_msg.h
Normal file
|
@ -0,0 +1,18 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#ifndef ADF_PFVF_PF_MSG_H
|
||||
#define ADF_PFVF_PF_MSG_H
|
||||
|
||||
#include "adf_accel_devices.h"
|
||||
|
||||
void adf_pf2vf_notify_restarting(struct adf_accel_dev *accel_dev);
|
||||
|
||||
typedef int (*adf_pf2vf_blkmsg_provider)(struct adf_accel_dev *accel_dev,
|
||||
u8 *buffer, u8 compat);
|
||||
|
||||
int adf_pf_capabilities_msg_provider(struct adf_accel_dev *accel_dev,
|
||||
u8 *buffer, u8 comapt);
|
||||
int adf_pf_ring_to_svc_msg_provider(struct adf_accel_dev *accel_dev,
|
||||
u8 *buffer, u8 comapt);
|
||||
|
||||
#endif /* ADF_PFVF_PF_MSG_H */
|
346
drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c
Normal file
346
drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.c
Normal file
|
@ -0,0 +1,346 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2015 - 2021 Intel Corporation */
|
||||
#include <linux/bitfield.h>
|
||||
#include <linux/spinlock.h>
|
||||
#include <linux/types.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_common_drv.h"
|
||||
#include "adf_pfvf_msg.h"
|
||||
#include "adf_pfvf_pf_msg.h"
|
||||
#include "adf_pfvf_pf_proto.h"
|
||||
#include "adf_pfvf_utils.h"
|
||||
|
||||
typedef u8 (*pf2vf_blkmsg_data_getter_fn)(u8 const *blkmsg, u8 byte);
|
||||
|
||||
static const adf_pf2vf_blkmsg_provider pf2vf_blkmsg_providers[] = {
|
||||
NULL, /* no message type defined for value 0 */
|
||||
NULL, /* no message type defined for value 1 */
|
||||
adf_pf_capabilities_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_CAP_SUMMARY */
|
||||
adf_pf_ring_to_svc_msg_provider, /* ADF_VF2PF_BLKMSG_REQ_RING_SVC_MAP */
|
||||
};
|
||||
|
||||
/**
|
||||
* adf_send_pf2vf_msg() - send PF to VF message
|
||||
* @accel_dev: Pointer to acceleration device
|
||||
* @vf_nr: VF number to which the message will be sent
|
||||
* @msg: Message to send
|
||||
*
|
||||
* This function allows the PF to send a message to a specific VF.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg)
|
||||
{
|
||||
struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
|
||||
u32 pfvf_offset = pfvf_ops->get_pf2vf_offset(vf_nr);
|
||||
|
||||
return pfvf_ops->send_msg(accel_dev, msg, pfvf_offset,
|
||||
&accel_dev->pf.vf_info[vf_nr].pf2vf_lock);
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_recv_vf2pf_msg() - receive a VF to PF message
|
||||
* @accel_dev: Pointer to acceleration device
|
||||
* @vf_nr: Number of the VF from where the message will be received
|
||||
*
|
||||
* This function allows the PF to receive a message from a specific VF.
|
||||
*
|
||||
* Return: a valid message on success, zero otherwise.
|
||||
*/
|
||||
static struct pfvf_message adf_recv_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr)
|
||||
{
|
||||
struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
|
||||
struct adf_pfvf_ops *pfvf_ops = GET_PFVF_OPS(accel_dev);
|
||||
u32 pfvf_offset = pfvf_ops->get_vf2pf_offset(vf_nr);
|
||||
|
||||
return pfvf_ops->recv_msg(accel_dev, pfvf_offset, vf_info->vf_compat_ver);
|
||||
}
|
||||
|
||||
static adf_pf2vf_blkmsg_provider get_blkmsg_response_provider(u8 type)
|
||||
{
|
||||
if (type >= ARRAY_SIZE(pf2vf_blkmsg_providers))
|
||||
return NULL;
|
||||
|
||||
return pf2vf_blkmsg_providers[type];
|
||||
}
|
||||
|
||||
/* Byte pf2vf_blkmsg_data_getter_fn callback */
|
||||
static u8 adf_pf2vf_blkmsg_get_byte(u8 const *blkmsg, u8 index)
|
||||
{
|
||||
return blkmsg[index];
|
||||
}
|
||||
|
||||
/* CRC pf2vf_blkmsg_data_getter_fn callback */
|
||||
static u8 adf_pf2vf_blkmsg_get_crc(u8 const *blkmsg, u8 count)
|
||||
{
|
||||
/* count is 0-based, turn it into a length */
|
||||
return adf_pfvf_calc_blkmsg_crc(blkmsg, count + 1);
|
||||
}
|
||||
|
||||
static int adf_pf2vf_blkmsg_get_data(struct adf_accel_vf_info *vf_info,
|
||||
u8 type, u8 byte, u8 max_size, u8 *data,
|
||||
pf2vf_blkmsg_data_getter_fn data_getter)
|
||||
{
|
||||
u8 blkmsg[ADF_PFVF_BLKMSG_MSG_MAX_SIZE] = { 0 };
|
||||
struct adf_accel_dev *accel_dev = vf_info->accel_dev;
|
||||
adf_pf2vf_blkmsg_provider provider;
|
||||
u8 msg_size;
|
||||
|
||||
provider = get_blkmsg_response_provider(type);
|
||||
|
||||
if (unlikely(!provider)) {
|
||||
pr_err("QAT: No registered provider for message %d\n", type);
|
||||
*data = ADF_PF2VF_INVALID_BLOCK_TYPE;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (unlikely((*provider)(accel_dev, blkmsg, vf_info->vf_compat_ver))) {
|
||||
pr_err("QAT: unknown error from provider for message %d\n", type);
|
||||
*data = ADF_PF2VF_UNSPECIFIED_ERROR;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
msg_size = ADF_PFVF_BLKMSG_HEADER_SIZE + blkmsg[ADF_PFVF_BLKMSG_LEN_BYTE];
|
||||
|
||||
if (unlikely(msg_size >= max_size)) {
|
||||
pr_err("QAT: Invalid size %d provided for message type %d\n",
|
||||
msg_size, type);
|
||||
*data = ADF_PF2VF_PAYLOAD_TRUNCATED;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
if (unlikely(byte >= msg_size)) {
|
||||
pr_err("QAT: Out-of-bound byte number %d (msg size %d)\n",
|
||||
byte, msg_size);
|
||||
*data = ADF_PF2VF_INVALID_BYTE_NUM_REQ;
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
*data = data_getter(blkmsg, byte);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static struct pfvf_message handle_blkmsg_req(struct adf_accel_vf_info *vf_info,
|
||||
struct pfvf_message req)
|
||||
{
|
||||
u8 resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_ERROR;
|
||||
struct pfvf_message resp = { 0 };
|
||||
u8 resp_data = 0;
|
||||
u8 blk_type;
|
||||
u8 blk_byte;
|
||||
u8 byte_max;
|
||||
|
||||
switch (req.type) {
|
||||
case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ:
|
||||
blk_type = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_TYPE_MASK, req.data)
|
||||
+ ADF_VF2PF_MEDIUM_BLOCK_TYPE_MAX + 1;
|
||||
blk_byte = FIELD_GET(ADF_VF2PF_LARGE_BLOCK_BYTE_MASK, req.data);
|
||||
byte_max = ADF_VF2PF_LARGE_BLOCK_BYTE_MAX;
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ:
|
||||
blk_type = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_TYPE_MASK, req.data)
|
||||
+ ADF_VF2PF_SMALL_BLOCK_TYPE_MAX + 1;
|
||||
blk_byte = FIELD_GET(ADF_VF2PF_MEDIUM_BLOCK_BYTE_MASK, req.data);
|
||||
byte_max = ADF_VF2PF_MEDIUM_BLOCK_BYTE_MAX;
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ:
|
||||
blk_type = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_TYPE_MASK, req.data);
|
||||
blk_byte = FIELD_GET(ADF_VF2PF_SMALL_BLOCK_BYTE_MASK, req.data);
|
||||
byte_max = ADF_VF2PF_SMALL_BLOCK_BYTE_MAX;
|
||||
break;
|
||||
}
|
||||
|
||||
/* Is this a request for CRC or data? */
|
||||
if (FIELD_GET(ADF_VF2PF_BLOCK_CRC_REQ_MASK, req.data)) {
|
||||
dev_dbg(&GET_DEV(vf_info->accel_dev),
|
||||
"BlockMsg of type %d for CRC over %d bytes received from VF%d\n",
|
||||
blk_type, blk_byte, vf_info->vf_nr);
|
||||
|
||||
if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte,
|
||||
byte_max, &resp_data,
|
||||
adf_pf2vf_blkmsg_get_crc))
|
||||
resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_CRC;
|
||||
} else {
|
||||
dev_dbg(&GET_DEV(vf_info->accel_dev),
|
||||
"BlockMsg of type %d for data byte %d received from VF%d\n",
|
||||
blk_type, blk_byte, vf_info->vf_nr);
|
||||
|
||||
if (!adf_pf2vf_blkmsg_get_data(vf_info, blk_type, blk_byte,
|
||||
byte_max, &resp_data,
|
||||
adf_pf2vf_blkmsg_get_byte))
|
||||
resp_type = ADF_PF2VF_BLKMSG_RESP_TYPE_DATA;
|
||||
}
|
||||
|
||||
resp.type = ADF_PF2VF_MSGTYPE_BLKMSG_RESP;
|
||||
resp.data = FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_TYPE_MASK, resp_type) |
|
||||
FIELD_PREP(ADF_PF2VF_BLKMSG_RESP_DATA_MASK, resp_data);
|
||||
|
||||
return resp;
|
||||
}
|
||||
|
||||
static struct pfvf_message handle_rp_reset_req(struct adf_accel_dev *accel_dev, u8 vf_nr,
|
||||
struct pfvf_message req)
|
||||
{
|
||||
struct adf_hw_device_data *hw_data = accel_dev->hw_device;
|
||||
struct pfvf_message resp = {
|
||||
.type = ADF_PF2VF_MSGTYPE_RP_RESET_RESP,
|
||||
.data = RPRESET_SUCCESS
|
||||
};
|
||||
u32 bank_number;
|
||||
u32 rsvd_field;
|
||||
|
||||
bank_number = FIELD_GET(ADF_VF2PF_RNG_RESET_RP_MASK, req.data);
|
||||
rsvd_field = FIELD_GET(ADF_VF2PF_RNG_RESET_RSVD_MASK, req.data);
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Ring Pair Reset Message received from VF%d for bank 0x%x\n",
|
||||
vf_nr, bank_number);
|
||||
|
||||
if (!hw_data->ring_pair_reset || rsvd_field) {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Ring Pair Reset for VF%d is not supported\n", vf_nr);
|
||||
resp.data = RPRESET_NOT_SUPPORTED;
|
||||
goto out;
|
||||
}
|
||||
|
||||
if (bank_number >= hw_data->num_banks_per_vf) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Invalid bank number (0x%x) from VF%d for Ring Reset\n",
|
||||
bank_number, vf_nr);
|
||||
resp.data = RPRESET_INVAL_BANK;
|
||||
goto out;
|
||||
}
|
||||
|
||||
/* Convert the VF provided value to PF bank number */
|
||||
bank_number = vf_nr * hw_data->num_banks_per_vf + bank_number;
|
||||
if (hw_data->ring_pair_reset(accel_dev, bank_number)) {
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Ring pair reset for VF%d failure\n", vf_nr);
|
||||
resp.data = RPRESET_TIMEOUT;
|
||||
goto out;
|
||||
}
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Ring pair reset for VF%d successfully\n", vf_nr);
|
||||
|
||||
out:
|
||||
return resp;
|
||||
}
|
||||
|
||||
static int adf_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr,
|
||||
struct pfvf_message msg, struct pfvf_message *resp)
|
||||
{
|
||||
struct adf_accel_vf_info *vf_info = &accel_dev->pf.vf_info[vf_nr];
|
||||
|
||||
switch (msg.type) {
|
||||
case ADF_VF2PF_MSGTYPE_COMPAT_VER_REQ:
|
||||
{
|
||||
u8 vf_compat_ver = msg.data;
|
||||
u8 compat;
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"VersionRequest received from VF%d (vers %d) to PF (vers %d)\n",
|
||||
vf_nr, vf_compat_ver, ADF_PFVF_COMPAT_THIS_VERSION);
|
||||
|
||||
if (vf_compat_ver <= ADF_PFVF_COMPAT_THIS_VERSION)
|
||||
compat = ADF_PF2VF_VF_COMPATIBLE;
|
||||
else
|
||||
compat = ADF_PF2VF_VF_COMPAT_UNKNOWN;
|
||||
|
||||
vf_info->vf_compat_ver = vf_compat_ver;
|
||||
|
||||
resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP;
|
||||
resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK,
|
||||
ADF_PFVF_COMPAT_THIS_VERSION) |
|
||||
FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat);
|
||||
}
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_VERSION_REQ:
|
||||
{
|
||||
u8 compat;
|
||||
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Legacy VersionRequest received from VF%d to PF (vers 1.1)\n",
|
||||
vf_nr);
|
||||
|
||||
/* legacy driver, VF compat_ver is 0 */
|
||||
vf_info->vf_compat_ver = 0;
|
||||
|
||||
/* PF always newer than legacy VF */
|
||||
compat = ADF_PF2VF_VF_COMPATIBLE;
|
||||
|
||||
/* Set legacy major and minor version to the latest, 1.1 */
|
||||
resp->type = ADF_PF2VF_MSGTYPE_VERSION_RESP;
|
||||
resp->data = FIELD_PREP(ADF_PF2VF_VERSION_RESP_VERS_MASK, 0x11) |
|
||||
FIELD_PREP(ADF_PF2VF_VERSION_RESP_RESULT_MASK, compat);
|
||||
}
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_INIT:
|
||||
{
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Init message received from VF%d\n", vf_nr);
|
||||
vf_info->init = true;
|
||||
}
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_SHUTDOWN:
|
||||
{
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Shutdown message received from VF%d\n", vf_nr);
|
||||
vf_info->init = false;
|
||||
}
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_LARGE_BLOCK_REQ:
|
||||
case ADF_VF2PF_MSGTYPE_MEDIUM_BLOCK_REQ:
|
||||
case ADF_VF2PF_MSGTYPE_SMALL_BLOCK_REQ:
|
||||
*resp = handle_blkmsg_req(vf_info, msg);
|
||||
break;
|
||||
case ADF_VF2PF_MSGTYPE_RP_RESET:
|
||||
*resp = handle_rp_reset_req(accel_dev, vf_nr, msg);
|
||||
break;
|
||||
default:
|
||||
dev_dbg(&GET_DEV(accel_dev),
|
||||
"Unknown message from VF%d (type 0x%.4x, data: 0x%.4x)\n",
|
||||
vf_nr, msg.type, msg.data);
|
||||
return -ENOMSG;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
bool adf_recv_and_handle_vf2pf_msg(struct adf_accel_dev *accel_dev, u32 vf_nr)
|
||||
{
|
||||
struct pfvf_message req;
|
||||
struct pfvf_message resp = {0};
|
||||
|
||||
req = adf_recv_vf2pf_msg(accel_dev, vf_nr);
|
||||
if (!req.type) /* Legacy or no message */
|
||||
return true;
|
||||
|
||||
if (adf_handle_vf2pf_msg(accel_dev, vf_nr, req, &resp))
|
||||
return false;
|
||||
|
||||
if (resp.type && adf_send_pf2vf_msg(accel_dev, vf_nr, resp))
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Failed to send response to VF%d\n", vf_nr);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* adf_enable_pf2vf_comms() - Function enables communication from pf to vf
|
||||
*
|
||||
* @accel_dev: Pointer to acceleration device virtual function.
|
||||
*
|
||||
* This function carries out the necessary steps to setup and start the PFVF
|
||||
* communication channel, if any.
|
||||
*
|
||||
* Return: 0 on success, error code otherwise.
|
||||
*/
|
||||
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev)
|
||||
{
|
||||
adf_pfvf_crc_init();
|
||||
spin_lock_init(&accel_dev->pf.vf2pf_ints_lock);
|
||||
|
||||
return 0;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(adf_enable_pf2vf_comms);
|
13
drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h
Normal file
13
drivers/crypto/qat/qat_common/adf_pfvf_pf_proto.h
Normal file
|
@ -0,0 +1,13 @@
|
|||
/* SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) */
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#ifndef ADF_PFVF_PF_PROTO_H
|
||||
#define ADF_PFVF_PF_PROTO_H
|
||||
|
||||
#include <linux/types.h>
|
||||
#include "adf_accel_devices.h"
|
||||
|
||||
int adf_send_pf2vf_msg(struct adf_accel_dev *accel_dev, u8 vf_nr, struct pfvf_message msg);
|
||||
|
||||
int adf_enable_pf2vf_comms(struct adf_accel_dev *accel_dev);
|
||||
|
||||
#endif /* ADF_PFVF_PF_PROTO_H */
|
65
drivers/crypto/qat/qat_common/adf_pfvf_utils.c
Normal file
65
drivers/crypto/qat/qat_common/adf_pfvf_utils.c
Normal file
|
@ -0,0 +1,65 @@
|
|||
// SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
|
||||
/* Copyright(c) 2021 Intel Corporation */
|
||||
#include <linux/crc8.h>
|
||||
#include <linux/pci.h>
|
||||
#include <linux/types.h>
|
||||
#include "adf_accel_devices.h"
|
||||
#include "adf_pfvf_msg.h"
|
||||
#include "adf_pfvf_utils.h"
|
||||
|
||||
/* CRC Calculation */
|
||||
DECLARE_CRC8_TABLE(pfvf_crc8_table);
|
||||
#define ADF_PFVF_CRC8_POLYNOMIAL 0x97
|
||||
|
||||
void adf_pfvf_crc_init(void)
|
||||
{
|
||||
crc8_populate_msb(pfvf_crc8_table, ADF_PFVF_CRC8_POLYNOMIAL);
|
||||
}
|
||||
|
||||
u8 adf_pfvf_calc_blkmsg_crc(u8 const *buf, u8 buf_len)
|
||||
{
|
||||
return crc8(pfvf_crc8_table, buf, buf_len, CRC8_INIT_VALUE);
|
||||
}
|
||||
|
||||
static bool set_value_on_csr_msg(struct adf_accel_dev *accel_dev, u32 *csr_msg,
|
||||
u32 value, const struct pfvf_field_format *fmt)
|
||||
{
|
||||
if (unlikely((value & fmt->mask) != value)) {
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"PFVF message value 0x%X out of range, %u max allowed\n",
|
||||
value, fmt->mask);
|
||||
return false;
|
||||
}
|
||||
|
||||
*csr_msg |= value << fmt->offset;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
u32 adf_pfvf_csr_msg_of(struct adf_accel_dev *accel_dev,
|
||||
struct pfvf_message msg,
|
||||
const struct pfvf_csr_format *fmt)
|
||||
{
|
||||
u32 csr_msg = 0;
|
||||
|
||||
if (!set_value_on_csr_msg(accel_dev, &csr_msg, msg.type, &fmt->type) ||
|
||||
!set_value_on_csr_msg(accel_dev, &csr_msg, msg.data, &fmt->data))
|
||||
return 0;
|
||||
|
||||
return csr_msg | ADF_PFVF_MSGORIGIN_SYSTEM;
|
||||
}
|
||||
|
||||
struct pfvf_message adf_pfvf_message_of(struct adf_accel_dev *accel_dev, u32 csr_msg,
|
||||
const struct pfvf_csr_format *fmt)
|
||||
{
|
||||
struct pfvf_message msg = { 0 };
|
||||
|
||||
msg.type = (csr_msg >> fmt->type.offset) & fmt->type.mask;
|
||||
msg.data = (csr_msg >> fmt->data.offset) & fmt->data.mask;
|
||||
|
||||
if (unlikely(!msg.type))
|
||||
dev_err(&GET_DEV(accel_dev),
|
||||
"Invalid PFVF msg with no type received\n");
|
||||
|
||||
return msg;
|
||||
}
|
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue