From 914c4fc1b792dd963f9606b2e8b33e3d6606eb84 Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Wed, 23 Oct 2019 09:04:59 +0300 Subject: [PATCH 1/2] mlxsw: spectrum: Use guaranteed buffer size as pool size limit There are two resources associated with shared buffer size: cap_total_buffer_size, and cap_guaranteed_shared_buffer. So far, mlxsw has been using the former as a limit to determine how large a pool size is allowed to be. However, the total size also includes headrooms and reserved space, which really cannot be used for shared buffer pools. Therefore convert mlxsw to use the latter resource as a limit. Adjust hard-coded pool sizes to be the guaranteed size minus 256000 bytes for CPU port pool. On Spectrum-1 that actually leads to an increase. A follow-up patch will have this size calculated automatically. Signed-off-by: Petr Machata Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- drivers/net/ethernet/mellanox/mlxsw/resources.h | 4 ++-- .../ethernet/mellanox/mlxsw/spectrum_buffers.c | 15 ++++++++------- .../net/ethernet/mellanox/mlxsw/spectrum_qdisc.c | 3 ++- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/resources.h b/drivers/net/ethernet/mellanox/mlxsw/resources.h index 33a9fc9ef6a4..85f919fe851b 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/resources.h +++ b/drivers/net/ethernet/mellanox/mlxsw/resources.h @@ -26,7 +26,7 @@ enum mlxsw_res_id { MLXSW_RES_ID_MAX_LAG_MEMBERS, MLXSW_RES_ID_LOCAL_PORTS_IN_1X, MLXSW_RES_ID_LOCAL_PORTS_IN_2X, - MLXSW_RES_ID_MAX_BUFFER_SIZE, + MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER, MLXSW_RES_ID_CELL_SIZE, MLXSW_RES_ID_MAX_HEADROOM_SIZE, MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS, @@ -82,7 +82,7 @@ static u16 mlxsw_res_ids[] = { [MLXSW_RES_ID_MAX_LAG_MEMBERS] = 0x2521, [MLXSW_RES_ID_LOCAL_PORTS_IN_1X] = 0x2610, [MLXSW_RES_ID_LOCAL_PORTS_IN_2X] = 0x2611, - [MLXSW_RES_ID_MAX_BUFFER_SIZE] = 0x2802, /* Bytes */ + [MLXSW_RES_ID_GUARANTEED_SHARED_BUFFER] = 0x2805, /* Bytes */ [MLXSW_RES_ID_CELL_SIZE] = 0x2803, /* Bytes */ [MLXSW_RES_ID_MAX_HEADROOM_SIZE] = 0x2811, /* Bytes */ [MLXSW_RES_ID_ACL_MAX_TCAM_REGIONS] = 0x2901, diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c index b9eeae37a4dc..637151682cf2 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c @@ -421,8 +421,8 @@ static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp) .freeze_size = _freeze_size, \ } -#define MLXSW_SP1_SB_PR_INGRESS_SIZE 12440000 -#define MLXSW_SP1_SB_PR_EGRESS_SIZE 13232000 +#define MLXSW_SP1_SB_PR_INGRESS_SIZE 13768608 +#define MLXSW_SP1_SB_PR_EGRESS_SIZE 13768608 #define MLXSW_SP1_SB_PR_CPU_SIZE (256 * 1000) /* Order according to mlxsw_sp1_sb_pool_dess */ @@ -445,8 +445,8 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { MLXSW_SP1_SB_PR_CPU_SIZE, true, false), }; -#define MLXSW_SP2_SB_PR_INGRESS_SIZE 35297568 -#define MLXSW_SP2_SB_PR_EGRESS_SIZE 35297568 +#define MLXSW_SP2_SB_PR_INGRESS_SIZE 34084800 +#define MLXSW_SP2_SB_PR_EGRESS_SIZE 34084800 #define MLXSW_SP2_SB_PR_CPU_SIZE (256 * 1000) /* Order according to mlxsw_sp2_sb_pool_dess */ @@ -904,7 +904,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, CELL_SIZE)) return -EIO; - if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_BUFFER_SIZE)) + if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, GUARANTEED_SHARED_BUFFER)) return -EIO; if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_HEADROOM_SIZE)) @@ -915,7 +915,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) return -ENOMEM; mlxsw_sp->sb->cell_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, CELL_SIZE); mlxsw_sp->sb->sb_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, - MAX_BUFFER_SIZE); + GUARANTEED_SHARED_BUFFER); max_headroom_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_HEADROOM_SIZE); /* Round down, because this limit must not be overstepped. */ @@ -1013,7 +1013,8 @@ int mlxsw_sp_sb_pool_set(struct mlxsw_core *mlxsw_core, mode = (enum mlxsw_reg_sbpr_mode) threshold_type; pr = &mlxsw_sp->sb_vals->prs[pool_index]; - if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) { + if (size > MLXSW_CORE_RES_GET(mlxsw_sp->core, + GUARANTEED_SHARED_BUFFER)) { NL_SET_ERR_MSG_MOD(extack, "Exceeded shared buffer size"); return -EINVAL; } diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c index bdf53cf350f6..68cc6737d45c 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_qdisc.c @@ -305,7 +305,8 @@ mlxsw_sp_qdisc_red_check_params(struct mlxsw_sp_port *mlxsw_sp_port, p->max); return -EINVAL; } - if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_BUFFER_SIZE)) { + if (p->max > MLXSW_CORE_RES_GET(mlxsw_sp->core, + GUARANTEED_SHARED_BUFFER)) { dev_err(mlxsw_sp->bus_info->dev, "spectrum: RED: max value %u is too big\n", p->max); return -EINVAL; From bc9f6e94bcb5652860c1b3bc82ca27697c496b4d Mon Sep 17 00:00:00 2001 From: Petr Machata Date: Wed, 23 Oct 2019 09:05:00 +0300 Subject: [PATCH 2/2] mlxsw: spectrum_buffers: Calculate the size of the main pool Instead of hard-coding the size of the largest pool, calculate it from the reported guaranteed shared buffer size and sizes of other pools (currently only the CPU port pool). Signed-off-by: Petr Machata Signed-off-by: Ido Schimmel Signed-off-by: David S. Miller --- .../mellanox/mlxsw/spectrum_buffers.c | 46 ++++++++++++++----- 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c index 637151682cf2..5fd9a72c8471 100644 --- a/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c +++ b/drivers/net/ethernet/mellanox/mlxsw/spectrum_buffers.c @@ -35,6 +35,7 @@ struct mlxsw_sp_sb_cm { }; #define MLXSW_SP_SB_INFI -1U +#define MLXSW_SP_SB_REST -2U struct mlxsw_sp_sb_pm { u32 min_buff; @@ -421,19 +422,16 @@ static void mlxsw_sp_sb_ports_fini(struct mlxsw_sp *mlxsw_sp) .freeze_size = _freeze_size, \ } -#define MLXSW_SP1_SB_PR_INGRESS_SIZE 13768608 -#define MLXSW_SP1_SB_PR_EGRESS_SIZE 13768608 #define MLXSW_SP1_SB_PR_CPU_SIZE (256 * 1000) /* Order according to mlxsw_sp1_sb_pool_dess */ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP1_SB_PR_INGRESS_SIZE), + MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), - MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP1_SB_PR_EGRESS_SIZE, true, false), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST, + true, false), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, 0), @@ -445,19 +443,16 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp1_sb_prs[] = { MLXSW_SP1_SB_PR_CPU_SIZE, true, false), }; -#define MLXSW_SP2_SB_PR_INGRESS_SIZE 34084800 -#define MLXSW_SP2_SB_PR_EGRESS_SIZE 34084800 #define MLXSW_SP2_SB_PR_CPU_SIZE (256 * 1000) /* Order according to mlxsw_sp2_sb_pool_dess */ static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = { - MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP2_SB_PR_INGRESS_SIZE), + MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), - MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, - MLXSW_SP2_SB_PR_EGRESS_SIZE, true, false), + MLXSW_SP_SB_PR_EXT(MLXSW_REG_SBPR_MODE_DYNAMIC, MLXSW_SP_SB_REST, + true, false), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), MLXSW_SP_SB_PR(MLXSW_REG_SBPR_MODE_STATIC, 0), @@ -471,11 +466,33 @@ static const struct mlxsw_sp_sb_pr mlxsw_sp2_sb_prs[] = { static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, const struct mlxsw_sp_sb_pr *prs, + const struct mlxsw_sp_sb_pool_des *pool_dess, size_t prs_len) { + /* Round down, unlike mlxsw_sp_bytes_cells(). */ + u32 sb_cells = mlxsw_sp->sb->sb_size / mlxsw_sp->sb->cell_size; + u32 rest_cells[2] = {sb_cells, sb_cells}; int i; int err; + /* Calculate how much space to give to the "REST" pools in either + * direction. + */ + for (i = 0; i < prs_len; i++) { + enum mlxsw_reg_sbxx_dir dir = pool_dess[i].dir; + u32 size = prs[i].size; + u32 size_cells; + + if (size == MLXSW_SP_SB_INFI || size == MLXSW_SP_SB_REST) + continue; + + size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size); + if (WARN_ON_ONCE(size_cells > rest_cells[dir])) + continue; + + rest_cells[dir] -= size_cells; + } + for (i = 0; i < prs_len; i++) { u32 size = prs[i].size; u32 size_cells; @@ -483,6 +500,10 @@ static int mlxsw_sp_sb_prs_init(struct mlxsw_sp *mlxsw_sp, if (size == MLXSW_SP_SB_INFI) { err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode, 0, true); + } else if (size == MLXSW_SP_SB_REST) { + size_cells = rest_cells[pool_dess[i].dir]; + err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode, + size_cells, false); } else { size_cells = mlxsw_sp_bytes_cells(mlxsw_sp, size); err = mlxsw_sp_sb_pr_write(mlxsw_sp, i, prs[i].mode, @@ -926,6 +947,7 @@ int mlxsw_sp_buffers_init(struct mlxsw_sp *mlxsw_sp) if (err) goto err_sb_ports_init; err = mlxsw_sp_sb_prs_init(mlxsw_sp, mlxsw_sp->sb_vals->prs, + mlxsw_sp->sb_vals->pool_dess, mlxsw_sp->sb_vals->pool_count); if (err) goto err_sb_prs_init;