Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: talitos - fix SEC1 32k ahash request limitation
Since commit c662b043cdca ("crypto: af_alg/hash: Support
MSG_SPLICE_PAGES"), the crypto core may pass large scatterlists spanning
multiple pages to drivers supporting ahash operations. As a result, a
driver can now receive large ahash requests.
The SEC1 engine has a limitation where a single descriptor cannot
process more than 32k of data. The current implementation attempts to
handle the entire request within a single descriptor, which leads to
failures raised by the driver:
"length exceeds h/w max limit"
Address this limitation by splitting large ahash requests into multiple
descriptors, each respecting the 32k hardware limit. This allows
processing arbitrarily large requests.
Cc: stable@vger.kernel.org
Fixes: c662b043cdca ("crypto: af_alg/hash: Support MSG_SPLICE_PAGES")
Signed-off-by: Paul Louvel <paul.louvel@bootlin.com>
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
authored by
Paul Louvel
and committed by
Herbert Xu
655ef638
01d798e9
+194
-116
+194
-116
drivers/crypto/talitos.c
+194
-116
drivers/crypto/talitos.c
···
12
12
* All rights reserved.
13
13
*/
14
14
15
+
#include <linux/workqueue.h>
15
16
#include <linux/kernel.h>
16
17
#include <linux/module.h>
17
18
#include <linux/mod_devicetable.h>
···
871
870
unsigned int swinit;
872
871
unsigned int first;
873
872
unsigned int last;
873
+
unsigned int last_request;
874
874
unsigned int to_hash_later;
875
875
unsigned int nbuf;
876
876
struct scatterlist bufsl[2];
877
877
struct scatterlist *psrc;
878
+
879
+
struct scatterlist request_bufsl[2];
880
+
struct ahash_request *areq;
881
+
struct scatterlist *request_sl;
882
+
unsigned int remaining_ahash_request_bytes;
883
+
unsigned int current_ahash_request_bytes;
884
+
struct work_struct sec1_ahash_process_remaining;
878
885
};
879
886
880
887
struct talitos_export_state {
···
1768
1759
1769
1760
kfree(edesc);
1770
1761
1771
-
ahash_request_complete(areq, err);
1762
+
if (err) {
1763
+
ahash_request_complete(areq, err);
1764
+
return;
1765
+
}
1766
+
1767
+
req_ctx->remaining_ahash_request_bytes -=
1768
+
req_ctx->current_ahash_request_bytes;
1769
+
1770
+
if (!req_ctx->remaining_ahash_request_bytes) {
1771
+
ahash_request_complete(areq, 0);
1772
+
return;
1773
+
}
1774
+
1775
+
schedule_work(&req_ctx->sec1_ahash_process_remaining);
1772
1776
}
1773
1777
1774
1778
/*
···
1947
1925
nbytes, 0, 0, 0, areq->base.flags, false);
1948
1926
}
1949
1927
1928
+
static int ahash_process_req_one(struct ahash_request *areq, unsigned int nbytes)
1929
+
{
1930
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
1931
+
struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
1932
+
struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
1933
+
struct talitos_edesc *edesc;
1934
+
unsigned int blocksize =
1935
+
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
1936
+
unsigned int nbytes_to_hash;
1937
+
unsigned int to_hash_later;
1938
+
unsigned int nsg;
1939
+
int nents;
1940
+
struct device *dev = ctx->dev;
1941
+
struct talitos_private *priv = dev_get_drvdata(dev);
1942
+
bool is_sec1 = has_ftr_sec1(priv);
1943
+
u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
1944
+
1945
+
if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
1946
+
/* Buffer up to one whole block */
1947
+
nents = sg_nents_for_len(req_ctx->request_sl, nbytes);
1948
+
if (nents < 0) {
1949
+
dev_err(dev, "Invalid number of src SG.\n");
1950
+
return nents;
1951
+
}
1952
+
sg_copy_to_buffer(req_ctx->request_sl, nents,
1953
+
ctx_buf + req_ctx->nbuf, nbytes);
1954
+
req_ctx->nbuf += nbytes;
1955
+
return 0;
1956
+
}
1957
+
1958
+
/* At least (blocksize + 1) bytes are available to hash */
1959
+
nbytes_to_hash = nbytes + req_ctx->nbuf;
1960
+
to_hash_later = nbytes_to_hash & (blocksize - 1);
1961
+
1962
+
if (req_ctx->last)
1963
+
to_hash_later = 0;
1964
+
else if (to_hash_later)
1965
+
/* There is a partial block. Hash the full block(s) now */
1966
+
nbytes_to_hash -= to_hash_later;
1967
+
else {
1968
+
/* Keep one block buffered */
1969
+
nbytes_to_hash -= blocksize;
1970
+
to_hash_later = blocksize;
1971
+
}
1972
+
1973
+
/* Chain in any previously buffered data */
1974
+
if (!is_sec1 && req_ctx->nbuf) {
1975
+
nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
1976
+
sg_init_table(req_ctx->bufsl, nsg);
1977
+
sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
1978
+
if (nsg > 1)
1979
+
sg_chain(req_ctx->bufsl, 2, req_ctx->request_sl);
1980
+
req_ctx->psrc = req_ctx->bufsl;
1981
+
} else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
1982
+
int offset;
1983
+
1984
+
if (nbytes_to_hash > blocksize)
1985
+
offset = blocksize - req_ctx->nbuf;
1986
+
else
1987
+
offset = nbytes_to_hash - req_ctx->nbuf;
1988
+
nents = sg_nents_for_len(req_ctx->request_sl, offset);
1989
+
if (nents < 0) {
1990
+
dev_err(dev, "Invalid number of src SG.\n");
1991
+
return nents;
1992
+
}
1993
+
sg_copy_to_buffer(req_ctx->request_sl, nents,
1994
+
ctx_buf + req_ctx->nbuf, offset);
1995
+
req_ctx->nbuf += offset;
1996
+
req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, req_ctx->request_sl,
1997
+
offset);
1998
+
} else
1999
+
req_ctx->psrc = req_ctx->request_sl;
2000
+
2001
+
if (to_hash_later) {
2002
+
nents = sg_nents_for_len(req_ctx->request_sl, nbytes);
2003
+
if (nents < 0) {
2004
+
dev_err(dev, "Invalid number of src SG.\n");
2005
+
return nents;
2006
+
}
2007
+
sg_pcopy_to_buffer(req_ctx->request_sl, nents,
2008
+
req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2009
+
to_hash_later,
2010
+
nbytes - to_hash_later);
2011
+
}
2012
+
req_ctx->to_hash_later = to_hash_later;
2013
+
2014
+
/* Allocate extended descriptor */
2015
+
edesc = ahash_edesc_alloc(req_ctx->areq, nbytes_to_hash);
2016
+
if (IS_ERR(edesc))
2017
+
return PTR_ERR(edesc);
2018
+
2019
+
edesc->desc.hdr = ctx->desc_hdr_template;
2020
+
2021
+
/* On last one, request SEC to pad; otherwise continue */
2022
+
if (req_ctx->last)
2023
+
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2024
+
else
2025
+
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2026
+
2027
+
/* request SEC to INIT hash. */
2028
+
if (req_ctx->first && !req_ctx->swinit)
2029
+
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2030
+
2031
+
/* When the tfm context has a keylen, it's an HMAC.
2032
+
* A first or last (ie. not middle) descriptor must request HMAC.
2033
+
*/
2034
+
if (ctx->keylen && (req_ctx->first || req_ctx->last))
2035
+
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2036
+
2037
+
return common_nonsnoop_hash(edesc, req_ctx->areq, nbytes_to_hash, ahash_done);
2038
+
}
2039
+
2040
+
static void sec1_ahash_process_remaining(struct work_struct *work)
2041
+
{
2042
+
struct talitos_ahash_req_ctx *req_ctx =
2043
+
container_of(work, struct talitos_ahash_req_ctx,
2044
+
sec1_ahash_process_remaining);
2045
+
int err = 0;
2046
+
2047
+
req_ctx->request_sl = scatterwalk_ffwd(req_ctx->request_bufsl,
2048
+
req_ctx->request_sl, TALITOS1_MAX_DATA_LEN);
2049
+
2050
+
if (req_ctx->remaining_ahash_request_bytes > TALITOS1_MAX_DATA_LEN)
2051
+
req_ctx->current_ahash_request_bytes = TALITOS1_MAX_DATA_LEN;
2052
+
else {
2053
+
req_ctx->current_ahash_request_bytes =
2054
+
req_ctx->remaining_ahash_request_bytes;
2055
+
2056
+
if (req_ctx->last_request)
2057
+
req_ctx->last = 1;
2058
+
}
2059
+
2060
+
err = ahash_process_req_one(req_ctx->areq,
2061
+
req_ctx->current_ahash_request_bytes);
2062
+
2063
+
if (err != -EINPROGRESS)
2064
+
ahash_request_complete(req_ctx->areq, err);
2065
+
}
2066
+
2067
+
static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
2068
+
{
2069
+
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2070
+
struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2071
+
struct device *dev = ctx->dev;
2072
+
struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2073
+
struct talitos_private *priv = dev_get_drvdata(dev);
2074
+
bool is_sec1 = has_ftr_sec1(priv);
2075
+
2076
+
req_ctx->areq = areq;
2077
+
req_ctx->request_sl = areq->src;
2078
+
req_ctx->remaining_ahash_request_bytes = nbytes;
2079
+
2080
+
if (is_sec1) {
2081
+
if (nbytes > TALITOS1_MAX_DATA_LEN)
2082
+
nbytes = TALITOS1_MAX_DATA_LEN;
2083
+
else if (req_ctx->last_request)
2084
+
req_ctx->last = 1;
2085
+
}
2086
+
2087
+
req_ctx->current_ahash_request_bytes = nbytes;
2088
+
2089
+
return ahash_process_req_one(req_ctx->areq,
2090
+
req_ctx->current_ahash_request_bytes);
2091
+
}
2092
+
1950
2093
static int ahash_init(struct ahash_request *areq)
1951
2094
{
1952
2095
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
···
2130
1943
? TALITOS_MDEU_CONTEXT_SIZE_MD5_SHA1_SHA256
2131
1944
: TALITOS_MDEU_CONTEXT_SIZE_SHA384_SHA512;
2132
1945
req_ctx->hw_context_size = size;
1946
+
req_ctx->last_request = 0;
1947
+
req_ctx->last = 0;
1948
+
INIT_WORK(&req_ctx->sec1_ahash_process_remaining, sec1_ahash_process_remaining);
2133
1949
2134
1950
dma = dma_map_single(dev, req_ctx->hw_context, req_ctx->hw_context_size,
2135
1951
DMA_TO_DEVICE);
···
2168
1978
return 0;
2169
1979
}
2170
1980
2171
-
static int ahash_process_req(struct ahash_request *areq, unsigned int nbytes)
2172
-
{
2173
-
struct crypto_ahash *tfm = crypto_ahash_reqtfm(areq);
2174
-
struct talitos_ctx *ctx = crypto_ahash_ctx(tfm);
2175
-
struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2176
-
struct talitos_edesc *edesc;
2177
-
unsigned int blocksize =
2178
-
crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm));
2179
-
unsigned int nbytes_to_hash;
2180
-
unsigned int to_hash_later;
2181
-
unsigned int nsg;
2182
-
int nents;
2183
-
struct device *dev = ctx->dev;
2184
-
struct talitos_private *priv = dev_get_drvdata(dev);
2185
-
bool is_sec1 = has_ftr_sec1(priv);
2186
-
u8 *ctx_buf = req_ctx->buf[req_ctx->buf_idx];
2187
-
2188
-
if (!req_ctx->last && (nbytes + req_ctx->nbuf <= blocksize)) {
2189
-
/* Buffer up to one whole block */
2190
-
nents = sg_nents_for_len(areq->src, nbytes);
2191
-
if (nents < 0) {
2192
-
dev_err(dev, "Invalid number of src SG.\n");
2193
-
return nents;
2194
-
}
2195
-
sg_copy_to_buffer(areq->src, nents,
2196
-
ctx_buf + req_ctx->nbuf, nbytes);
2197
-
req_ctx->nbuf += nbytes;
2198
-
return 0;
2199
-
}
2200
-
2201
-
/* At least (blocksize + 1) bytes are available to hash */
2202
-
nbytes_to_hash = nbytes + req_ctx->nbuf;
2203
-
to_hash_later = nbytes_to_hash & (blocksize - 1);
2204
-
2205
-
if (req_ctx->last)
2206
-
to_hash_later = 0;
2207
-
else if (to_hash_later)
2208
-
/* There is a partial block. Hash the full block(s) now */
2209
-
nbytes_to_hash -= to_hash_later;
2210
-
else {
2211
-
/* Keep one block buffered */
2212
-
nbytes_to_hash -= blocksize;
2213
-
to_hash_later = blocksize;
2214
-
}
2215
-
2216
-
/* Chain in any previously buffered data */
2217
-
if (!is_sec1 && req_ctx->nbuf) {
2218
-
nsg = (req_ctx->nbuf < nbytes_to_hash) ? 2 : 1;
2219
-
sg_init_table(req_ctx->bufsl, nsg);
2220
-
sg_set_buf(req_ctx->bufsl, ctx_buf, req_ctx->nbuf);
2221
-
if (nsg > 1)
2222
-
sg_chain(req_ctx->bufsl, 2, areq->src);
2223
-
req_ctx->psrc = req_ctx->bufsl;
2224
-
} else if (is_sec1 && req_ctx->nbuf && req_ctx->nbuf < blocksize) {
2225
-
int offset;
2226
-
2227
-
if (nbytes_to_hash > blocksize)
2228
-
offset = blocksize - req_ctx->nbuf;
2229
-
else
2230
-
offset = nbytes_to_hash - req_ctx->nbuf;
2231
-
nents = sg_nents_for_len(areq->src, offset);
2232
-
if (nents < 0) {
2233
-
dev_err(dev, "Invalid number of src SG.\n");
2234
-
return nents;
2235
-
}
2236
-
sg_copy_to_buffer(areq->src, nents,
2237
-
ctx_buf + req_ctx->nbuf, offset);
2238
-
req_ctx->nbuf += offset;
2239
-
req_ctx->psrc = scatterwalk_ffwd(req_ctx->bufsl, areq->src,
2240
-
offset);
2241
-
} else
2242
-
req_ctx->psrc = areq->src;
2243
-
2244
-
if (to_hash_later) {
2245
-
nents = sg_nents_for_len(areq->src, nbytes);
2246
-
if (nents < 0) {
2247
-
dev_err(dev, "Invalid number of src SG.\n");
2248
-
return nents;
2249
-
}
2250
-
sg_pcopy_to_buffer(areq->src, nents,
2251
-
req_ctx->buf[(req_ctx->buf_idx + 1) & 1],
2252
-
to_hash_later,
2253
-
nbytes - to_hash_later);
2254
-
}
2255
-
req_ctx->to_hash_later = to_hash_later;
2256
-
2257
-
/* Allocate extended descriptor */
2258
-
edesc = ahash_edesc_alloc(areq, nbytes_to_hash);
2259
-
if (IS_ERR(edesc))
2260
-
return PTR_ERR(edesc);
2261
-
2262
-
edesc->desc.hdr = ctx->desc_hdr_template;
2263
-
2264
-
/* On last one, request SEC to pad; otherwise continue */
2265
-
if (req_ctx->last)
2266
-
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_PAD;
2267
-
else
2268
-
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_CONT;
2269
-
2270
-
/* request SEC to INIT hash. */
2271
-
if (req_ctx->first && !req_ctx->swinit)
2272
-
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_INIT;
2273
-
2274
-
/* When the tfm context has a keylen, it's an HMAC.
2275
-
* A first or last (ie. not middle) descriptor must request HMAC.
2276
-
*/
2277
-
if (ctx->keylen && (req_ctx->first || req_ctx->last))
2278
-
edesc->desc.hdr |= DESC_HDR_MODE0_MDEU_HMAC;
2279
-
2280
-
return common_nonsnoop_hash(edesc, areq, nbytes_to_hash, ahash_done);
2281
-
}
2282
-
2283
1981
static int ahash_update(struct ahash_request *areq)
2284
1982
{
2285
1983
struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2286
1984
2287
-
req_ctx->last = 0;
1985
+
req_ctx->last_request = 0;
2288
1986
2289
1987
return ahash_process_req(areq, areq->nbytes);
2290
1988
}
···
2181
2103
{
2182
2104
struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2183
2105
2184
-
req_ctx->last = 1;
2106
+
req_ctx->last_request = 1;
2185
2107
2186
2108
return ahash_process_req(areq, 0);
2187
2109
}
···
2190
2112
{
2191
2113
struct talitos_ahash_req_ctx *req_ctx = ahash_request_ctx(areq);
2192
2114
2193
-
req_ctx->last = 1;
2115
+
req_ctx->last_request = 1;
2194
2116
2195
2117
return ahash_process_req(areq, areq->nbytes);
2196
2118
}