Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
crypto: acomp - Move stream management into scomp layer
Rather than allocating the stream memory in the request object,
move it into a per-cpu buffer managed by scomp. This takes the
stress off the user from having to manage large request objects
and setting up their own per-cpu buffers in order to do so.
Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au>
+84
-93
-30
crypto/acompress.c
-30
crypto/acompress.c
···
123
123
}
124
124
EXPORT_SYMBOL_GPL(crypto_alloc_acomp_node);
125
125
126
-
struct acomp_req *acomp_request_alloc(struct crypto_acomp *acomp)
127
-
{
128
-
struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
129
-
struct acomp_req *req;
130
-
131
-
req = __acomp_request_alloc(acomp);
132
-
if (req && (tfm->__crt_alg->cra_type != &crypto_acomp_type))
133
-
return crypto_acomp_scomp_alloc_ctx(req);
134
-
135
-
return req;
136
-
}
137
-
EXPORT_SYMBOL_GPL(acomp_request_alloc);
138
-
139
-
void acomp_request_free(struct acomp_req *req)
140
-
{
141
-
struct crypto_acomp *acomp = crypto_acomp_reqtfm(req);
142
-
struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
143
-
144
-
if (tfm->__crt_alg->cra_type != &crypto_acomp_type)
145
-
crypto_acomp_scomp_free_ctx(req);
146
-
147
-
if (req->base.flags & CRYPTO_ACOMP_ALLOC_OUTPUT) {
148
-
acomp->dst_free(req->dst);
149
-
req->dst = NULL;
150
-
}
151
-
152
-
__acomp_request_free(req);
153
-
}
154
-
EXPORT_SYMBOL_GPL(acomp_request_free);
155
-
156
126
void comp_prepare_alg(struct comp_alg_common *alg)
157
127
{
158
128
struct crypto_alg *base = &alg->base;
-2
crypto/compress.h
-2
crypto/compress.h
···
15
15
struct comp_alg_common;
16
16
17
17
int crypto_init_scomp_ops_async(struct crypto_tfm *tfm);
18
-
struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req);
19
-
void crypto_acomp_scomp_free_ctx(struct acomp_req *req);
20
18
21
19
void comp_prepare_alg(struct comp_alg_common *alg);
22
20
+58
-32
crypto/scompress.c
+58
-32
crypto/scompress.c
···
98
98
return -ENOMEM;
99
99
}
100
100
101
+
static void scomp_free_streams(struct scomp_alg *alg)
102
+
{
103
+
struct crypto_acomp_stream __percpu *stream = alg->stream;
104
+
int i;
105
+
106
+
for_each_possible_cpu(i) {
107
+
struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i);
108
+
109
+
if (!ps->ctx)
110
+
break;
111
+
112
+
alg->free_ctx(ps);
113
+
}
114
+
115
+
free_percpu(stream);
116
+
}
117
+
118
+
static int scomp_alloc_streams(struct scomp_alg *alg)
119
+
{
120
+
struct crypto_acomp_stream __percpu *stream;
121
+
int i;
122
+
123
+
stream = alloc_percpu(struct crypto_acomp_stream);
124
+
if (!stream)
125
+
return -ENOMEM;
126
+
127
+
for_each_possible_cpu(i) {
128
+
struct crypto_acomp_stream *ps = per_cpu_ptr(stream, i);
129
+
130
+
ps->ctx = alg->alloc_ctx();
131
+
if (IS_ERR(ps->ctx)) {
132
+
scomp_free_streams(alg);
133
+
return PTR_ERR(ps->ctx);
134
+
}
135
+
136
+
spin_lock_init(&ps->lock);
137
+
}
138
+
139
+
alg->stream = stream;
140
+
return 0;
141
+
}
142
+
101
143
static int crypto_scomp_init_tfm(struct crypto_tfm *tfm)
102
144
{
145
+
struct scomp_alg *alg = crypto_scomp_alg(__crypto_scomp_tfm(tfm));
103
146
int ret = 0;
104
147
105
148
mutex_lock(&scomp_lock);
149
+
if (!alg->stream) {
150
+
ret = scomp_alloc_streams(alg);
151
+
if (ret)
152
+
goto unlock;
153
+
}
106
154
if (!scomp_scratch_users++)
107
155
ret = crypto_scomp_alloc_scratches();
156
+
unlock:
108
157
mutex_unlock(&scomp_lock);
109
158
110
159
return ret;
···
164
115
struct crypto_acomp *tfm = crypto_acomp_reqtfm(req);
165
116
void **tfm_ctx = acomp_tfm_ctx(tfm);
166
117
struct crypto_scomp *scomp = *tfm_ctx;
167
-
void **ctx = acomp_request_ctx(req);
118
+
struct crypto_acomp_stream *stream;
168
119
struct scomp_scratch *scratch;
169
120
void *src, *dst;
170
121
unsigned int dlen;
···
197
148
else
198
149
dst = scratch->dst;
199
150
151
+
stream = raw_cpu_ptr(crypto_scomp_alg(scomp)->stream);
152
+
spin_lock(&stream->lock);
200
153
if (dir)
201
154
ret = crypto_scomp_compress(scomp, src, req->slen,
202
-
dst, &req->dlen, *ctx);
155
+
dst, &req->dlen, stream->ctx);
203
156
else
204
157
ret = crypto_scomp_decompress(scomp, src, req->slen,
205
-
dst, &req->dlen, *ctx);
158
+
dst, &req->dlen, stream->ctx);
159
+
spin_unlock(&stream->lock);
206
160
if (!ret) {
207
161
if (!req->dst) {
208
162
req->dst = sgl_alloc(req->dlen, GFP_ATOMIC, NULL);
···
278
226
crt->compress = scomp_acomp_compress;
279
227
crt->decompress = scomp_acomp_decompress;
280
228
crt->dst_free = sgl_free;
281
-
crt->reqsize = sizeof(void *);
282
229
283
230
return 0;
284
231
}
285
232
286
-
struct acomp_req *crypto_acomp_scomp_alloc_ctx(struct acomp_req *req)
233
+
static void crypto_scomp_destroy(struct crypto_alg *alg)
287
234
{
288
-
struct crypto_acomp *acomp = crypto_acomp_reqtfm(req);
289
-
struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
290
-
struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm);
291
-
struct crypto_scomp *scomp = *tfm_ctx;
292
-
void *ctx;
293
-
294
-
ctx = crypto_scomp_alloc_ctx(scomp);
295
-
if (IS_ERR(ctx)) {
296
-
kfree(req);
297
-
return NULL;
298
-
}
299
-
300
-
*req->__ctx = ctx;
301
-
302
-
return req;
303
-
}
304
-
305
-
void crypto_acomp_scomp_free_ctx(struct acomp_req *req)
306
-
{
307
-
struct crypto_acomp *acomp = crypto_acomp_reqtfm(req);
308
-
struct crypto_tfm *tfm = crypto_acomp_tfm(acomp);
309
-
struct crypto_scomp **tfm_ctx = crypto_tfm_ctx(tfm);
310
-
struct crypto_scomp *scomp = *tfm_ctx;
311
-
void *ctx = *req->__ctx;
312
-
313
-
if (ctx)
314
-
crypto_scomp_free_ctx(scomp, ctx);
235
+
scomp_free_streams(__crypto_scomp_alg(alg));
315
236
}
316
237
317
238
static const struct crypto_type crypto_scomp_type = {
318
239
.extsize = crypto_alg_extsize,
319
240
.init_tfm = crypto_scomp_init_tfm,
241
+
.destroy = crypto_scomp_destroy,
320
242
#ifdef CONFIG_PROC_FS
321
243
.show = crypto_scomp_show,
322
244
#endif
+24
-2
include/crypto/acompress.h
+24
-2
include/crypto/acompress.h
···
10
10
#define _CRYPTO_ACOMP_H
11
11
12
12
#include <linux/atomic.h>
13
+
#include <linux/compiler_types.h>
13
14
#include <linux/container_of.h>
14
15
#include <linux/crypto.h>
16
+
#include <linux/slab.h>
17
+
#include <linux/spinlock_types.h>
18
+
#include <linux/types.h>
15
19
16
20
#define CRYPTO_ACOMP_ALLOC_OUTPUT 0x00000001
17
21
#define CRYPTO_ACOMP_DST_MAX 131072
···
58
54
struct crypto_tfm base;
59
55
};
60
56
57
+
struct crypto_acomp_stream {
58
+
spinlock_t lock;
59
+
void *ctx;
60
+
};
61
+
61
62
#define COMP_ALG_COMMON { \
62
63
struct crypto_alg base; \
64
+
struct crypto_acomp_stream __percpu *stream; \
63
65
}
64
66
struct comp_alg_common COMP_ALG_COMMON;
65
67
···
183
173
*
184
174
* Return: allocated handle in case of success or NULL in case of an error
185
175
*/
186
-
struct acomp_req *acomp_request_alloc(struct crypto_acomp *tfm);
176
+
static inline struct acomp_req *acomp_request_alloc_noprof(struct crypto_acomp *tfm)
177
+
{
178
+
struct acomp_req *req;
179
+
180
+
req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL);
181
+
if (likely(req))
182
+
acomp_request_set_tfm(req, tfm);
183
+
return req;
184
+
}
185
+
#define acomp_request_alloc(...) alloc_hooks(acomp_request_alloc_noprof(__VA_ARGS__))
187
186
188
187
/**
189
188
* acomp_request_free() -- zeroize and free asynchronous (de)compression
···
201
182
*
202
183
* @req: request to free
203
184
*/
204
-
void acomp_request_free(struct acomp_req *req);
185
+
static inline void acomp_request_free(struct acomp_req *req)
186
+
{
187
+
kfree_sensitive(req);
188
+
}
205
189
206
190
/**
207
191
* acomp_request_set_callback() -- Sets an asynchronous callback
+1
-16
include/crypto/internal/acompress.h
+1
-16
include/crypto/internal/acompress.h
···
32
32
*
33
33
* @reqsize: Context size for (de)compression requests
34
34
* @base: Common crypto API algorithm data structure
35
+
* @stream: Per-cpu memory for algorithm
35
36
* @calg: Cmonn algorithm data structure shared with scomp
36
37
*/
37
38
struct acomp_alg {
···
67
66
int err)
68
67
{
69
68
crypto_request_complete(&req->base, err);
70
-
}
71
-
72
-
static inline struct acomp_req *__acomp_request_alloc_noprof(struct crypto_acomp *tfm)
73
-
{
74
-
struct acomp_req *req;
75
-
76
-
req = kzalloc_noprof(sizeof(*req) + crypto_acomp_reqsize(tfm), GFP_KERNEL);
77
-
if (likely(req))
78
-
acomp_request_set_tfm(req, tfm);
79
-
return req;
80
-
}
81
-
#define __acomp_request_alloc(...) alloc_hooks(__acomp_request_alloc_noprof(__VA_ARGS__))
82
-
83
-
static inline void __acomp_request_free(struct acomp_req *req)
84
-
{
85
-
kfree_sensitive(req);
86
69
}
87
70
88
71
/**
+1
-11
include/crypto/internal/scompress.h
+1
-11
include/crypto/internal/scompress.h
···
28
28
* @compress: Function performs a compress operation
29
29
* @decompress: Function performs a de-compress operation
30
30
* @base: Common crypto API algorithm data structure
31
+
* @stream: Per-cpu memory for algorithm
31
32
* @calg: Cmonn algorithm data structure shared with acomp
32
33
*/
33
34
struct scomp_alg {
···
70
69
static inline struct scomp_alg *crypto_scomp_alg(struct crypto_scomp *tfm)
71
70
{
72
71
return __crypto_scomp_alg(crypto_scomp_tfm(tfm)->__crt_alg);
73
-
}
74
-
75
-
static inline void *crypto_scomp_alloc_ctx(struct crypto_scomp *tfm)
76
-
{
77
-
return crypto_scomp_alg(tfm)->alloc_ctx();
78
-
}
79
-
80
-
static inline void crypto_scomp_free_ctx(struct crypto_scomp *tfm,
81
-
void *ctx)
82
-
{
83
-
return crypto_scomp_alg(tfm)->free_ctx(ctx);
84
72
}
85
73
86
74
static inline int crypto_scomp_compress(struct crypto_scomp *tfm,