Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2017 Marvell
4 *
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
6 */
7
8#include <linux/dma-mapping.h>
9#include <linux/spinlock.h>
10
11#include "safexcel.h"
12
13int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
14 struct safexcel_desc_ring *cdr,
15 struct safexcel_desc_ring *rdr)
16{
17 int i;
18 struct safexcel_command_desc *cdesc;
19 dma_addr_t atok;
20
21 /* Actual command descriptor ring */
22 cdr->offset = priv->config.cd_offset;
23 cdr->base = dmam_alloc_coherent(priv->dev,
24 cdr->offset * EIP197_DEFAULT_RING_SIZE,
25 &cdr->base_dma, GFP_KERNEL);
26 if (!cdr->base)
27 return -ENOMEM;
28 cdr->write = cdr->base;
29 cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
30 cdr->read = cdr->base;
31
32 /* Command descriptor shadow ring for storing additional token data */
33 cdr->shoffset = priv->config.cdsh_offset;
34 cdr->shbase = dmam_alloc_coherent(priv->dev,
35 cdr->shoffset *
36 EIP197_DEFAULT_RING_SIZE,
37 &cdr->shbase_dma, GFP_KERNEL);
38 if (!cdr->shbase)
39 return -ENOMEM;
40 cdr->shwrite = cdr->shbase;
41 cdr->shbase_end = cdr->shbase + cdr->shoffset *
42 (EIP197_DEFAULT_RING_SIZE - 1);
43
44 /*
45 * Populate command descriptors with physical pointers to shadow descs.
46 * Note that we only need to do this once if we don't overwrite them.
47 */
48 cdesc = cdr->base;
49 atok = cdr->shbase_dma;
50 for (i = 0; i < EIP197_DEFAULT_RING_SIZE; i++) {
51 cdesc->atok_lo = lower_32_bits(atok);
52 cdesc->atok_hi = upper_32_bits(atok);
53 cdesc = (void *)cdesc + cdr->offset;
54 atok += cdr->shoffset;
55 }
56
57 rdr->offset = priv->config.rd_offset;
58 /* Use shoffset for result token offset here */
59 rdr->shoffset = priv->config.res_offset;
60 rdr->base = dmam_alloc_coherent(priv->dev,
61 rdr->offset * EIP197_DEFAULT_RING_SIZE,
62 &rdr->base_dma, GFP_KERNEL);
63 if (!rdr->base)
64 return -ENOMEM;
65 rdr->write = rdr->base;
66 rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
67 rdr->read = rdr->base;
68
69 return 0;
70}
71
72inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
73{
74 return (atomic_inc_return(&priv->ring_used) % priv->config.rings);
75}
76
77static void *safexcel_ring_next_cwptr(struct safexcel_crypto_priv *priv,
78 struct safexcel_desc_ring *ring,
79 bool first,
80 struct safexcel_token **atoken)
81{
82 void *ptr = ring->write;
83
84 if (first)
85 *atoken = ring->shwrite;
86
87 if ((ring->write == ring->read - ring->offset) ||
88 (ring->read == ring->base && ring->write == ring->base_end))
89 return ERR_PTR(-ENOMEM);
90
91 if (ring->write == ring->base_end) {
92 ring->write = ring->base;
93 ring->shwrite = ring->shbase;
94 } else {
95 ring->write += ring->offset;
96 ring->shwrite += ring->shoffset;
97 }
98
99 return ptr;
100}
101
102static void *safexcel_ring_next_rwptr(struct safexcel_crypto_priv *priv,
103 struct safexcel_desc_ring *ring,
104 struct result_data_desc **rtoken)
105{
106 void *ptr = ring->write;
107
108 /* Result token at relative offset shoffset */
109 *rtoken = ring->write + ring->shoffset;
110
111 if ((ring->write == ring->read - ring->offset) ||
112 (ring->read == ring->base && ring->write == ring->base_end))
113 return ERR_PTR(-ENOMEM);
114
115 if (ring->write == ring->base_end)
116 ring->write = ring->base;
117 else
118 ring->write += ring->offset;
119
120 return ptr;
121}
122
123void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
124 struct safexcel_desc_ring *ring)
125{
126 void *ptr = ring->read;
127
128 if (ring->write == ring->read)
129 return ERR_PTR(-ENOENT);
130
131 if (ring->read == ring->base_end)
132 ring->read = ring->base;
133 else
134 ring->read += ring->offset;
135
136 return ptr;
137}
138
139inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,
140 int ring)
141{
142 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
143
144 return rdr->read;
145}
146
147inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
148 int ring)
149{
150 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
151
152 return (rdr->read - rdr->base) / rdr->offset;
153}
154
155inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
156 int ring,
157 struct safexcel_result_desc *rdesc)
158{
159 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
160
161 return ((void *)rdesc - rdr->base) / rdr->offset;
162}
163
164void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
165 struct safexcel_desc_ring *ring)
166{
167 if (ring->write == ring->read)
168 return;
169
170 if (ring->write == ring->base) {
171 ring->write = ring->base_end;
172 ring->shwrite = ring->shbase_end;
173 } else {
174 ring->write -= ring->offset;
175 ring->shwrite -= ring->shoffset;
176 }
177}
178
179struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
180 int ring_id,
181 bool first, bool last,
182 dma_addr_t data, u32 data_len,
183 u32 full_data_len,
184 dma_addr_t context,
185 struct safexcel_token **atoken)
186{
187 struct safexcel_command_desc *cdesc;
188
189 cdesc = safexcel_ring_next_cwptr(priv, &priv->ring[ring_id].cdr,
190 first, atoken);
191 if (IS_ERR(cdesc))
192 return cdesc;
193
194 cdesc->particle_size = data_len;
195 cdesc->rsvd0 = 0;
196 cdesc->last_seg = last;
197 cdesc->first_seg = first;
198 cdesc->additional_cdata_size = 0;
199 cdesc->rsvd1 = 0;
200 cdesc->data_lo = lower_32_bits(data);
201 cdesc->data_hi = upper_32_bits(data);
202
203 if (first) {
204 /*
205 * Note that the length here MUST be >0 or else the EIP(1)97
206 * may hang. Newer EIP197 firmware actually incorporates this
207 * fix already, but that doesn't help the EIP97 and we may
208 * also be running older firmware.
209 */
210 cdesc->control_data.packet_length = full_data_len ?: 1;
211 cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
212 EIP197_OPTION_64BIT_CTX |
213 EIP197_OPTION_CTX_CTRL_IN_CMD |
214 EIP197_OPTION_RC_AUTO;
215 cdesc->control_data.type = EIP197_TYPE_BCLA;
216 cdesc->control_data.context_lo = lower_32_bits(context) |
217 EIP197_CONTEXT_SMALL;
218 cdesc->control_data.context_hi = upper_32_bits(context);
219 }
220
221 return cdesc;
222}
223
224struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
225 int ring_id,
226 bool first, bool last,
227 dma_addr_t data, u32 len)
228{
229 struct safexcel_result_desc *rdesc;
230 struct result_data_desc *rtoken;
231
232 rdesc = safexcel_ring_next_rwptr(priv, &priv->ring[ring_id].rdr,
233 &rtoken);
234 if (IS_ERR(rdesc))
235 return rdesc;
236
237 rdesc->particle_size = len;
238 rdesc->rsvd0 = 0;
239 rdesc->descriptor_overflow = 1; /* assume error */
240 rdesc->buffer_overflow = 1; /* assume error */
241 rdesc->last_seg = last;
242 rdesc->first_seg = first;
243 rdesc->result_size = EIP197_RD64_RESULT_SIZE;
244 rdesc->rsvd1 = 0;
245 rdesc->data_lo = lower_32_bits(data);
246 rdesc->data_hi = upper_32_bits(data);
247
248 /* Clear length in result token */
249 rtoken->packet_length = 0;
250 /* Assume errors - HW will clear if not the case */
251 rtoken->error_code = 0x7fff;
252
253 return rdesc;
254}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2017 Marvell
4 *
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
6 */
7
8#include <linux/dma-mapping.h>
9#include <linux/spinlock.h>
10
11#include "safexcel.h"
12
13int safexcel_init_ring_descriptors(struct safexcel_crypto_priv *priv,
14 struct safexcel_desc_ring *cdr,
15 struct safexcel_desc_ring *rdr)
16{
17 cdr->offset = sizeof(u32) * priv->config.cd_offset;
18 cdr->base = dmam_alloc_coherent(priv->dev,
19 cdr->offset * EIP197_DEFAULT_RING_SIZE,
20 &cdr->base_dma, GFP_KERNEL);
21 if (!cdr->base)
22 return -ENOMEM;
23 cdr->write = cdr->base;
24 cdr->base_end = cdr->base + cdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
25 cdr->read = cdr->base;
26
27 rdr->offset = sizeof(u32) * priv->config.rd_offset;
28 rdr->base = dmam_alloc_coherent(priv->dev,
29 rdr->offset * EIP197_DEFAULT_RING_SIZE,
30 &rdr->base_dma, GFP_KERNEL);
31 if (!rdr->base)
32 return -ENOMEM;
33 rdr->write = rdr->base;
34 rdr->base_end = rdr->base + rdr->offset * (EIP197_DEFAULT_RING_SIZE - 1);
35 rdr->read = rdr->base;
36
37 return 0;
38}
39
40inline int safexcel_select_ring(struct safexcel_crypto_priv *priv)
41{
42 return (atomic_inc_return(&priv->ring_used) % priv->config.rings);
43}
44
45static void *safexcel_ring_next_wptr(struct safexcel_crypto_priv *priv,
46 struct safexcel_desc_ring *ring)
47{
48 void *ptr = ring->write;
49
50 if ((ring->write == ring->read - ring->offset) ||
51 (ring->read == ring->base && ring->write == ring->base_end))
52 return ERR_PTR(-ENOMEM);
53
54 if (ring->write == ring->base_end)
55 ring->write = ring->base;
56 else
57 ring->write += ring->offset;
58
59 return ptr;
60}
61
62void *safexcel_ring_next_rptr(struct safexcel_crypto_priv *priv,
63 struct safexcel_desc_ring *ring)
64{
65 void *ptr = ring->read;
66
67 if (ring->write == ring->read)
68 return ERR_PTR(-ENOENT);
69
70 if (ring->read == ring->base_end)
71 ring->read = ring->base;
72 else
73 ring->read += ring->offset;
74
75 return ptr;
76}
77
78inline void *safexcel_ring_curr_rptr(struct safexcel_crypto_priv *priv,
79 int ring)
80{
81 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
82
83 return rdr->read;
84}
85
86inline int safexcel_ring_first_rdr_index(struct safexcel_crypto_priv *priv,
87 int ring)
88{
89 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
90
91 return (rdr->read - rdr->base) / rdr->offset;
92}
93
94inline int safexcel_ring_rdr_rdesc_index(struct safexcel_crypto_priv *priv,
95 int ring,
96 struct safexcel_result_desc *rdesc)
97{
98 struct safexcel_desc_ring *rdr = &priv->ring[ring].rdr;
99
100 return ((void *)rdesc - rdr->base) / rdr->offset;
101}
102
103void safexcel_ring_rollback_wptr(struct safexcel_crypto_priv *priv,
104 struct safexcel_desc_ring *ring)
105{
106 if (ring->write == ring->read)
107 return;
108
109 if (ring->write == ring->base)
110 ring->write = ring->base_end;
111 else
112 ring->write -= ring->offset;
113}
114
115struct safexcel_command_desc *safexcel_add_cdesc(struct safexcel_crypto_priv *priv,
116 int ring_id,
117 bool first, bool last,
118 dma_addr_t data, u32 data_len,
119 u32 full_data_len,
120 dma_addr_t context) {
121 struct safexcel_command_desc *cdesc;
122 int i;
123
124 cdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].cdr);
125 if (IS_ERR(cdesc))
126 return cdesc;
127
128 memset(cdesc, 0, sizeof(struct safexcel_command_desc));
129
130 cdesc->first_seg = first;
131 cdesc->last_seg = last;
132 cdesc->particle_size = data_len;
133 cdesc->data_lo = lower_32_bits(data);
134 cdesc->data_hi = upper_32_bits(data);
135
136 if (first && context) {
137 struct safexcel_token *token =
138 (struct safexcel_token *)cdesc->control_data.token;
139
140 /*
141 * Note that the length here MUST be >0 or else the EIP(1)97
142 * may hang. Newer EIP197 firmware actually incorporates this
143 * fix already, but that doesn't help the EIP97 and we may
144 * also be running older firmware.
145 */
146 cdesc->control_data.packet_length = full_data_len ?: 1;
147 cdesc->control_data.options = EIP197_OPTION_MAGIC_VALUE |
148 EIP197_OPTION_64BIT_CTX |
149 EIP197_OPTION_CTX_CTRL_IN_CMD;
150 cdesc->control_data.context_lo =
151 (lower_32_bits(context) & GENMASK(31, 2)) >> 2;
152 cdesc->control_data.context_hi = upper_32_bits(context);
153
154 if (priv->version == EIP197B_MRVL ||
155 priv->version == EIP197D_MRVL)
156 cdesc->control_data.options |= EIP197_OPTION_RC_AUTO;
157
158 /* TODO: large xform HMAC with SHA-384/512 uses refresh = 3 */
159 cdesc->control_data.refresh = 2;
160
161 for (i = 0; i < EIP197_MAX_TOKENS; i++)
162 eip197_noop_token(&token[i]);
163 }
164
165 return cdesc;
166}
167
168struct safexcel_result_desc *safexcel_add_rdesc(struct safexcel_crypto_priv *priv,
169 int ring_id,
170 bool first, bool last,
171 dma_addr_t data, u32 len)
172{
173 struct safexcel_result_desc *rdesc;
174
175 rdesc = safexcel_ring_next_wptr(priv, &priv->ring[ring_id].rdr);
176 if (IS_ERR(rdesc))
177 return rdesc;
178
179 memset(rdesc, 0, sizeof(struct safexcel_result_desc));
180
181 rdesc->first_seg = first;
182 rdesc->last_seg = last;
183 rdesc->particle_size = len;
184 rdesc->data_lo = lower_32_bits(data);
185 rdesc->data_hi = upper_32_bits(data);
186
187 return rdesc;
188}