3 * linux/drivers/s390/net/qeth_eddp.c ($Revision: 1.13 $)
5 * Enhanced Device Driver Packing (EDDP) support for the qeth driver.
7 * Copyright 2004 IBM Corporation
9 * Author(s): Thomas Spatzier <tspat@de.ibm.com>
11 * $Revision: 1.13 $ $Date: 2005/05/04 20:19:18 $
14 #include <linux/config.h>
15 #include <linux/errno.h>
17 #include <linux/inetdevice.h>
18 #include <linux/netdevice.h>
19 #include <linux/kernel.h>
20 #include <linux/tcp.h>
22 #include <linux/skbuff.h>
28 #include "qeth_eddp.h"
31 qeth_eddp_check_buffers_for_context(struct qeth_qdio_out_q *queue,
32 struct qeth_eddp_context *ctx)
34 int index = queue->next_buf_to_fill;
35 int elements_needed = ctx->num_elements;
36 int elements_in_buffer;
38 int buffers_needed = 0;
40 QETH_DBF_TEXT(trace, 5, "eddpcbfc");
41 while(elements_needed > 0) {
43 if (atomic_read(&queue->bufs[index].state) !=
47 elements_in_buffer = QETH_MAX_BUFFER_ELEMENTS(queue->card) -
48 queue->bufs[index].next_element_to_fill;
49 skbs_in_buffer = elements_in_buffer / ctx->elements_per_skb;
50 elements_needed -= skbs_in_buffer * ctx->elements_per_skb;
51 index = (index + 1) % QDIO_MAX_BUFFERS_PER_Q;
53 return buffers_needed;
57 qeth_eddp_free_context(struct qeth_eddp_context *ctx)
61 QETH_DBF_TEXT(trace, 5, "eddpfctx");
62 for (i = 0; i < ctx->num_pages; ++i)
63 free_page((unsigned long)ctx->pages[i]);
65 if (ctx->elements != NULL)
72 qeth_eddp_get_context(struct qeth_eddp_context *ctx)
74 atomic_inc(&ctx->refcnt);
78 qeth_eddp_put_context(struct qeth_eddp_context *ctx)
80 if (atomic_dec_return(&ctx->refcnt) == 0)
81 qeth_eddp_free_context(ctx);
85 qeth_eddp_buf_release_contexts(struct qeth_qdio_out_buffer *buf)
87 struct qeth_eddp_context_reference *ref;
89 QETH_DBF_TEXT(trace, 6, "eddprctx");
90 while (!list_empty(&buf->ctx_list)){
91 ref = list_entry(buf->ctx_list.next,
92 struct qeth_eddp_context_reference, list);
93 qeth_eddp_put_context(ref->ctx);
100 qeth_eddp_buf_ref_context(struct qeth_qdio_out_buffer *buf,
101 struct qeth_eddp_context *ctx)
103 struct qeth_eddp_context_reference *ref;
105 QETH_DBF_TEXT(trace, 6, "eddprfcx");
106 ref = kmalloc(sizeof(struct qeth_eddp_context_reference), GFP_ATOMIC);
109 qeth_eddp_get_context(ctx);
111 list_add_tail(&ref->list, &buf->ctx_list);
116 qeth_eddp_fill_buffer(struct qeth_qdio_out_q *queue,
117 struct qeth_eddp_context *ctx,
120 struct qeth_qdio_out_buffer *buf = NULL;
121 struct qdio_buffer *buffer;
122 int elements = ctx->num_elements;
128 QETH_DBF_TEXT(trace, 5, "eddpfibu");
129 while (elements > 0) {
130 buf = &queue->bufs[index];
131 if (atomic_read(&buf->state) != QETH_QDIO_BUF_EMPTY){
132 /* normally this should not happen since we checked for
133 * available elements in qeth_check_elements_for_context
138 PRINT_WARN("could only partially fill eddp "
143 /* check if the whole next skb fits into current buffer */
144 if ((QETH_MAX_BUFFER_ELEMENTS(queue->card) -
145 buf->next_element_to_fill)
146 < ctx->elements_per_skb){
147 /* no -> go to next buffer */
148 atomic_set(&buf->state, QETH_QDIO_BUF_PRIMED);
149 index = (index + 1) % QDIO_MAX_BUFFERS_PER_Q;
151 /* new buffer, so we have to add ctx to buffer'ctx_list
152 * and increment ctx's refcnt */
158 if (qeth_eddp_buf_ref_context(buf, ctx)){
159 PRINT_WARN("no memory to create eddp context "
164 buffer = buf->buffer;
165 /* fill one skb into buffer */
166 for (i = 0; i < ctx->elements_per_skb; ++i){
167 buffer->element[buf->next_element_to_fill].addr =
168 ctx->elements[element].addr;
169 buffer->element[buf->next_element_to_fill].length =
170 ctx->elements[element].length;
171 buffer->element[buf->next_element_to_fill].flags =
172 ctx->elements[element].flags;
173 buf->next_element_to_fill++;
179 if (!queue->do_pack) {
180 QETH_DBF_TEXT(trace, 6, "fillbfnp");
181 /* set state to PRIMED -> will be flushed */
182 if (buf->next_element_to_fill > 0){
183 atomic_set(&buf->state, QETH_QDIO_BUF_PRIMED);
187 #ifdef CONFIG_QETH_PERF_STATS
188 queue->card->perf_stats.skbs_sent_pack++;
190 QETH_DBF_TEXT(trace, 6, "fillbfpa");
191 if (buf->next_element_to_fill >=
192 QETH_MAX_BUFFER_ELEMENTS(queue->card)) {
194 * packed buffer if full -> set state PRIMED
197 atomic_set(&buf->state, QETH_QDIO_BUF_PRIMED);
206 qeth_eddp_create_segment_hdrs(struct qeth_eddp_context *ctx,
207 struct qeth_eddp_data *eddp, int data_len)
213 struct qeth_eddp_element *element;
215 QETH_DBF_TEXT(trace, 5, "eddpcrsh");
216 page = ctx->pages[ctx->offset >> PAGE_SHIFT];
217 page_offset = ctx->offset % PAGE_SIZE;
218 element = &ctx->elements[ctx->num_elements];
219 pkt_len = eddp->nhl + eddp->thl + data_len;
220 /* FIXME: layer2 and VLAN !!! */
221 if (eddp->qh.hdr.l2.id == QETH_HEADER_TYPE_LAYER2)
223 if (eddp->mac.h_proto == __constant_htons(ETH_P_8021Q))
224 pkt_len += VLAN_HLEN;
225 /* does complete packet fit in current page ? */
226 page_remainder = PAGE_SIZE - page_offset;
227 if (page_remainder < (sizeof(struct qeth_hdr) + pkt_len)){
228 /* no -> go to start of next page */
229 ctx->offset += page_remainder;
230 page = ctx->pages[ctx->offset >> PAGE_SHIFT];
233 memcpy(page + page_offset, &eddp->qh, sizeof(struct qeth_hdr));
234 element->addr = page + page_offset;
235 element->length = sizeof(struct qeth_hdr);
236 ctx->offset += sizeof(struct qeth_hdr);
237 page_offset += sizeof(struct qeth_hdr);
238 /* add mac header (?) */
239 if (eddp->qh.hdr.l2.id == QETH_HEADER_TYPE_LAYER2){
240 memcpy(page + page_offset, &eddp->mac, ETH_HLEN);
241 element->length += ETH_HLEN;
242 ctx->offset += ETH_HLEN;
243 page_offset += ETH_HLEN;
246 if (eddp->mac.h_proto == __constant_htons(ETH_P_8021Q)){
247 memcpy(page + page_offset, &eddp->vlan, VLAN_HLEN);
248 element->length += VLAN_HLEN;
249 ctx->offset += VLAN_HLEN;
250 page_offset += VLAN_HLEN;
252 /* add network header */
253 memcpy(page + page_offset, (u8 *)&eddp->nh, eddp->nhl);
254 element->length += eddp->nhl;
255 eddp->nh_in_ctx = page + page_offset;
256 ctx->offset += eddp->nhl;
257 page_offset += eddp->nhl;
258 /* add transport header */
259 memcpy(page + page_offset, (u8 *)&eddp->th, eddp->thl);
260 element->length += eddp->thl;
261 eddp->th_in_ctx = page + page_offset;
262 ctx->offset += eddp->thl;
266 qeth_eddp_copy_data_tcp(char *dst, struct qeth_eddp_data *eddp, int len,
269 struct skb_frag_struct *frag;
274 QETH_DBF_TEXT(trace, 5, "eddpcdtc");
275 if (skb_shinfo(eddp->skb)->nr_frags == 0) {
276 memcpy(dst, eddp->skb->data + eddp->skb_offset, len);
277 *hcsum = csum_partial(eddp->skb->data + eddp->skb_offset, len,
279 eddp->skb_offset += len;
282 if (eddp->frag < 0) {
283 /* we're in skb->data */
284 left_in_frag = (eddp->skb->len - eddp->skb->data_len)
286 src = eddp->skb->data + eddp->skb_offset;
288 frag = &skb_shinfo(eddp->skb)->
290 left_in_frag = frag->size - eddp->frag_offset;
292 (page_to_pfn(frag->page) << PAGE_SHIFT)+
293 frag->page_offset + eddp->frag_offset);
295 if (left_in_frag <= 0) {
297 eddp->frag_offset = 0;
300 copy_len = min(left_in_frag, len);
301 memcpy(dst, src, copy_len);
302 *hcsum = csum_partial(src, copy_len, *hcsum);
304 eddp->frag_offset += copy_len;
305 eddp->skb_offset += copy_len;
312 qeth_eddp_create_segment_data_tcp(struct qeth_eddp_context *ctx,
313 struct qeth_eddp_data *eddp, int data_len,
319 struct qeth_eddp_element *element;
322 QETH_DBF_TEXT(trace, 5, "eddpcsdt");
323 page = ctx->pages[ctx->offset >> PAGE_SHIFT];
324 page_offset = ctx->offset % PAGE_SIZE;
325 element = &ctx->elements[ctx->num_elements];
327 page_remainder = PAGE_SIZE - page_offset;
328 if (page_remainder < data_len){
329 qeth_eddp_copy_data_tcp(page + page_offset, eddp,
330 page_remainder, &hcsum);
331 element->length += page_remainder;
333 element->flags = SBAL_FLAGS_FIRST_FRAG;
335 element->flags = SBAL_FLAGS_MIDDLE_FRAG;
338 data_len -= page_remainder;
339 ctx->offset += page_remainder;
340 page = ctx->pages[ctx->offset >> PAGE_SHIFT];
342 element->addr = page + page_offset;
344 qeth_eddp_copy_data_tcp(page + page_offset, eddp,
346 element->length += data_len;
348 element->flags = SBAL_FLAGS_LAST_FRAG;
350 ctx->offset += data_len;
355 ((struct tcphdr *)eddp->th_in_ctx)->check = csum_fold(hcsum);
359 qeth_eddp_check_tcp4_hdr(struct qeth_eddp_data *eddp, int data_len)
361 u32 phcsum; /* pseudo header checksum */
363 QETH_DBF_TEXT(trace, 5, "eddpckt4");
364 eddp->th.tcp.h.check = 0;
365 /* compute pseudo header checksum */
366 phcsum = csum_tcpudp_nofold(eddp->nh.ip4.h.saddr, eddp->nh.ip4.h.daddr,
367 eddp->thl + data_len, IPPROTO_TCP, 0);
368 /* compute checksum of tcp header */
369 return csum_partial((u8 *)&eddp->th, eddp->thl, phcsum);
373 qeth_eddp_check_tcp6_hdr(struct qeth_eddp_data *eddp, int data_len)
376 u32 phcsum; /* pseudo header checksum */
378 QETH_DBF_TEXT(trace, 5, "eddpckt6");
379 eddp->th.tcp.h.check = 0;
380 /* compute pseudo header checksum */
381 phcsum = csum_partial((u8 *)&eddp->nh.ip6.h.saddr,
382 sizeof(struct in6_addr), 0);
383 phcsum = csum_partial((u8 *)&eddp->nh.ip6.h.daddr,
384 sizeof(struct in6_addr), phcsum);
385 proto = htonl(IPPROTO_TCP);
386 phcsum = csum_partial((u8 *)&proto, sizeof(u32), phcsum);
390 static inline struct qeth_eddp_data *
391 qeth_eddp_create_eddp_data(struct qeth_hdr *qh, u8 *nh, u8 nhl, u8 *th, u8 thl)
393 struct qeth_eddp_data *eddp;
395 QETH_DBF_TEXT(trace, 5, "eddpcrda");
396 eddp = kmalloc(sizeof(struct qeth_eddp_data), GFP_ATOMIC);
398 memset(eddp, 0, sizeof(struct qeth_eddp_data));
401 memcpy(&eddp->qh, qh, sizeof(struct qeth_hdr));
402 memcpy(&eddp->nh, nh, nhl);
403 memcpy(&eddp->th, th, thl);
404 eddp->frag = -1; /* initially we're in skb->data */
410 __qeth_eddp_fill_context_tcp(struct qeth_eddp_context *ctx,
411 struct qeth_eddp_data *eddp)
417 QETH_DBF_TEXT(trace, 5, "eddpftcp");
418 eddp->skb_offset = sizeof(struct qeth_hdr) + eddp->nhl + eddp->thl;
419 tcph = eddp->skb->h.th;
420 while (eddp->skb_offset < eddp->skb->len) {
421 data_len = min((int)skb_shinfo(eddp->skb)->tso_size,
422 (int)(eddp->skb->len - eddp->skb_offset));
423 /* prepare qdio hdr */
424 if (eddp->qh.hdr.l2.id == QETH_HEADER_TYPE_LAYER2){
425 eddp->qh.hdr.l2.pkt_length = data_len + ETH_HLEN +
426 eddp->nhl + eddp->thl -
427 sizeof(struct qeth_hdr);
428 #ifdef CONFIG_QETH_VLAN
429 if (eddp->mac.h_proto == __constant_htons(ETH_P_8021Q))
430 eddp->qh.hdr.l2.pkt_length += VLAN_HLEN;
431 #endif /* CONFIG_QETH_VLAN */
433 eddp->qh.hdr.l3.length = data_len + eddp->nhl +
436 if (eddp->skb->protocol == ETH_P_IP){
437 eddp->nh.ip4.h.tot_len = data_len + eddp->nhl +
439 eddp->nh.ip4.h.check = 0;
440 eddp->nh.ip4.h.check =
441 ip_fast_csum((u8 *)&eddp->nh.ip4.h,
444 eddp->nh.ip6.h.payload_len = data_len + eddp->thl;
445 /* prepare tcp hdr */
446 if (data_len == (eddp->skb->len - eddp->skb_offset)){
447 /* last segment -> set FIN and PSH flags */
448 eddp->th.tcp.h.fin = tcph->fin;
449 eddp->th.tcp.h.psh = tcph->psh;
451 if (eddp->skb->protocol == ETH_P_IP)
452 hcsum = qeth_eddp_check_tcp4_hdr(eddp, data_len);
454 hcsum = qeth_eddp_check_tcp6_hdr(eddp, data_len);
455 /* fill the next segment into the context */
456 qeth_eddp_create_segment_hdrs(ctx, eddp, data_len);
457 qeth_eddp_create_segment_data_tcp(ctx, eddp, data_len, hcsum);
458 if (eddp->skb_offset >= eddp->skb->len)
460 /* prepare headers for next round */
461 if (eddp->skb->protocol == ETH_P_IP)
463 eddp->th.tcp.h.seq += data_len;
468 qeth_eddp_fill_context_tcp(struct qeth_eddp_context *ctx,
469 struct sk_buff *skb, struct qeth_hdr *qhdr)
471 struct qeth_eddp_data *eddp = NULL;
473 QETH_DBF_TEXT(trace, 5, "eddpficx");
474 /* create our segmentation headers and copy original headers */
475 if (skb->protocol == ETH_P_IP)
476 eddp = qeth_eddp_create_eddp_data(qhdr, (u8 *)skb->nh.iph,
478 (u8 *)skb->h.th, skb->h.th->doff*4);
480 eddp = qeth_eddp_create_eddp_data(qhdr, (u8 *)skb->nh.ipv6h,
481 sizeof(struct ipv6hdr),
482 (u8 *)skb->h.th, skb->h.th->doff*4);
485 QETH_DBF_TEXT(trace, 2, "eddpfcnm");
488 if (qhdr->hdr.l2.id == QETH_HEADER_TYPE_LAYER2) {
489 memcpy(&eddp->mac, eth_hdr(skb), ETH_HLEN);
490 #ifdef CONFIG_QETH_VLAN
491 if (eddp->mac.h_proto == __constant_htons(ETH_P_8021Q)) {
492 eddp->vlan[0] = __constant_htons(skb->protocol);
493 eddp->vlan[1] = htons(vlan_tx_tag_get(skb));
495 #endif /* CONFIG_QETH_VLAN */
497 /* the next flags will only be set on the last segment */
498 eddp->th.tcp.h.fin = 0;
499 eddp->th.tcp.h.psh = 0;
501 /* begin segmentation and fill context */
502 __qeth_eddp_fill_context_tcp(ctx, eddp);
508 qeth_eddp_calc_num_pages(struct qeth_eddp_context *ctx, struct sk_buff *skb,
513 QETH_DBF_TEXT(trace, 5, "eddpcanp");
514 /* can we put multiple skbs in one page? */
515 skbs_per_page = PAGE_SIZE / (skb_shinfo(skb)->tso_size + hdr_len);
516 if (skbs_per_page > 1){
517 ctx->num_pages = (skb_shinfo(skb)->tso_segs + 1) /
519 ctx->elements_per_skb = 1;
521 /* no -> how many elements per skb? */
522 ctx->elements_per_skb = (skb_shinfo(skb)->tso_size + hdr_len +
523 PAGE_SIZE) >> PAGE_SHIFT;
524 ctx->num_pages = ctx->elements_per_skb *
525 (skb_shinfo(skb)->tso_segs + 1);
527 ctx->num_elements = ctx->elements_per_skb *
528 (skb_shinfo(skb)->tso_segs + 1);
531 static inline struct qeth_eddp_context *
532 qeth_eddp_create_context_generic(struct qeth_card *card, struct sk_buff *skb,
535 struct qeth_eddp_context *ctx = NULL;
539 QETH_DBF_TEXT(trace, 5, "creddpcg");
540 /* create the context and allocate pages */
541 ctx = kmalloc(sizeof(struct qeth_eddp_context), GFP_ATOMIC);
543 QETH_DBF_TEXT(trace, 2, "ceddpcn1");
546 memset(ctx, 0, sizeof(struct qeth_eddp_context));
547 ctx->type = QETH_LARGE_SEND_EDDP;
548 qeth_eddp_calc_num_pages(ctx, skb, hdr_len);
549 if (ctx->elements_per_skb > QETH_MAX_BUFFER_ELEMENTS(card)){
550 QETH_DBF_TEXT(trace, 2, "ceddpcis");
554 ctx->pages = kmalloc(ctx->num_pages * sizeof(u8 *), GFP_ATOMIC);
555 if (ctx->pages == NULL){
556 QETH_DBF_TEXT(trace, 2, "ceddpcn2");
560 memset(ctx->pages, 0, ctx->num_pages * sizeof(u8 *));
561 for (i = 0; i < ctx->num_pages; ++i){
562 addr = (u8 *)__get_free_page(GFP_ATOMIC);
564 QETH_DBF_TEXT(trace, 2, "ceddpcn3");
566 qeth_eddp_free_context(ctx);
569 memset(addr, 0, PAGE_SIZE);
570 ctx->pages[i] = addr;
572 ctx->elements = kmalloc(ctx->num_elements *
573 sizeof(struct qeth_eddp_element), GFP_ATOMIC);
574 if (ctx->elements == NULL){
575 QETH_DBF_TEXT(trace, 2, "ceddpcn4");
576 qeth_eddp_free_context(ctx);
579 memset(ctx->elements, 0,
580 ctx->num_elements * sizeof(struct qeth_eddp_element));
581 /* reset num_elements; will be incremented again in fill_buffer to
582 * reflect number of actually used elements */
583 ctx->num_elements = 0;
587 static inline struct qeth_eddp_context *
588 qeth_eddp_create_context_tcp(struct qeth_card *card, struct sk_buff *skb,
589 struct qeth_hdr *qhdr)
591 struct qeth_eddp_context *ctx = NULL;
593 QETH_DBF_TEXT(trace, 5, "creddpct");
594 if (skb->protocol == ETH_P_IP)
595 ctx = qeth_eddp_create_context_generic(card, skb,
596 sizeof(struct qeth_hdr) + skb->nh.iph->ihl*4 +
598 else if (skb->protocol == ETH_P_IPV6)
599 ctx = qeth_eddp_create_context_generic(card, skb,
600 sizeof(struct qeth_hdr) + sizeof(struct ipv6hdr) +
603 QETH_DBF_TEXT(trace, 2, "cetcpinv");
606 QETH_DBF_TEXT(trace, 2, "creddpnl");
609 if (qeth_eddp_fill_context_tcp(ctx, skb, qhdr)){
610 QETH_DBF_TEXT(trace, 2, "ceddptfe");
611 qeth_eddp_free_context(ctx);
614 atomic_set(&ctx->refcnt, 1);
618 struct qeth_eddp_context *
619 qeth_eddp_create_context(struct qeth_card *card, struct sk_buff *skb,
620 struct qeth_hdr *qhdr)
622 QETH_DBF_TEXT(trace, 5, "creddpc");
623 switch (skb->sk->sk_protocol){
625 return qeth_eddp_create_context_tcp(card, skb, qhdr);
627 QETH_DBF_TEXT(trace, 2, "eddpinvp");