Automatic merge of /spare/repo/netdev-2.6 branch 8139too-iomap
[linux-2.6] / net / ipv4 / ipvs / ip_vs_core.c
1 /*
2  * IPVS         An implementation of the IP virtual server support for the
3  *              LINUX operating system.  IPVS is now implemented as a module
4  *              over the Netfilter framework. IPVS can be used to build a
5  *              high-performance and highly available server based on a
6  *              cluster of servers.
7  *
8  * Version:     $Id: ip_vs_core.c,v 1.34 2003/05/10 03:05:23 wensong Exp $
9  *
10  * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
11  *              Peter Kese <peter.kese@ijs.si>
12  *              Julian Anastasov <ja@ssi.bg>
13  *
14  *              This program is free software; you can redistribute it and/or
15  *              modify it under the terms of the GNU General Public License
16  *              as published by the Free Software Foundation; either version
17  *              2 of the License, or (at your option) any later version.
18  *
19  * The IPVS code for kernel 2.2 was done by Wensong Zhang and Peter Kese,
20  * with changes/fixes from Julian Anastasov, Lars Marowsky-Bree, Horms
21  * and others.
22  *
23  * Changes:
24  *      Paul `Rusty' Russell            properly handle non-linear skbs
25  *
26  */
27
28 #include <linux/module.h>
29 #include <linux/kernel.h>
30 #include <linux/ip.h>
31 #include <linux/tcp.h>
32 #include <linux/icmp.h>
33
34 #include <net/ip.h>
35 #include <net/tcp.h>
36 #include <net/udp.h>
37 #include <net/icmp.h>                   /* for icmp_send */
38 #include <net/route.h>
39
40 #include <linux/netfilter.h>
41 #include <linux/netfilter_ipv4.h>
42
43 #include <net/ip_vs.h>
44
45
46 EXPORT_SYMBOL(register_ip_vs_scheduler);
47 EXPORT_SYMBOL(unregister_ip_vs_scheduler);
48 EXPORT_SYMBOL(ip_vs_skb_replace);
49 EXPORT_SYMBOL(ip_vs_proto_name);
50 EXPORT_SYMBOL(ip_vs_conn_new);
51 EXPORT_SYMBOL(ip_vs_conn_in_get);
52 EXPORT_SYMBOL(ip_vs_conn_out_get);
53 #ifdef CONFIG_IP_VS_PROTO_TCP
54 EXPORT_SYMBOL(ip_vs_tcp_conn_listen);
55 #endif
56 EXPORT_SYMBOL(ip_vs_conn_put);
57 #ifdef CONFIG_IP_VS_DEBUG
58 EXPORT_SYMBOL(ip_vs_get_debug_level);
59 #endif
60 EXPORT_SYMBOL(ip_vs_make_skb_writable);
61
62
63 /* ID used in ICMP lookups */
64 #define icmp_id(icmph)          (((icmph)->un).echo.id)
65
66 const char *ip_vs_proto_name(unsigned proto)
67 {
68         static char buf[20];
69
70         switch (proto) {
71         case IPPROTO_IP:
72                 return "IP";
73         case IPPROTO_UDP:
74                 return "UDP";
75         case IPPROTO_TCP:
76                 return "TCP";
77         case IPPROTO_ICMP:
78                 return "ICMP";
79         default:
80                 sprintf(buf, "IP_%d", proto);
81                 return buf;
82         }
83 }
84
85 void ip_vs_init_hash_table(struct list_head *table, int rows)
86 {
87         while (--rows >= 0)
88                 INIT_LIST_HEAD(&table[rows]);
89 }
90
91 static inline void
92 ip_vs_in_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
93 {
94         struct ip_vs_dest *dest = cp->dest;
95         if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
96                 spin_lock(&dest->stats.lock);
97                 dest->stats.inpkts++;
98                 dest->stats.inbytes += skb->len;
99                 spin_unlock(&dest->stats.lock);
100
101                 spin_lock(&dest->svc->stats.lock);
102                 dest->svc->stats.inpkts++;
103                 dest->svc->stats.inbytes += skb->len;
104                 spin_unlock(&dest->svc->stats.lock);
105
106                 spin_lock(&ip_vs_stats.lock);
107                 ip_vs_stats.inpkts++;
108                 ip_vs_stats.inbytes += skb->len;
109                 spin_unlock(&ip_vs_stats.lock);
110         }
111 }
112
113
114 static inline void
115 ip_vs_out_stats(struct ip_vs_conn *cp, struct sk_buff *skb)
116 {
117         struct ip_vs_dest *dest = cp->dest;
118         if (dest && (dest->flags & IP_VS_DEST_F_AVAILABLE)) {
119                 spin_lock(&dest->stats.lock);
120                 dest->stats.outpkts++;
121                 dest->stats.outbytes += skb->len;
122                 spin_unlock(&dest->stats.lock);
123
124                 spin_lock(&dest->svc->stats.lock);
125                 dest->svc->stats.outpkts++;
126                 dest->svc->stats.outbytes += skb->len;
127                 spin_unlock(&dest->svc->stats.lock);
128
129                 spin_lock(&ip_vs_stats.lock);
130                 ip_vs_stats.outpkts++;
131                 ip_vs_stats.outbytes += skb->len;
132                 spin_unlock(&ip_vs_stats.lock);
133         }
134 }
135
136
137 static inline void
138 ip_vs_conn_stats(struct ip_vs_conn *cp, struct ip_vs_service *svc)
139 {
140         spin_lock(&cp->dest->stats.lock);
141         cp->dest->stats.conns++;
142         spin_unlock(&cp->dest->stats.lock);
143
144         spin_lock(&svc->stats.lock);
145         svc->stats.conns++;
146         spin_unlock(&svc->stats.lock);
147
148         spin_lock(&ip_vs_stats.lock);
149         ip_vs_stats.conns++;
150         spin_unlock(&ip_vs_stats.lock);
151 }
152
153
154 static inline int
155 ip_vs_set_state(struct ip_vs_conn *cp, int direction,
156                 const struct sk_buff *skb,
157                 struct ip_vs_protocol *pp)
158 {
159         if (unlikely(!pp->state_transition))
160                 return 0;
161         return pp->state_transition(cp, direction, skb, pp);
162 }
163
164
165 int ip_vs_make_skb_writable(struct sk_buff **pskb, int writable_len)
166 {
167         struct sk_buff *skb = *pskb;
168
169         /* skb is already used, better copy skb and its payload */
170         if (unlikely(skb_shared(skb) || skb->sk))
171                 goto copy_skb;
172
173         /* skb data is already used, copy it */
174         if (unlikely(skb_cloned(skb)))
175                 goto copy_data;
176
177         return pskb_may_pull(skb, writable_len);
178
179   copy_data:
180         if (unlikely(writable_len > skb->len))
181                 return 0;
182         return !pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
183
184   copy_skb:
185         if (unlikely(writable_len > skb->len))
186                 return 0;
187         skb = skb_copy(skb, GFP_ATOMIC);
188         if (!skb)
189                 return 0;
190         BUG_ON(skb_is_nonlinear(skb));
191
192         /* Rest of kernel will get very unhappy if we pass it a
193            suddenly-orphaned skbuff */
194         if ((*pskb)->sk)
195                 skb_set_owner_w(skb, (*pskb)->sk);
196         kfree_skb(*pskb);
197         *pskb = skb;
198         return 1;
199 }
200
201 /*
202  *  IPVS persistent scheduling function
203  *  It creates a connection entry according to its template if exists,
204  *  or selects a server and creates a connection entry plus a template.
205  *  Locking: we are svc user (svc->refcnt), so we hold all dests too
206  *  Protocols supported: TCP, UDP
207  */
208 static struct ip_vs_conn *
209 ip_vs_sched_persist(struct ip_vs_service *svc,
210                     const struct sk_buff *skb,
211                     __u16 ports[2])
212 {
213         struct ip_vs_conn *cp = NULL;
214         struct iphdr *iph = skb->nh.iph;
215         struct ip_vs_dest *dest;
216         struct ip_vs_conn *ct;
217         __u16  dport;    /* destination port to forward */
218         __u32  snet;     /* source network of the client, after masking */
219
220         /* Mask saddr with the netmask to adjust template granularity */
221         snet = iph->saddr & svc->netmask;
222
223         IP_VS_DBG(6, "p-schedule: src %u.%u.%u.%u:%u dest %u.%u.%u.%u:%u "
224                   "mnet %u.%u.%u.%u\n",
225                   NIPQUAD(iph->saddr), ntohs(ports[0]),
226                   NIPQUAD(iph->daddr), ntohs(ports[1]),
227                   NIPQUAD(snet));
228
229         /*
230          * As far as we know, FTP is a very complicated network protocol, and
231          * it uses control connection and data connections. For active FTP,
232          * FTP server initialize data connection to the client, its source port
233          * is often 20. For passive FTP, FTP server tells the clients the port
234          * that it passively listens to,  and the client issues the data
235          * connection. In the tunneling or direct routing mode, the load
236          * balancer is on the client-to-server half of connection, the port
237          * number is unknown to the load balancer. So, a conn template like
238          * <caddr, 0, vaddr, 0, daddr, 0> is created for persistent FTP
239          * service, and a template like <caddr, 0, vaddr, vport, daddr, dport>
240          * is created for other persistent services.
241          */
242         if (ports[1] == svc->port) {
243                 /* Check if a template already exists */
244                 if (svc->port != FTPPORT)
245                         ct = ip_vs_conn_in_get(iph->protocol, snet, 0,
246                                                iph->daddr, ports[1]);
247                 else
248                         ct = ip_vs_conn_in_get(iph->protocol, snet, 0,
249                                                iph->daddr, 0);
250
251                 if (!ct || !ip_vs_check_template(ct)) {
252                         /*
253                          * No template found or the dest of the connection
254                          * template is not available.
255                          */
256                         dest = svc->scheduler->schedule(svc, skb);
257                         if (dest == NULL) {
258                                 IP_VS_DBG(1, "p-schedule: no dest found.\n");
259                                 return NULL;
260                         }
261
262                         /*
263                          * Create a template like <protocol,caddr,0,
264                          * vaddr,vport,daddr,dport> for non-ftp service,
265                          * and <protocol,caddr,0,vaddr,0,daddr,0>
266                          * for ftp service.
267                          */
268                         if (svc->port != FTPPORT)
269                                 ct = ip_vs_conn_new(iph->protocol,
270                                                     snet, 0,
271                                                     iph->daddr,
272                                                     ports[1],
273                                                     dest->addr, dest->port,
274                                                     0,
275                                                     dest);
276                         else
277                                 ct = ip_vs_conn_new(iph->protocol,
278                                                     snet, 0,
279                                                     iph->daddr, 0,
280                                                     dest->addr, 0,
281                                                     0,
282                                                     dest);
283                         if (ct == NULL)
284                                 return NULL;
285
286                         ct->timeout = svc->timeout;
287                 } else {
288                         /* set destination with the found template */
289                         dest = ct->dest;
290                 }
291                 dport = dest->port;
292         } else {
293                 /*
294                  * Note: persistent fwmark-based services and persistent
295                  * port zero service are handled here.
296                  * fwmark template: <IPPROTO_IP,caddr,0,fwmark,0,daddr,0>
297                  * port zero template: <protocol,caddr,0,vaddr,0,daddr,0>
298                  */
299                 if (svc->fwmark)
300                         ct = ip_vs_conn_in_get(IPPROTO_IP, snet, 0,
301                                                htonl(svc->fwmark), 0);
302                 else
303                         ct = ip_vs_conn_in_get(iph->protocol, snet, 0,
304                                                iph->daddr, 0);
305
306                 if (!ct || !ip_vs_check_template(ct)) {
307                         /*
308                          * If it is not persistent port zero, return NULL,
309                          * otherwise create a connection template.
310                          */
311                         if (svc->port)
312                                 return NULL;
313
314                         dest = svc->scheduler->schedule(svc, skb);
315                         if (dest == NULL) {
316                                 IP_VS_DBG(1, "p-schedule: no dest found.\n");
317                                 return NULL;
318                         }
319
320                         /*
321                          * Create a template according to the service
322                          */
323                         if (svc->fwmark)
324                                 ct = ip_vs_conn_new(IPPROTO_IP,
325                                                     snet, 0,
326                                                     htonl(svc->fwmark), 0,
327                                                     dest->addr, 0,
328                                                     0,
329                                                     dest);
330                         else
331                                 ct = ip_vs_conn_new(iph->protocol,
332                                                     snet, 0,
333                                                     iph->daddr, 0,
334                                                     dest->addr, 0,
335                                                     0,
336                                                     dest);
337                         if (ct == NULL)
338                                 return NULL;
339
340                         ct->timeout = svc->timeout;
341                 } else {
342                         /* set destination with the found template */
343                         dest = ct->dest;
344                 }
345                 dport = ports[1];
346         }
347
348         /*
349          *    Create a new connection according to the template
350          */
351         cp = ip_vs_conn_new(iph->protocol,
352                             iph->saddr, ports[0],
353                             iph->daddr, ports[1],
354                             dest->addr, dport,
355                             0,
356                             dest);
357         if (cp == NULL) {
358                 ip_vs_conn_put(ct);
359                 return NULL;
360         }
361
362         /*
363          *    Add its control
364          */
365         ip_vs_control_add(cp, ct);
366         ip_vs_conn_put(ct);
367
368         ip_vs_conn_stats(cp, svc);
369         return cp;
370 }
371
372
373 /*
374  *  IPVS main scheduling function
375  *  It selects a server according to the virtual service, and
376  *  creates a connection entry.
377  *  Protocols supported: TCP, UDP
378  */
379 struct ip_vs_conn *
380 ip_vs_schedule(struct ip_vs_service *svc, const struct sk_buff *skb)
381 {
382         struct ip_vs_conn *cp = NULL;
383         struct iphdr *iph = skb->nh.iph;
384         struct ip_vs_dest *dest;
385         __u16 _ports[2], *pptr;
386
387         pptr = skb_header_pointer(skb, iph->ihl*4,
388                                   sizeof(_ports), _ports);
389         if (pptr == NULL)
390                 return NULL;
391
392         /*
393          *    Persistent service
394          */
395         if (svc->flags & IP_VS_SVC_F_PERSISTENT)
396                 return ip_vs_sched_persist(svc, skb, pptr);
397
398         /*
399          *    Non-persistent service
400          */
401         if (!svc->fwmark && pptr[1] != svc->port) {
402                 if (!svc->port)
403                         IP_VS_ERR("Schedule: port zero only supported "
404                                   "in persistent services, "
405                                   "check your ipvs configuration\n");
406                 return NULL;
407         }
408
409         dest = svc->scheduler->schedule(svc, skb);
410         if (dest == NULL) {
411                 IP_VS_DBG(1, "Schedule: no dest found.\n");
412                 return NULL;
413         }
414
415         /*
416          *    Create a connection entry.
417          */
418         cp = ip_vs_conn_new(iph->protocol,
419                             iph->saddr, pptr[0],
420                             iph->daddr, pptr[1],
421                             dest->addr, dest->port?dest->port:pptr[1],
422                             0,
423                             dest);
424         if (cp == NULL)
425                 return NULL;
426
427         IP_VS_DBG(6, "Schedule fwd:%c c:%u.%u.%u.%u:%u v:%u.%u.%u.%u:%u "
428                   "d:%u.%u.%u.%u:%u flg:%X cnt:%d\n",
429                   ip_vs_fwd_tag(cp),
430                   NIPQUAD(cp->caddr), ntohs(cp->cport),
431                   NIPQUAD(cp->vaddr), ntohs(cp->vport),
432                   NIPQUAD(cp->daddr), ntohs(cp->dport),
433                   cp->flags, atomic_read(&cp->refcnt));
434
435         ip_vs_conn_stats(cp, svc);
436         return cp;
437 }
438
439
440 /*
441  *  Pass or drop the packet.
442  *  Called by ip_vs_in, when the virtual service is available but
443  *  no destination is available for a new connection.
444  */
445 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb,
446                 struct ip_vs_protocol *pp)
447 {
448         __u16 _ports[2], *pptr;
449         struct iphdr *iph = skb->nh.iph;
450
451         pptr = skb_header_pointer(skb, iph->ihl*4,
452                                   sizeof(_ports), _ports);
453         if (pptr == NULL) {
454                 ip_vs_service_put(svc);
455                 return NF_DROP;
456         }
457
458         /* if it is fwmark-based service, the cache_bypass sysctl is up
459            and the destination is RTN_UNICAST (and not local), then create
460            a cache_bypass connection entry */
461         if (sysctl_ip_vs_cache_bypass && svc->fwmark
462             && (inet_addr_type(iph->daddr) == RTN_UNICAST)) {
463                 int ret, cs;
464                 struct ip_vs_conn *cp;
465
466                 ip_vs_service_put(svc);
467
468                 /* create a new connection entry */
469                 IP_VS_DBG(6, "ip_vs_leave: create a cache_bypass entry\n");
470                 cp = ip_vs_conn_new(iph->protocol,
471                                     iph->saddr, pptr[0],
472                                     iph->daddr, pptr[1],
473                                     0, 0,
474                                     IP_VS_CONN_F_BYPASS,
475                                     NULL);
476                 if (cp == NULL)
477                         return NF_DROP;
478
479                 /* statistics */
480                 ip_vs_in_stats(cp, skb);
481
482                 /* set state */
483                 cs = ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pp);
484
485                 /* transmit the first SYN packet */
486                 ret = cp->packet_xmit(skb, cp, pp);
487                 /* do not touch skb anymore */
488
489                 atomic_inc(&cp->in_pkts);
490                 ip_vs_conn_put(cp);
491                 return ret;
492         }
493
494         /*
495          * When the virtual ftp service is presented, packets destined
496          * for other services on the VIP may get here (except services
497          * listed in the ipvs table), pass the packets, because it is
498          * not ipvs job to decide to drop the packets.
499          */
500         if ((svc->port == FTPPORT) && (pptr[1] != FTPPORT)) {
501                 ip_vs_service_put(svc);
502                 return NF_ACCEPT;
503         }
504
505         ip_vs_service_put(svc);
506
507         /*
508          * Notify the client that the destination is unreachable, and
509          * release the socket buffer.
510          * Since it is in IP layer, the TCP socket is not actually
511          * created, the TCP RST packet cannot be sent, instead that
512          * ICMP_PORT_UNREACH is sent here no matter it is TCP/UDP. --WZ
513          */
514         icmp_send(skb, ICMP_DEST_UNREACH, ICMP_PORT_UNREACH, 0);
515         return NF_DROP;
516 }
517
518
519 /*
520  *      It is hooked before NF_IP_PRI_NAT_SRC at the NF_IP_POST_ROUTING
521  *      chain, and is used for VS/NAT.
522  *      It detects packets for VS/NAT connections and sends the packets
523  *      immediately. This can avoid that iptable_nat mangles the packets
524  *      for VS/NAT.
525  */
526 static unsigned int ip_vs_post_routing(unsigned int hooknum,
527                                        struct sk_buff **pskb,
528                                        const struct net_device *in,
529                                        const struct net_device *out,
530                                        int (*okfn)(struct sk_buff *))
531 {
532         if (!((*pskb)->nfcache & NFC_IPVS_PROPERTY))
533                 return NF_ACCEPT;
534
535         /* The packet was sent from IPVS, exit this chain */
536         (*okfn)(*pskb);
537
538         return NF_STOLEN;
539 }
540
541 u16 ip_vs_checksum_complete(struct sk_buff *skb, int offset)
542 {
543         return (u16) csum_fold(skb_checksum(skb, offset, skb->len - offset, 0));
544 }
545
546 static inline struct sk_buff *
547 ip_vs_gather_frags(struct sk_buff *skb, u_int32_t user)
548 {
549         skb = ip_defrag(skb, user);
550         if (skb)
551                 ip_send_check(skb->nh.iph);
552         return skb;
553 }
554
555 /*
556  * Packet has been made sufficiently writable in caller
557  * - inout: 1=in->out, 0=out->in
558  */
559 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp,
560                     struct ip_vs_conn *cp, int inout)
561 {
562         struct iphdr *iph        = skb->nh.iph;
563         unsigned int icmp_offset = iph->ihl*4;
564         struct icmphdr *icmph    = (struct icmphdr *)(skb->nh.raw + icmp_offset);
565         struct iphdr *ciph       = (struct iphdr *)(icmph + 1);
566
567         if (inout) {
568                 iph->saddr = cp->vaddr;
569                 ip_send_check(iph);
570                 ciph->daddr = cp->vaddr;
571                 ip_send_check(ciph);
572         } else {
573                 iph->daddr = cp->daddr;
574                 ip_send_check(iph);
575                 ciph->saddr = cp->daddr;
576                 ip_send_check(ciph);
577         }
578
579         /* the TCP/UDP port */
580         if (IPPROTO_TCP == ciph->protocol || IPPROTO_UDP == ciph->protocol) {
581                 __u16 *ports = (void *)ciph + ciph->ihl*4;
582
583                 if (inout)
584                         ports[1] = cp->vport;
585                 else
586                         ports[0] = cp->dport;
587         }
588
589         /* And finally the ICMP checksum */
590         icmph->checksum = 0;
591         icmph->checksum = ip_vs_checksum_complete(skb, icmp_offset);
592         skb->ip_summed = CHECKSUM_UNNECESSARY;
593
594         if (inout)
595                 IP_VS_DBG_PKT(11, pp, skb, (void *)ciph - (void *)iph,
596                         "Forwarding altered outgoing ICMP");
597         else
598                 IP_VS_DBG_PKT(11, pp, skb, (void *)ciph - (void *)iph,
599                         "Forwarding altered incoming ICMP");
600 }
601
602 /*
603  *      Handle ICMP messages in the inside-to-outside direction (outgoing).
604  *      Find any that might be relevant, check against existing connections,
605  *      forward to the right destination host if relevant.
606  *      Currently handles error types - unreachable, quench, ttl exceeded.
607  *      (Only used in VS/NAT)
608  */
609 static int ip_vs_out_icmp(struct sk_buff **pskb, int *related)
610 {
611         struct sk_buff *skb = *pskb;
612         struct iphdr *iph;
613         struct icmphdr  _icmph, *ic;
614         struct iphdr    _ciph, *cih;    /* The ip header contained within the ICMP */
615         struct ip_vs_conn *cp;
616         struct ip_vs_protocol *pp;
617         unsigned int offset, ihl, verdict;
618
619         *related = 1;
620
621         /* reassemble IP fragments */
622         if (skb->nh.iph->frag_off & __constant_htons(IP_MF|IP_OFFSET)) {
623                 skb = ip_vs_gather_frags(skb, IP_DEFRAG_VS_OUT);
624                 if (!skb)
625                         return NF_STOLEN;
626                 *pskb = skb;
627         }
628
629         iph = skb->nh.iph;
630         offset = ihl = iph->ihl * 4;
631         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
632         if (ic == NULL)
633                 return NF_DROP;
634
635         IP_VS_DBG(12, "Outgoing ICMP (%d,%d) %u.%u.%u.%u->%u.%u.%u.%u\n",
636                   ic->type, ntohs(icmp_id(ic)),
637                   NIPQUAD(iph->saddr), NIPQUAD(iph->daddr));
638
639         /*
640          * Work through seeing if this is for us.
641          * These checks are supposed to be in an order that means easy
642          * things are checked first to speed up processing.... however
643          * this means that some packets will manage to get a long way
644          * down this stack and then be rejected, but that's life.
645          */
646         if ((ic->type != ICMP_DEST_UNREACH) &&
647             (ic->type != ICMP_SOURCE_QUENCH) &&
648             (ic->type != ICMP_TIME_EXCEEDED)) {
649                 *related = 0;
650                 return NF_ACCEPT;
651         }
652
653         /* Now find the contained IP header */
654         offset += sizeof(_icmph);
655         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
656         if (cih == NULL)
657                 return NF_ACCEPT; /* The packet looks wrong, ignore */
658
659         pp = ip_vs_proto_get(cih->protocol);
660         if (!pp)
661                 return NF_ACCEPT;
662
663         /* Is the embedded protocol header present? */
664         if (unlikely(cih->frag_off & __constant_htons(IP_OFFSET) &&
665                      pp->dont_defrag))
666                 return NF_ACCEPT;
667
668         IP_VS_DBG_PKT(11, pp, skb, offset, "Checking outgoing ICMP for");
669
670         offset += cih->ihl * 4;
671
672         /* The embedded headers contain source and dest in reverse order */
673         cp = pp->conn_out_get(skb, pp, cih, offset, 1);
674         if (!cp)
675                 return NF_ACCEPT;
676
677         verdict = NF_DROP;
678
679         if (IP_VS_FWD_METHOD(cp) != 0) {
680                 IP_VS_ERR("shouldn't reach here, because the box is on the"
681                           "half connection in the tun/dr module.\n");
682         }
683
684         /* Ensure the checksum is correct */
685         if (skb->ip_summed != CHECKSUM_UNNECESSARY &&
686             ip_vs_checksum_complete(skb, ihl)) {
687                 /* Failed checksum! */
688                 IP_VS_DBG(1, "Forward ICMP: failed checksum from %d.%d.%d.%d!\n",
689                           NIPQUAD(iph->saddr));
690                 goto out;
691         }
692
693         if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol)
694                 offset += 2 * sizeof(__u16);
695         if (!ip_vs_make_skb_writable(pskb, offset))
696                 goto out;
697         skb = *pskb;
698
699         ip_vs_nat_icmp(skb, pp, cp, 1);
700
701         /* do the statistics and put it back */
702         ip_vs_out_stats(cp, skb);
703
704         skb->nfcache |= NFC_IPVS_PROPERTY;
705         verdict = NF_ACCEPT;
706
707   out:
708         __ip_vs_conn_put(cp);
709
710         return verdict;
711 }
712
713 static inline int is_tcp_reset(const struct sk_buff *skb)
714 {
715         struct tcphdr _tcph, *th;
716
717         th = skb_header_pointer(skb, skb->nh.iph->ihl * 4,
718                                 sizeof(_tcph), &_tcph);
719         if (th == NULL)
720                 return 0;
721         return th->rst;
722 }
723
724 /*
725  *      It is hooked at the NF_IP_FORWARD chain, used only for VS/NAT.
726  *      Check if outgoing packet belongs to the established ip_vs_conn,
727  *      rewrite addresses of the packet and send it on its way...
728  */
729 static unsigned int
730 ip_vs_out(unsigned int hooknum, struct sk_buff **pskb,
731           const struct net_device *in, const struct net_device *out,
732           int (*okfn)(struct sk_buff *))
733 {
734         struct sk_buff  *skb = *pskb;
735         struct iphdr    *iph;
736         struct ip_vs_protocol *pp;
737         struct ip_vs_conn *cp;
738         int ihl;
739
740         EnterFunction(11);
741
742         if (skb->nfcache & NFC_IPVS_PROPERTY)
743                 return NF_ACCEPT;
744
745         iph = skb->nh.iph;
746         if (unlikely(iph->protocol == IPPROTO_ICMP)) {
747                 int related, verdict = ip_vs_out_icmp(pskb, &related);
748
749                 if (related)
750                         return verdict;
751                 skb = *pskb;
752                 iph = skb->nh.iph;
753         }
754
755         pp = ip_vs_proto_get(iph->protocol);
756         if (unlikely(!pp))
757                 return NF_ACCEPT;
758
759         /* reassemble IP fragments */
760         if (unlikely(iph->frag_off & __constant_htons(IP_MF|IP_OFFSET) &&
761                      !pp->dont_defrag)) {
762                 skb = ip_vs_gather_frags(skb, IP_DEFRAG_VS_OUT);
763                 if (!skb)
764                         return NF_STOLEN;
765                 iph = skb->nh.iph;
766                 *pskb = skb;
767         }
768
769         ihl = iph->ihl << 2;
770
771         /*
772          * Check if the packet belongs to an existing entry
773          */
774         cp = pp->conn_out_get(skb, pp, iph, ihl, 0);
775
776         if (unlikely(!cp)) {
777                 if (sysctl_ip_vs_nat_icmp_send &&
778                     (pp->protocol == IPPROTO_TCP ||
779                      pp->protocol == IPPROTO_UDP)) {
780                         __u16 _ports[2], *pptr;
781
782                         pptr = skb_header_pointer(skb, ihl,
783                                                   sizeof(_ports), _ports);
784                         if (pptr == NULL)
785                                 return NF_ACCEPT;       /* Not for me */
786                         if (ip_vs_lookup_real_service(iph->protocol,
787                                                       iph->saddr, pptr[0])) {
788                                 /*
789                                  * Notify the real server: there is no
790                                  * existing entry if it is not RST
791                                  * packet or not TCP packet.
792                                  */
793                                 if (iph->protocol != IPPROTO_TCP
794                                     || !is_tcp_reset(skb)) {
795                                         icmp_send(skb,ICMP_DEST_UNREACH,
796                                                   ICMP_PORT_UNREACH, 0);
797                                         return NF_DROP;
798                                 }
799                         }
800                 }
801                 IP_VS_DBG_PKT(12, pp, skb, 0,
802                               "packet continues traversal as normal");
803                 return NF_ACCEPT;
804         }
805
806         IP_VS_DBG_PKT(11, pp, skb, 0, "Outgoing packet");
807
808         if (!ip_vs_make_skb_writable(pskb, ihl))
809                 goto drop;
810
811         /* mangle the packet */
812         if (pp->snat_handler && !pp->snat_handler(pskb, pp, cp))
813                 goto drop;
814         skb = *pskb;
815         skb->nh.iph->saddr = cp->vaddr;
816         ip_send_check(skb->nh.iph);
817
818         IP_VS_DBG_PKT(10, pp, skb, 0, "After SNAT");
819
820         ip_vs_out_stats(cp, skb);
821         ip_vs_set_state(cp, IP_VS_DIR_OUTPUT, skb, pp);
822         ip_vs_conn_put(cp);
823
824         skb->nfcache |= NFC_IPVS_PROPERTY;
825
826         LeaveFunction(11);
827         return NF_ACCEPT;
828
829   drop:
830         ip_vs_conn_put(cp);
831         kfree_skb(*pskb);
832         return NF_STOLEN;
833 }
834
835
836 /*
837  *      Handle ICMP messages in the outside-to-inside direction (incoming).
838  *      Find any that might be relevant, check against existing connections,
839  *      forward to the right destination host if relevant.
840  *      Currently handles error types - unreachable, quench, ttl exceeded.
841  */
842 static int 
843 ip_vs_in_icmp(struct sk_buff **pskb, int *related, unsigned int hooknum)
844 {
845         struct sk_buff *skb = *pskb;
846         struct iphdr *iph;
847         struct icmphdr  _icmph, *ic;
848         struct iphdr    _ciph, *cih;    /* The ip header contained within the ICMP */
849         struct ip_vs_conn *cp;
850         struct ip_vs_protocol *pp;
851         unsigned int offset, ihl, verdict;
852
853         *related = 1;
854
855         /* reassemble IP fragments */
856         if (skb->nh.iph->frag_off & __constant_htons(IP_MF|IP_OFFSET)) {
857                 skb = ip_vs_gather_frags(skb,
858                                          hooknum == NF_IP_LOCAL_IN ?
859                                          IP_DEFRAG_VS_IN : IP_DEFRAG_VS_FWD);
860                 if (!skb)
861                         return NF_STOLEN;
862                 *pskb = skb;
863         }
864
865         iph = skb->nh.iph;
866         offset = ihl = iph->ihl * 4;
867         ic = skb_header_pointer(skb, offset, sizeof(_icmph), &_icmph);
868         if (ic == NULL)
869                 return NF_DROP;
870
871         IP_VS_DBG(12, "Incoming ICMP (%d,%d) %u.%u.%u.%u->%u.%u.%u.%u\n",
872                   ic->type, ntohs(icmp_id(ic)),
873                   NIPQUAD(iph->saddr), NIPQUAD(iph->daddr));
874
875         /*
876          * Work through seeing if this is for us.
877          * These checks are supposed to be in an order that means easy
878          * things are checked first to speed up processing.... however
879          * this means that some packets will manage to get a long way
880          * down this stack and then be rejected, but that's life.
881          */
882         if ((ic->type != ICMP_DEST_UNREACH) &&
883             (ic->type != ICMP_SOURCE_QUENCH) &&
884             (ic->type != ICMP_TIME_EXCEEDED)) {
885                 *related = 0;
886                 return NF_ACCEPT;
887         }
888
889         /* Now find the contained IP header */
890         offset += sizeof(_icmph);
891         cih = skb_header_pointer(skb, offset, sizeof(_ciph), &_ciph);
892         if (cih == NULL)
893                 return NF_ACCEPT; /* The packet looks wrong, ignore */
894
895         pp = ip_vs_proto_get(cih->protocol);
896         if (!pp)
897                 return NF_ACCEPT;
898
899         /* Is the embedded protocol header present? */
900         if (unlikely(cih->frag_off & __constant_htons(IP_OFFSET) &&
901                      pp->dont_defrag))
902                 return NF_ACCEPT;
903
904         IP_VS_DBG_PKT(11, pp, skb, offset, "Checking incoming ICMP for");
905
906         offset += cih->ihl * 4;
907
908         /* The embedded headers contain source and dest in reverse order */
909         cp = pp->conn_in_get(skb, pp, cih, offset, 1);
910         if (!cp)
911                 return NF_ACCEPT;
912
913         verdict = NF_DROP;
914
915         /* Ensure the checksum is correct */
916         if (skb->ip_summed != CHECKSUM_UNNECESSARY &&
917             ip_vs_checksum_complete(skb, ihl)) {
918                 /* Failed checksum! */
919                 IP_VS_DBG(1, "Incoming ICMP: failed checksum from %d.%d.%d.%d!\n",
920                           NIPQUAD(iph->saddr));
921                 goto out;
922         }
923
924         /* do the statistics and put it back */
925         ip_vs_in_stats(cp, skb);
926         if (IPPROTO_TCP == cih->protocol || IPPROTO_UDP == cih->protocol)
927                 offset += 2 * sizeof(__u16);
928         verdict = ip_vs_icmp_xmit(skb, cp, pp, offset);
929         /* do not touch skb anymore */
930
931   out:
932         __ip_vs_conn_put(cp);
933
934         return verdict;
935 }
936
937 /*
938  *      Check if it's for virtual services, look it up,
939  *      and send it on its way...
940  */
941 static unsigned int
942 ip_vs_in(unsigned int hooknum, struct sk_buff **pskb,
943          const struct net_device *in, const struct net_device *out,
944          int (*okfn)(struct sk_buff *))
945 {
946         struct sk_buff  *skb = *pskb;
947         struct iphdr    *iph;
948         struct ip_vs_protocol *pp;
949         struct ip_vs_conn *cp;
950         int ret, restart;
951         int ihl;
952
953         /*
954          *      Big tappo: only PACKET_HOST (neither loopback nor mcasts)
955          *      ... don't know why 1st test DOES NOT include 2nd (?)
956          */
957         if (unlikely(skb->pkt_type != PACKET_HOST
958                      || skb->dev == &loopback_dev || skb->sk)) {
959                 IP_VS_DBG(12, "packet type=%d proto=%d daddr=%d.%d.%d.%d ignored\n",
960                           skb->pkt_type,
961                           skb->nh.iph->protocol,
962                           NIPQUAD(skb->nh.iph->daddr));
963                 return NF_ACCEPT;
964         }
965
966         iph = skb->nh.iph;
967         if (unlikely(iph->protocol == IPPROTO_ICMP)) {
968                 int related, verdict = ip_vs_in_icmp(pskb, &related, hooknum);
969
970                 if (related)
971                         return verdict;
972                 skb = *pskb;
973                 iph = skb->nh.iph;
974         }
975
976         /* Protocol supported? */
977         pp = ip_vs_proto_get(iph->protocol);
978         if (unlikely(!pp))
979                 return NF_ACCEPT;
980
981         ihl = iph->ihl << 2;
982
983         /*
984          * Check if the packet belongs to an existing connection entry
985          */
986         cp = pp->conn_in_get(skb, pp, iph, ihl, 0);
987
988         if (unlikely(!cp)) {
989                 int v;
990
991                 if (!pp->conn_schedule(skb, pp, &v, &cp))
992                         return v;
993         }
994
995         if (unlikely(!cp)) {
996                 /* sorry, all this trouble for a no-hit :) */
997                 IP_VS_DBG_PKT(12, pp, skb, 0,
998                               "packet continues traversal as normal");
999                 return NF_ACCEPT;
1000         }
1001
1002         IP_VS_DBG_PKT(11, pp, skb, 0, "Incoming packet");
1003
1004         /* Check the server status */
1005         if (cp->dest && !(cp->dest->flags & IP_VS_DEST_F_AVAILABLE)) {
1006                 /* the destination server is not available */
1007
1008                 if (sysctl_ip_vs_expire_nodest_conn) {
1009                         /* try to expire the connection immediately */
1010                         ip_vs_conn_expire_now(cp);
1011                 } else {
1012                         /* don't restart its timer, and silently
1013                            drop the packet. */
1014                         __ip_vs_conn_put(cp);
1015                 }
1016                 return NF_DROP;
1017         }
1018
1019         ip_vs_in_stats(cp, skb);
1020         restart = ip_vs_set_state(cp, IP_VS_DIR_INPUT, skb, pp);
1021         if (cp->packet_xmit)
1022                 ret = cp->packet_xmit(skb, cp, pp);
1023                 /* do not touch skb anymore */
1024         else {
1025                 IP_VS_DBG_RL("warning: packet_xmit is null");
1026                 ret = NF_ACCEPT;
1027         }
1028
1029         /* increase its packet counter and check if it is needed
1030            to be synchronized */
1031         atomic_inc(&cp->in_pkts);
1032         if ((ip_vs_sync_state & IP_VS_STATE_MASTER) &&
1033             (cp->protocol != IPPROTO_TCP ||
1034              cp->state == IP_VS_TCP_S_ESTABLISHED) &&
1035             (atomic_read(&cp->in_pkts) % sysctl_ip_vs_sync_threshold[1]
1036              == sysctl_ip_vs_sync_threshold[0]))
1037                 ip_vs_sync_conn(cp);
1038
1039         ip_vs_conn_put(cp);
1040         return ret;
1041 }
1042
1043
1044 /*
1045  *      It is hooked at the NF_IP_FORWARD chain, in order to catch ICMP
1046  *      related packets destined for 0.0.0.0/0.
1047  *      When fwmark-based virtual service is used, such as transparent
1048  *      cache cluster, TCP packets can be marked and routed to ip_vs_in,
1049  *      but ICMP destined for 0.0.0.0/0 cannot not be easily marked and
1050  *      sent to ip_vs_in_icmp. So, catch them at the NF_IP_FORWARD chain
1051  *      and send them to ip_vs_in_icmp.
1052  */
1053 static unsigned int
1054 ip_vs_forward_icmp(unsigned int hooknum, struct sk_buff **pskb,
1055                    const struct net_device *in, const struct net_device *out,
1056                    int (*okfn)(struct sk_buff *))
1057 {
1058         int r;
1059
1060         if ((*pskb)->nh.iph->protocol != IPPROTO_ICMP)
1061                 return NF_ACCEPT;
1062
1063         return ip_vs_in_icmp(pskb, &r, hooknum);
1064 }
1065
1066
1067 /* After packet filtering, forward packet through VS/DR, VS/TUN,
1068    or VS/NAT(change destination), so that filtering rules can be
1069    applied to IPVS. */
1070 static struct nf_hook_ops ip_vs_in_ops = {
1071         .hook           = ip_vs_in,
1072         .owner          = THIS_MODULE,
1073         .pf             = PF_INET,
1074         .hooknum        = NF_IP_LOCAL_IN,
1075         .priority       = 100,
1076 };
1077
1078 /* After packet filtering, change source only for VS/NAT */
1079 static struct nf_hook_ops ip_vs_out_ops = {
1080         .hook           = ip_vs_out,
1081         .owner          = THIS_MODULE,
1082         .pf             = PF_INET,
1083         .hooknum        = NF_IP_FORWARD,
1084         .priority       = 100,
1085 };
1086
1087 /* After packet filtering (but before ip_vs_out_icmp), catch icmp
1088    destined for 0.0.0.0/0, which is for incoming IPVS connections */
1089 static struct nf_hook_ops ip_vs_forward_icmp_ops = {
1090         .hook           = ip_vs_forward_icmp,
1091         .owner          = THIS_MODULE,
1092         .pf             = PF_INET,
1093         .hooknum        = NF_IP_FORWARD,
1094         .priority       = 99,
1095 };
1096
1097 /* Before the netfilter connection tracking, exit from POST_ROUTING */
1098 static struct nf_hook_ops ip_vs_post_routing_ops = {
1099         .hook           = ip_vs_post_routing,
1100         .owner          = THIS_MODULE,
1101         .pf             = PF_INET,
1102         .hooknum        = NF_IP_POST_ROUTING,
1103         .priority       = NF_IP_PRI_NAT_SRC-1,
1104 };
1105
1106
1107 /*
1108  *      Initialize IP Virtual Server
1109  */
1110 static int __init ip_vs_init(void)
1111 {
1112         int ret;
1113
1114         ret = ip_vs_control_init();
1115         if (ret < 0) {
1116                 IP_VS_ERR("can't setup control.\n");
1117                 goto cleanup_nothing;
1118         }
1119
1120         ip_vs_protocol_init();
1121
1122         ret = ip_vs_app_init();
1123         if (ret < 0) {
1124                 IP_VS_ERR("can't setup application helper.\n");
1125                 goto cleanup_protocol;
1126         }
1127
1128         ret = ip_vs_conn_init();
1129         if (ret < 0) {
1130                 IP_VS_ERR("can't setup connection table.\n");
1131                 goto cleanup_app;
1132         }
1133
1134         ret = nf_register_hook(&ip_vs_in_ops);
1135         if (ret < 0) {
1136                 IP_VS_ERR("can't register in hook.\n");
1137                 goto cleanup_conn;
1138         }
1139
1140         ret = nf_register_hook(&ip_vs_out_ops);
1141         if (ret < 0) {
1142                 IP_VS_ERR("can't register out hook.\n");
1143                 goto cleanup_inops;
1144         }
1145         ret = nf_register_hook(&ip_vs_post_routing_ops);
1146         if (ret < 0) {
1147                 IP_VS_ERR("can't register post_routing hook.\n");
1148                 goto cleanup_outops;
1149         }
1150         ret = nf_register_hook(&ip_vs_forward_icmp_ops);
1151         if (ret < 0) {
1152                 IP_VS_ERR("can't register forward_icmp hook.\n");
1153                 goto cleanup_postroutingops;
1154         }
1155
1156         IP_VS_INFO("ipvs loaded.\n");
1157         return ret;
1158
1159   cleanup_postroutingops:
1160         nf_unregister_hook(&ip_vs_post_routing_ops);
1161   cleanup_outops:
1162         nf_unregister_hook(&ip_vs_out_ops);
1163   cleanup_inops:
1164         nf_unregister_hook(&ip_vs_in_ops);
1165   cleanup_conn:
1166         ip_vs_conn_cleanup();
1167   cleanup_app:
1168         ip_vs_app_cleanup();
1169   cleanup_protocol:
1170         ip_vs_protocol_cleanup();
1171         ip_vs_control_cleanup();
1172   cleanup_nothing:
1173         return ret;
1174 }
1175
1176 static void __exit ip_vs_cleanup(void)
1177 {
1178         nf_unregister_hook(&ip_vs_forward_icmp_ops);
1179         nf_unregister_hook(&ip_vs_post_routing_ops);
1180         nf_unregister_hook(&ip_vs_out_ops);
1181         nf_unregister_hook(&ip_vs_in_ops);
1182         ip_vs_conn_cleanup();
1183         ip_vs_app_cleanup();
1184         ip_vs_protocol_cleanup();
1185         ip_vs_control_cleanup();
1186         IP_VS_INFO("ipvs unloaded.\n");
1187 }
1188
1189 module_init(ip_vs_init);
1190 module_exit(ip_vs_cleanup);
1191 MODULE_LICENSE("GPL");