git.oblomov.eu Git - linux-2.6/blob - net/sched/sch_cbq.c

   1 /*
   2  * net/sched/sch_cbq.c  Class-Based Queueing discipline.
   3  *
   4  *              This program is free software; you can redistribute it and/or
   5  *              modify it under the terms of the GNU General Public License
   6  *              as published by the Free Software Foundation; either version
   7  *              2 of the License, or (at your option) any later version.
   8  *
   9  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  10  *
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/types.h>
  15 #include <linux/kernel.h>
  16 #include <linux/string.h>
  17 #include <linux/errno.h>
  18 #include <linux/skbuff.h>
  19 #include <net/netlink.h>
  20 #include <net/pkt_sched.h>
  21
  22
  23 /*      Class-Based Queueing (CBQ) algorithm.
  24         =======================================
  25
  26         Sources: [1] Sally Floyd and Van Jacobson, "Link-sharing and Resource
  27                  Management Models for Packet Networks",
  28                  IEEE/ACM Transactions on Networking, Vol.3, No.4, 1995
  29
  30                  [2] Sally Floyd, "Notes on CBQ and Guaranteed Service", 1995
  31
  32                  [3] Sally Floyd, "Notes on Class-Based Queueing: Setting
  33                  Parameters", 1996
  34
  35                  [4] Sally Floyd and Michael Speer, "Experimental Results
  36                  for Class-Based Queueing", 1998, not published.
  37
  38         -----------------------------------------------------------------------
  39
  40         Algorithm skeleton was taken from NS simulator cbq.cc.
  41         If someone wants to check this code against the LBL version,
  42         he should take into account that ONLY the skeleton was borrowed,
  43         the implementation is different. Particularly:
  44
  45         --- The WRR algorithm is different. Our version looks more
  46         reasonable (I hope) and works when quanta are allowed to be
  47         less than MTU, which is always the case when real time classes
  48         have small rates. Note, that the statement of [3] is
  49         incomplete, delay may actually be estimated even if class
  50         per-round allotment is less than MTU. Namely, if per-round
  51         allotment is W*r_i, and r_1+...+r_k = r < 1
  52
  53         delay_i <= ([MTU/(W*r_i)]*W*r + W*r + k*MTU)/B
  54
  55         In the worst case we have IntServ estimate with D = W*r+k*MTU
  56         and C = MTU*r. The proof (if correct at all) is trivial.
  57
  58
  59         --- It seems that cbq-2.0 is not very accurate. At least, I cannot
  60         interpret some places, which look like wrong translations
  61         from NS. Anyone is advised to find these differences
  62         and explain to me, why I am wrong 8).
  63
  64         --- Linux has no EOI event, so that we cannot estimate true class
  65         idle time. Workaround is to consider the next dequeue event
  66         as sign that previous packet is finished. This is wrong because of
  67         internal device queueing, but on a permanently loaded link it is true.
  68         Moreover, combined with clock integrator, this scheme looks
  69         very close to an ideal solution.  */
  70
  71 struct cbq_sched_data;
  72
  73
  74 struct cbq_class
  75 {
  76         struct cbq_class        *next;          /* hash table link */
  77         struct cbq_class        *next_alive;    /* next class with backlog in this priority band */
  78
  79 /* Parameters */
  80         u32                     classid;
  81         unsigned char           priority;       /* class priority */
  82         unsigned char           priority2;      /* priority to be used after overlimit */
  83         unsigned char           ewma_log;       /* time constant for idle time calculation */
  84         unsigned char           ovl_strategy;
  85 #ifdef CONFIG_NET_CLS_POLICE
  86         unsigned char           police;
  87 #endif
  88
  89         u32                     defmap;
  90
  91         /* Link-sharing scheduler parameters */
  92         long                    maxidle;        /* Class parameters: see below. */
  93         long                    offtime;
  94         long                    minidle;
  95         u32                     avpkt;
  96         struct qdisc_rate_table *R_tab;
  97
  98         /* Overlimit strategy parameters */
  99         void                    (*overlimit)(struct cbq_class *cl);
 100         psched_tdiff_t          penalty;
 101
 102         /* General scheduler (WRR) parameters */
 103         long                    allot;
 104         long                    quantum;        /* Allotment per WRR round */
 105         long                    weight;         /* Relative allotment: see below */
 106
 107         struct Qdisc            *qdisc;         /* Ptr to CBQ discipline */
 108         struct cbq_class        *split;         /* Ptr to split node */
 109         struct cbq_class        *share;         /* Ptr to LS parent in the class tree */
 110         struct cbq_class        *tparent;       /* Ptr to tree parent in the class tree */
 111         struct cbq_class        *borrow;        /* NULL if class is bandwidth limited;
 112                                                    parent otherwise */
 113         struct cbq_class        *sibling;       /* Sibling chain */
 114         struct cbq_class        *children;      /* Pointer to children chain */
 115
 116         struct Qdisc            *q;             /* Elementary queueing discipline */
 117
 118
 119 /* Variables */
 120         unsigned char           cpriority;      /* Effective priority */
 121         unsigned char           delayed;
 122         unsigned char           level;          /* level of the class in hierarchy:
 123                                                    0 for leaf classes, and maximal
 124                                                    level of children + 1 for nodes.
 125                                                  */
 126
 127         psched_time_t           last;           /* Last end of service */
 128         psched_time_t           undertime;
 129         long                    avgidle;
 130         long                    deficit;        /* Saved deficit for WRR */
 131         psched_time_t           penalized;
 132         struct gnet_stats_basic bstats;
 133         struct gnet_stats_queue qstats;
 134         struct gnet_stats_rate_est rate_est;
 135         struct tc_cbq_xstats    xstats;
 136
 137         struct tcf_proto        *filter_list;
 138
 139         int                     refcnt;
 140         int                     filters;
 141
 142         struct cbq_class        *defaults[TC_PRIO_MAX+1];
 143 };
 144
 145 struct cbq_sched_data
 146 {
 147         struct cbq_class        *classes[16];           /* Hash table of all classes */
 148         int                     nclasses[TC_CBQ_MAXPRIO+1];
 149         unsigned                quanta[TC_CBQ_MAXPRIO+1];
 150
 151         struct cbq_class        link;
 152
 153         unsigned                activemask;
 154         struct cbq_class        *active[TC_CBQ_MAXPRIO+1];      /* List of all classes
 155                                                                    with backlog */
 156
 157 #ifdef CONFIG_NET_CLS_POLICE
 158         struct cbq_class        *rx_class;
 159 #endif
 160         struct cbq_class        *tx_class;
 161         struct cbq_class        *tx_borrowed;
 162         int                     tx_len;
 163         psched_time_t           now;            /* Cached timestamp */
 164         psched_time_t           now_rt;         /* Cached real time */
 165         unsigned                pmask;
 166
 167         struct hrtimer          delay_timer;
 168         struct qdisc_watchdog   watchdog;       /* Watchdog timer,
 169                                                    started when CBQ has
 170                                                    backlog, but cannot
 171                                                    transmit just now */
 172         psched_tdiff_t          wd_expires;
 173         int                     toplevel;
 174         u32                     hgenerator;
 175 };
 176
 177
 178 #define L2T(cl,len)     ((cl)->R_tab->data[(len)>>(cl)->R_tab->rate.cell_log])
 179
 180
 181 static __inline__ unsigned cbq_hash(u32 h)
 182 {
 183         h ^= h>>8;
 184         h ^= h>>4;
 185         return h&0xF;
 186 }
 187
 188 static __inline__ struct cbq_class *
 189 cbq_class_lookup(struct cbq_sched_data *q, u32 classid)
 190 {
 191         struct cbq_class *cl;
 192
 193         for (cl = q->classes[cbq_hash(classid)]; cl; cl = cl->next)
 194                 if (cl->classid == classid)
 195                         return cl;
 196         return NULL;
 197 }
 198
 199 #ifdef CONFIG_NET_CLS_POLICE
 200
 201 static struct cbq_class *
 202 cbq_reclassify(struct sk_buff *skb, struct cbq_class *this)
 203 {
 204         struct cbq_class *cl, *new;
 205
 206         for (cl = this->tparent; cl; cl = cl->tparent)
 207                 if ((new = cl->defaults[TC_PRIO_BESTEFFORT]) != NULL && new != this)
 208                         return new;
 209
 210         return NULL;
 211 }
 212
 213 #endif
 214
 215 /* Classify packet. The procedure is pretty complicated, but
 216    it allows us to combine link sharing and priority scheduling
 217    transparently.
 218
 219    Namely, you can put link sharing rules (f.e. route based) at root of CBQ,
 220    so that it resolves to split nodes. Then packets are classified
 221    by logical priority, or a more specific classifier may be attached
 222    to the split node.
 223  */
 224
 225 static struct cbq_class *
 226 cbq_classify(struct sk_buff *skb, struct Qdisc *sch, int *qerr)
 227 {
 228         struct cbq_sched_data *q = qdisc_priv(sch);
 229         struct cbq_class *head = &q->link;
 230         struct cbq_class **defmap;
 231         struct cbq_class *cl = NULL;
 232         u32 prio = skb->priority;
 233         struct tcf_result res;
 234
 235         /*
 236          *  Step 1. If skb->priority points to one of our classes, use it.
 237          */
 238         if (TC_H_MAJ(prio^sch->handle) == 0 &&
 239             (cl = cbq_class_lookup(q, prio)) != NULL)
 240                 return cl;
 241
 242         *qerr = NET_XMIT_BYPASS;
 243         for (;;) {
 244                 int result = 0;
 245                 defmap = head->defaults;
 246
 247                 /*
 248                  * Step 2+n. Apply classifier.
 249                  */
 250                 if (!head->filter_list || (result = tc_classify(skb, head->filter_list, &res)) < 0)
 251                         goto fallback;
 252
 253                 if ((cl = (void*)res.class) == NULL) {
 254                         if (TC_H_MAJ(res.classid))
 255                                 cl = cbq_class_lookup(q, res.classid);
 256                         else if ((cl = defmap[res.classid&TC_PRIO_MAX]) == NULL)
 257                                 cl = defmap[TC_PRIO_BESTEFFORT];
 258
 259                         if (cl == NULL || cl->level >= head->level)
 260                                 goto fallback;
 261                 }
 262
 263 #ifdef CONFIG_NET_CLS_ACT
 264                 switch (result) {
 265                 case TC_ACT_QUEUED:
 266                 case TC_ACT_STOLEN:
 267                         *qerr = NET_XMIT_SUCCESS;
 268                 case TC_ACT_SHOT:
 269                         return NULL;
 270                 }
 271 #elif defined(CONFIG_NET_CLS_POLICE)
 272                 switch (result) {
 273                 case TC_POLICE_RECLASSIFY:
 274                         return cbq_reclassify(skb, cl);
 275                 case TC_POLICE_SHOT:
 276                         return NULL;
 277                 default:
 278                         break;
 279                 }
 280 #endif
 281                 if (cl->level == 0)
 282                         return cl;
 283
 284                 /*
 285                  * Step 3+n. If classifier selected a link sharing class,
 286                  *         apply agency specific classifier.
 287                  *         Repeat this procdure until we hit a leaf node.
 288                  */
 289                 head = cl;
 290         }
 291
 292 fallback:
 293         cl = head;
 294
 295         /*
 296          * Step 4. No success...
 297          */
 298         if (TC_H_MAJ(prio) == 0 &&
 299             !(cl = head->defaults[prio&TC_PRIO_MAX]) &&
 300             !(cl = head->defaults[TC_PRIO_BESTEFFORT]))
 301                 return head;
 302
 303         return cl;
 304 }
 305
 306 /*
 307    A packet has just been enqueued on the empty class.
 308    cbq_activate_class adds it to the tail of active class list
 309    of its priority band.
 310  */
 311
 312 static __inline__ void cbq_activate_class(struct cbq_class *cl)
 313 {
 314         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
 315         int prio = cl->cpriority;
 316         struct cbq_class *cl_tail;
 317
 318         cl_tail = q->active[prio];
 319         q->active[prio] = cl;
 320
 321         if (cl_tail != NULL) {
 322                 cl->next_alive = cl_tail->next_alive;
 323                 cl_tail->next_alive = cl;
 324         } else {
 325                 cl->next_alive = cl;
 326                 q->activemask |= (1<<prio);
 327         }
 328 }
 329
 330 /*
 331    Unlink class from active chain.
 332    Note that this same procedure is done directly in cbq_dequeue*
 333    during round-robin procedure.
 334  */
 335
 336 static void cbq_deactivate_class(struct cbq_class *this)
 337 {
 338         struct cbq_sched_data *q = qdisc_priv(this->qdisc);
 339         int prio = this->cpriority;
 340         struct cbq_class *cl;
 341         struct cbq_class *cl_prev = q->active[prio];
 342
 343         do {
 344                 cl = cl_prev->next_alive;
 345                 if (cl == this) {
 346                         cl_prev->next_alive = cl->next_alive;
 347                         cl->next_alive = NULL;
 348
 349                         if (cl == q->active[prio]) {
 350                                 q->active[prio] = cl_prev;
 351                                 if (cl == q->active[prio]) {
 352                                         q->active[prio] = NULL;
 353                                         q->activemask &= ~(1<<prio);
 354                                         return;
 355                                 }
 356                         }
 357                         return;
 358                 }
 359         } while ((cl_prev = cl) != q->active[prio]);
 360 }
 361
 362 static void
 363 cbq_mark_toplevel(struct cbq_sched_data *q, struct cbq_class *cl)
 364 {
 365         int toplevel = q->toplevel;
 366
 367         if (toplevel > cl->level && !(cl->q->flags&TCQ_F_THROTTLED)) {
 368                 psched_time_t now;
 369                 psched_tdiff_t incr;
 370
 371                 now = psched_get_time();
 372                 incr = now - q->now_rt;
 373                 now = q->now + incr;
 374
 375                 do {
 376                         if (cl->undertime < now) {
 377                                 q->toplevel = cl->level;
 378                                 return;
 379                         }
 380                 } while ((cl=cl->borrow) != NULL && toplevel > cl->level);
 381         }
 382 }
 383
 384 static int
 385 cbq_enqueue(struct sk_buff *skb, struct Qdisc *sch)
 386 {
 387         struct cbq_sched_data *q = qdisc_priv(sch);
 388         int len = skb->len;
 389         int ret;
 390         struct cbq_class *cl = cbq_classify(skb, sch, &ret);
 391
 392 #ifdef CONFIG_NET_CLS_POLICE
 393         q->rx_class = cl;
 394 #endif
 395         if (cl == NULL) {
 396                 if (ret == NET_XMIT_BYPASS)
 397                         sch->qstats.drops++;
 398                 kfree_skb(skb);
 399                 return ret;
 400         }
 401
 402 #ifdef CONFIG_NET_CLS_POLICE
 403         cl->q->__parent = sch;
 404 #endif
 405         if ((ret = cl->q->enqueue(skb, cl->q)) == NET_XMIT_SUCCESS) {
 406                 sch->q.qlen++;
 407                 sch->bstats.packets++;
 408                 sch->bstats.bytes+=len;
 409                 cbq_mark_toplevel(q, cl);
 410                 if (!cl->next_alive)
 411                         cbq_activate_class(cl);
 412                 return ret;
 413         }
 414
 415         sch->qstats.drops++;
 416         cbq_mark_toplevel(q, cl);
 417         cl->qstats.drops++;
 418         return ret;
 419 }
 420
 421 static int
 422 cbq_requeue(struct sk_buff *skb, struct Qdisc *sch)
 423 {
 424         struct cbq_sched_data *q = qdisc_priv(sch);
 425         struct cbq_class *cl;
 426         int ret;
 427
 428         if ((cl = q->tx_class) == NULL) {
 429                 kfree_skb(skb);
 430                 sch->qstats.drops++;
 431                 return NET_XMIT_CN;
 432         }
 433         q->tx_class = NULL;
 434
 435         cbq_mark_toplevel(q, cl);
 436
 437 #ifdef CONFIG_NET_CLS_POLICE
 438         q->rx_class = cl;
 439         cl->q->__parent = sch;
 440 #endif
 441         if ((ret = cl->q->ops->requeue(skb, cl->q)) == 0) {
 442                 sch->q.qlen++;
 443                 sch->qstats.requeues++;
 444                 if (!cl->next_alive)
 445                         cbq_activate_class(cl);
 446                 return 0;
 447         }
 448         sch->qstats.drops++;
 449         cl->qstats.drops++;
 450         return ret;
 451 }
 452
 453 /* Overlimit actions */
 454
 455 /* TC_CBQ_OVL_CLASSIC: (default) penalize leaf class by adding offtime */
 456
 457 static void cbq_ovl_classic(struct cbq_class *cl)
 458 {
 459         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
 460         psched_tdiff_t delay = cl->undertime - q->now;
 461
 462         if (!cl->delayed) {
 463                 delay += cl->offtime;
 464
 465                 /*
 466                    Class goes to sleep, so that it will have no
 467                    chance to work avgidle. Let's forgive it 8)
 468
 469                    BTW cbq-2.0 has a crap in this
 470                    place, apparently they forgot to shift it by cl->ewma_log.
 471                  */
 472                 if (cl->avgidle < 0)
 473                         delay -= (-cl->avgidle) - ((-cl->avgidle) >> cl->ewma_log);
 474                 if (cl->avgidle < cl->minidle)
 475                         cl->avgidle = cl->minidle;
 476                 if (delay <= 0)
 477                         delay = 1;
 478                 cl->undertime = q->now + delay;
 479
 480                 cl->xstats.overactions++;
 481                 cl->delayed = 1;
 482         }
 483         if (q->wd_expires == 0 || q->wd_expires > delay)
 484                 q->wd_expires = delay;
 485
 486         /* Dirty work! We must schedule wakeups based on
 487            real available rate, rather than leaf rate,
 488            which may be tiny (even zero).
 489          */
 490         if (q->toplevel == TC_CBQ_MAXLEVEL) {
 491                 struct cbq_class *b;
 492                 psched_tdiff_t base_delay = q->wd_expires;
 493
 494                 for (b = cl->borrow; b; b = b->borrow) {
 495                         delay = b->undertime - q->now;
 496                         if (delay < base_delay) {
 497                                 if (delay <= 0)
 498                                         delay = 1;
 499                                 base_delay = delay;
 500                         }
 501                 }
 502
 503                 q->wd_expires = base_delay;
 504         }
 505 }
 506
 507 /* TC_CBQ_OVL_RCLASSIC: penalize by offtime classes in hierarchy, when
 508    they go overlimit
 509  */
 510
 511 static void cbq_ovl_rclassic(struct cbq_class *cl)
 512 {
 513         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
 514         struct cbq_class *this = cl;
 515
 516         do {
 517                 if (cl->level > q->toplevel) {
 518                         cl = NULL;
 519                         break;
 520                 }
 521         } while ((cl = cl->borrow) != NULL);
 522
 523         if (cl == NULL)
 524                 cl = this;
 525         cbq_ovl_classic(cl);
 526 }
 527
 528 /* TC_CBQ_OVL_DELAY: delay until it will go to underlimit */
 529
 530 static void cbq_ovl_delay(struct cbq_class *cl)
 531 {
 532         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
 533         psched_tdiff_t delay = cl->undertime - q->now;
 534
 535         if (!cl->delayed) {
 536                 psched_time_t sched = q->now;
 537                 ktime_t expires;
 538
 539                 delay += cl->offtime;
 540                 if (cl->avgidle < 0)
 541                         delay -= (-cl->avgidle) - ((-cl->avgidle) >> cl->ewma_log);
 542                 if (cl->avgidle < cl->minidle)
 543                         cl->avgidle = cl->minidle;
 544                 cl->undertime = q->now + delay;
 545
 546                 if (delay > 0) {
 547                         sched += delay + cl->penalty;
 548                         cl->penalized = sched;
 549                         cl->cpriority = TC_CBQ_MAXPRIO;
 550                         q->pmask |= (1<<TC_CBQ_MAXPRIO);
 551
 552                         expires = ktime_set(0, 0);
 553                         expires = ktime_add_ns(expires, PSCHED_US2NS(sched));
 554                         if (hrtimer_try_to_cancel(&q->delay_timer) &&
 555                             ktime_to_ns(ktime_sub(q->delay_timer.expires,
 556                                                   expires)) > 0)
 557                                 q->delay_timer.expires = expires;
 558                         hrtimer_restart(&q->delay_timer);
 559                         cl->delayed = 1;
 560                         cl->xstats.overactions++;
 561                         return;
 562                 }
 563                 delay = 1;
 564         }
 565         if (q->wd_expires == 0 || q->wd_expires > delay)
 566                 q->wd_expires = delay;
 567 }
 568
 569 /* TC_CBQ_OVL_LOWPRIO: penalize class by lowering its priority band */
 570
 571 static void cbq_ovl_lowprio(struct cbq_class *cl)
 572 {
 573         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
 574
 575         cl->penalized = q->now + cl->penalty;
 576
 577         if (cl->cpriority != cl->priority2) {
 578                 cl->cpriority = cl->priority2;
 579                 q->pmask |= (1<<cl->cpriority);
 580                 cl->xstats.overactions++;
 581         }
 582         cbq_ovl_classic(cl);
 583 }
 584
 585 /* TC_CBQ_OVL_DROP: penalize class by dropping */
 586
 587 static void cbq_ovl_drop(struct cbq_class *cl)
 588 {
 589         if (cl->q->ops->drop)
 590                 if (cl->q->ops->drop(cl->q))
 591                         cl->qdisc->q.qlen--;
 592         cl->xstats.overactions++;
 593         cbq_ovl_classic(cl);
 594 }
 595
 596 static psched_tdiff_t cbq_undelay_prio(struct cbq_sched_data *q, int prio,
 597                                        psched_time_t now)
 598 {
 599         struct cbq_class *cl;
 600         struct cbq_class *cl_prev = q->active[prio];
 601         psched_time_t sched = now;
 602
 603         if (cl_prev == NULL)
 604                 return 0;
 605
 606         do {
 607                 cl = cl_prev->next_alive;
 608                 if (now - cl->penalized > 0) {
 609                         cl_prev->next_alive = cl->next_alive;
 610                         cl->next_alive = NULL;
 611                         cl->cpriority = cl->priority;
 612                         cl->delayed = 0;
 613                         cbq_activate_class(cl);
 614
 615                         if (cl == q->active[prio]) {
 616                                 q->active[prio] = cl_prev;
 617                                 if (cl == q->active[prio]) {
 618                                         q->active[prio] = NULL;
 619                                         return 0;
 620                                 }
 621                         }
 622
 623                         cl = cl_prev->next_alive;
 624                 } else if (sched - cl->penalized > 0)
 625                         sched = cl->penalized;
 626         } while ((cl_prev = cl) != q->active[prio]);
 627
 628         return sched - now;
 629 }
 630
 631 static enum hrtimer_restart cbq_undelay(struct hrtimer *timer)
 632 {
 633         struct cbq_sched_data *q = container_of(timer, struct cbq_sched_data,
 634                                                 delay_timer);
 635         struct Qdisc *sch = q->watchdog.qdisc;
 636         psched_time_t now;
 637         psched_tdiff_t delay = 0;
 638         unsigned pmask;
 639
 640         now = psched_get_time();
 641
 642         pmask = q->pmask;
 643         q->pmask = 0;
 644
 645         while (pmask) {
 646                 int prio = ffz(~pmask);
 647                 psched_tdiff_t tmp;
 648
 649                 pmask &= ~(1<<prio);
 650
 651                 tmp = cbq_undelay_prio(q, prio, now);
 652                 if (tmp > 0) {
 653                         q->pmask |= 1<<prio;
 654                         if (tmp < delay || delay == 0)
 655                                 delay = tmp;
 656                 }
 657         }
 658
 659         if (delay) {
 660                 ktime_t time;
 661
 662                 time = ktime_set(0, 0);
 663                 time = ktime_add_ns(time, PSCHED_US2NS(now + delay));
 664                 hrtimer_start(&q->delay_timer, time, HRTIMER_MODE_ABS);
 665         }
 666
 667         sch->flags &= ~TCQ_F_THROTTLED;
 668         netif_schedule(sch->dev);
 669         return HRTIMER_NORESTART;
 670 }
 671
 672
 673 #ifdef CONFIG_NET_CLS_POLICE
 674
 675 static int cbq_reshape_fail(struct sk_buff *skb, struct Qdisc *child)
 676 {
 677         int len = skb->len;
 678         struct Qdisc *sch = child->__parent;
 679         struct cbq_sched_data *q = qdisc_priv(sch);
 680         struct cbq_class *cl = q->rx_class;
 681
 682         q->rx_class = NULL;
 683
 684         if (cl && (cl = cbq_reclassify(skb, cl)) != NULL) {
 685
 686                 cbq_mark_toplevel(q, cl);
 687
 688                 q->rx_class = cl;
 689                 cl->q->__parent = sch;
 690
 691                 if (cl->q->enqueue(skb, cl->q) == 0) {
 692                         sch->q.qlen++;
 693                         sch->bstats.packets++;
 694                         sch->bstats.bytes+=len;
 695                         if (!cl->next_alive)
 696                                 cbq_activate_class(cl);
 697                         return 0;
 698                 }
 699                 sch->qstats.drops++;
 700                 return 0;
 701         }
 702
 703         sch->qstats.drops++;
 704         return -1;
 705 }
 706 #endif
 707
 708 /*
 709    It is mission critical procedure.
 710
 711    We "regenerate" toplevel cutoff, if transmitting class
 712    has backlog and it is not regulated. It is not part of
 713    original CBQ description, but looks more reasonable.
 714    Probably, it is wrong. This question needs further investigation.
 715 */
 716
 717 static __inline__ void
 718 cbq_update_toplevel(struct cbq_sched_data *q, struct cbq_class *cl,
 719                     struct cbq_class *borrowed)
 720 {
 721         if (cl && q->toplevel >= borrowed->level) {
 722                 if (cl->q->q.qlen > 1) {
 723                         do {
 724                                 if (borrowed->undertime == PSCHED_PASTPERFECT) {
 725                                         q->toplevel = borrowed->level;
 726                                         return;
 727                                 }
 728                         } while ((borrowed=borrowed->borrow) != NULL);
 729                 }
 730 #if 0
 731         /* It is not necessary now. Uncommenting it
 732            will save CPU cycles, but decrease fairness.
 733          */
 734                 q->toplevel = TC_CBQ_MAXLEVEL;
 735 #endif
 736         }
 737 }
 738
 739 static void
 740 cbq_update(struct cbq_sched_data *q)
 741 {
 742         struct cbq_class *this = q->tx_class;
 743         struct cbq_class *cl = this;
 744         int len = q->tx_len;
 745
 746         q->tx_class = NULL;
 747
 748         for ( ; cl; cl = cl->share) {
 749                 long avgidle = cl->avgidle;
 750                 long idle;
 751
 752                 cl->bstats.packets++;
 753                 cl->bstats.bytes += len;
 754
 755                 /*
 756                    (now - last) is total time between packet right edges.
 757                    (last_pktlen/rate) is "virtual" busy time, so that
 758
 759                          idle = (now - last) - last_pktlen/rate
 760                  */
 761
 762                 idle = q->now - cl->last;
 763                 if ((unsigned long)idle > 128*1024*1024) {
 764                         avgidle = cl->maxidle;
 765                 } else {
 766                         idle -= L2T(cl, len);
 767
 768                 /* true_avgidle := (1-W)*true_avgidle + W*idle,
 769                    where W=2^{-ewma_log}. But cl->avgidle is scaled:
 770                    cl->avgidle == true_avgidle/W,
 771                    hence:
 772                  */
 773                         avgidle += idle - (avgidle>>cl->ewma_log);
 774                 }
 775
 776                 if (avgidle <= 0) {
 777                         /* Overlimit or at-limit */
 778
 779                         if (avgidle < cl->minidle)
 780                                 avgidle = cl->minidle;
 781
 782                         cl->avgidle = avgidle;
 783
 784                         /* Calculate expected time, when this class
 785                            will be allowed to send.
 786                            It will occur, when:
 787                            (1-W)*true_avgidle + W*delay = 0, i.e.
 788                            idle = (1/W - 1)*(-true_avgidle)
 789                            or
 790                            idle = (1 - W)*(-cl->avgidle);
 791                          */
 792                         idle = (-avgidle) - ((-avgidle) >> cl->ewma_log);
 793
 794                         /*
 795                            That is not all.
 796                            To maintain the rate allocated to the class,
 797                            we add to undertime virtual clock,
 798                            necessary to complete transmitted packet.
 799                            (len/phys_bandwidth has been already passed
 800                            to the moment of cbq_update)
 801                          */
 802
 803                         idle -= L2T(&q->link, len);
 804                         idle += L2T(cl, len);
 805
 806                         cl->undertime = q->now + idle;
 807                 } else {
 808                         /* Underlimit */
 809
 810                         cl->undertime = PSCHED_PASTPERFECT;
 811                         if (avgidle > cl->maxidle)
 812                                 cl->avgidle = cl->maxidle;
 813                         else
 814                                 cl->avgidle = avgidle;
 815                 }
 816                 cl->last = q->now;
 817         }
 818
 819         cbq_update_toplevel(q, this, q->tx_borrowed);
 820 }
 821
 822 static __inline__ struct cbq_class *
 823 cbq_under_limit(struct cbq_class *cl)
 824 {
 825         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
 826         struct cbq_class *this_cl = cl;
 827
 828         if (cl->tparent == NULL)
 829                 return cl;
 830
 831         if (cl->undertime == PSCHED_PASTPERFECT || q->now >= cl->undertime) {
 832                 cl->delayed = 0;
 833                 return cl;
 834         }
 835
 836         do {
 837                 /* It is very suspicious place. Now overlimit
 838                    action is generated for not bounded classes
 839                    only if link is completely congested.
 840                    Though it is in agree with ancestor-only paradigm,
 841                    it looks very stupid. Particularly,
 842                    it means that this chunk of code will either
 843                    never be called or result in strong amplification
 844                    of burstiness. Dangerous, silly, and, however,
 845                    no another solution exists.
 846                  */
 847                 if ((cl = cl->borrow) == NULL) {
 848                         this_cl->qstats.overlimits++;
 849                         this_cl->overlimit(this_cl);
 850                         return NULL;
 851                 }
 852                 if (cl->level > q->toplevel)
 853                         return NULL;
 854         } while (cl->undertime != PSCHED_PASTPERFECT && q->now < cl->undertime);
 855
 856         cl->delayed = 0;
 857         return cl;
 858 }
 859
 860 static __inline__ struct sk_buff *
 861 cbq_dequeue_prio(struct Qdisc *sch, int prio)
 862 {
 863         struct cbq_sched_data *q = qdisc_priv(sch);
 864         struct cbq_class *cl_tail, *cl_prev, *cl;
 865         struct sk_buff *skb;
 866         int deficit;
 867
 868         cl_tail = cl_prev = q->active[prio];
 869         cl = cl_prev->next_alive;
 870
 871         do {
 872                 deficit = 0;
 873
 874                 /* Start round */
 875                 do {
 876                         struct cbq_class *borrow = cl;
 877
 878                         if (cl->q->q.qlen &&
 879                             (borrow = cbq_under_limit(cl)) == NULL)
 880                                 goto skip_class;
 881
 882                         if (cl->deficit <= 0) {
 883                                 /* Class exhausted its allotment per
 884                                    this round. Switch to the next one.
 885                                  */
 886                                 deficit = 1;
 887                                 cl->deficit += cl->quantum;
 888                                 goto next_class;
 889                         }
 890
 891                         skb = cl->q->dequeue(cl->q);
 892
 893                         /* Class did not give us any skb :-(
 894                            It could occur even if cl->q->q.qlen != 0
 895                            f.e. if cl->q == "tbf"
 896                          */
 897                         if (skb == NULL)
 898                                 goto skip_class;
 899
 900                         cl->deficit -= skb->len;
 901                         q->tx_class = cl;
 902                         q->tx_borrowed = borrow;
 903                         if (borrow != cl) {
 904 #ifndef CBQ_XSTATS_BORROWS_BYTES
 905                                 borrow->xstats.borrows++;
 906                                 cl->xstats.borrows++;
 907 #else
 908                                 borrow->xstats.borrows += skb->len;
 909                                 cl->xstats.borrows += skb->len;
 910 #endif
 911                         }
 912                         q->tx_len = skb->len;
 913
 914                         if (cl->deficit <= 0) {
 915                                 q->active[prio] = cl;
 916                                 cl = cl->next_alive;
 917                                 cl->deficit += cl->quantum;
 918                         }
 919                         return skb;
 920
 921 skip_class:
 922                         if (cl->q->q.qlen == 0 || prio != cl->cpriority) {
 923                                 /* Class is empty or penalized.
 924                                    Unlink it from active chain.
 925                                  */
 926                                 cl_prev->next_alive = cl->next_alive;
 927                                 cl->next_alive = NULL;
 928
 929                                 /* Did cl_tail point to it? */
 930                                 if (cl == cl_tail) {
 931                                         /* Repair it! */
 932                                         cl_tail = cl_prev;
 933
 934                                         /* Was it the last class in this band? */
 935                                         if (cl == cl_tail) {
 936                                                 /* Kill the band! */
 937                                                 q->active[prio] = NULL;
 938                                                 q->activemask &= ~(1<<prio);
 939                                                 if (cl->q->q.qlen)
 940                                                         cbq_activate_class(cl);
 941                                                 return NULL;
 942                                         }
 943
 944                                         q->active[prio] = cl_tail;
 945                                 }
 946                                 if (cl->q->q.qlen)
 947                                         cbq_activate_class(cl);
 948
 949                                 cl = cl_prev;
 950                         }
 951
 952 next_class:
 953                         cl_prev = cl;
 954                         cl = cl->next_alive;
 955                 } while (cl_prev != cl_tail);
 956         } while (deficit);
 957
 958         q->active[prio] = cl_prev;
 959
 960         return NULL;
 961 }
 962
 963 static __inline__ struct sk_buff *
 964 cbq_dequeue_1(struct Qdisc *sch)
 965 {
 966         struct cbq_sched_data *q = qdisc_priv(sch);
 967         struct sk_buff *skb;
 968         unsigned activemask;
 969
 970         activemask = q->activemask&0xFF;
 971         while (activemask) {
 972                 int prio = ffz(~activemask);
 973                 activemask &= ~(1<<prio);
 974                 skb = cbq_dequeue_prio(sch, prio);
 975                 if (skb)
 976                         return skb;
 977         }
 978         return NULL;
 979 }
 980
 981 static struct sk_buff *
 982 cbq_dequeue(struct Qdisc *sch)
 983 {
 984         struct sk_buff *skb;
 985         struct cbq_sched_data *q = qdisc_priv(sch);
 986         psched_time_t now;
 987         psched_tdiff_t incr;
 988
 989         now = psched_get_time();
 990         incr = now - q->now_rt;
 991
 992         if (q->tx_class) {
 993                 psched_tdiff_t incr2;
 994                 /* Time integrator. We calculate EOS time
 995                    by adding expected packet transmission time.
 996                    If real time is greater, we warp artificial clock,
 997                    so that:
 998
 999                    cbq_time = max(real_time, work);
1000                  */
1001                 incr2 = L2T(&q->link, q->tx_len);
1002                 q->now += incr2;
1003                 cbq_update(q);
1004                 if ((incr -= incr2) < 0)
1005                         incr = 0;
1006         }
1007         q->now += incr;
1008         q->now_rt = now;
1009
1010         for (;;) {
1011                 q->wd_expires = 0;
1012
1013                 skb = cbq_dequeue_1(sch);
1014                 if (skb) {
1015                         sch->q.qlen--;
1016                         sch->flags &= ~TCQ_F_THROTTLED;
1017                         return skb;
1018                 }
1019
1020                 /* All the classes are overlimit.
1021
1022                    It is possible, if:
1023
1024                    1. Scheduler is empty.
1025                    2. Toplevel cutoff inhibited borrowing.
1026                    3. Root class is overlimit.
1027
1028                    Reset 2d and 3d conditions and retry.
1029
1030                    Note, that NS and cbq-2.0 are buggy, peeking
1031                    an arbitrary class is appropriate for ancestor-only
1032                    sharing, but not for toplevel algorithm.
1033
1034                    Our version is better, but slower, because it requires
1035                    two passes, but it is unavoidable with top-level sharing.
1036                 */
1037
1038                 if (q->toplevel == TC_CBQ_MAXLEVEL &&
1039                     q->link.undertime == PSCHED_PASTPERFECT)
1040                         break;
1041
1042                 q->toplevel = TC_CBQ_MAXLEVEL;
1043                 q->link.undertime = PSCHED_PASTPERFECT;
1044         }
1045
1046         /* No packets in scheduler or nobody wants to give them to us :-(
1047            Sigh... start watchdog timer in the last case. */
1048
1049         if (sch->q.qlen) {
1050                 sch->qstats.overlimits++;
1051                 if (q->wd_expires)
1052                         qdisc_watchdog_schedule(&q->watchdog,
1053                                                 now + q->wd_expires);
1054         }
1055         return NULL;
1056 }
1057
1058 /* CBQ class maintanance routines */
1059
1060 static void cbq_adjust_levels(struct cbq_class *this)
1061 {
1062         if (this == NULL)
1063                 return;
1064
1065         do {
1066                 int level = 0;
1067                 struct cbq_class *cl;
1068
1069                 if ((cl = this->children) != NULL) {
1070                         do {
1071                                 if (cl->level > level)
1072                                         level = cl->level;
1073                         } while ((cl = cl->sibling) != this->children);
1074                 }
1075                 this->level = level+1;
1076         } while ((this = this->tparent) != NULL);
1077 }
1078
1079 static void cbq_normalize_quanta(struct cbq_sched_data *q, int prio)
1080 {
1081         struct cbq_class *cl;
1082         unsigned h;
1083
1084         if (q->quanta[prio] == 0)
1085                 return;
1086
1087         for (h=0; h<16; h++) {
1088                 for (cl = q->classes[h]; cl; cl = cl->next) {
1089                         /* BUGGGG... Beware! This expression suffer of
1090                            arithmetic overflows!
1091                          */
1092                         if (cl->priority == prio) {
1093                                 cl->quantum = (cl->weight*cl->allot*q->nclasses[prio])/
1094                                         q->quanta[prio];
1095                         }
1096                         if (cl->quantum <= 0 || cl->quantum>32*cl->qdisc->dev->mtu) {
1097                                 printk(KERN_WARNING "CBQ: class %08x has bad quantum==%ld, repaired.\n", cl->classid, cl->quantum);
1098                                 cl->quantum = cl->qdisc->dev->mtu/2 + 1;
1099                         }
1100                 }
1101         }
1102 }
1103
1104 static void cbq_sync_defmap(struct cbq_class *cl)
1105 {
1106         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
1107         struct cbq_class *split = cl->split;
1108         unsigned h;
1109         int i;
1110
1111         if (split == NULL)
1112                 return;
1113
1114         for (i=0; i<=TC_PRIO_MAX; i++) {
1115                 if (split->defaults[i] == cl && !(cl->defmap&(1<<i)))
1116                         split->defaults[i] = NULL;
1117         }
1118
1119         for (i=0; i<=TC_PRIO_MAX; i++) {
1120                 int level = split->level;
1121
1122                 if (split->defaults[i])
1123                         continue;
1124
1125                 for (h=0; h<16; h++) {
1126                         struct cbq_class *c;
1127
1128                         for (c = q->classes[h]; c; c = c->next) {
1129                                 if (c->split == split && c->level < level &&
1130                                     c->defmap&(1<<i)) {
1131                                         split->defaults[i] = c;
1132                                         level = c->level;
1133                                 }
1134                         }
1135                 }
1136         }
1137 }
1138
1139 static void cbq_change_defmap(struct cbq_class *cl, u32 splitid, u32 def, u32 mask)
1140 {
1141         struct cbq_class *split = NULL;
1142
1143         if (splitid == 0) {
1144                 if ((split = cl->split) == NULL)
1145                         return;
1146                 splitid = split->classid;
1147         }
1148
1149         if (split == NULL || split->classid != splitid) {
1150                 for (split = cl->tparent; split; split = split->tparent)
1151                         if (split->classid == splitid)
1152                                 break;
1153         }
1154
1155         if (split == NULL)
1156                 return;
1157
1158         if (cl->split != split) {
1159                 cl->defmap = 0;
1160                 cbq_sync_defmap(cl);
1161                 cl->split = split;
1162                 cl->defmap = def&mask;
1163         } else
1164                 cl->defmap = (cl->defmap&~mask)|(def&mask);
1165
1166         cbq_sync_defmap(cl);
1167 }
1168
1169 static void cbq_unlink_class(struct cbq_class *this)
1170 {
1171         struct cbq_class *cl, **clp;
1172         struct cbq_sched_data *q = qdisc_priv(this->qdisc);
1173
1174         for (clp = &q->classes[cbq_hash(this->classid)]; (cl = *clp) != NULL; clp = &cl->next) {
1175                 if (cl == this) {
1176                         *clp = cl->next;
1177                         cl->next = NULL;
1178                         break;
1179                 }
1180         }
1181
1182         if (this->tparent) {
1183                 clp=&this->sibling;
1184                 cl = *clp;
1185                 do {
1186                         if (cl == this) {
1187                                 *clp = cl->sibling;
1188                                 break;
1189                         }
1190                         clp = &cl->sibling;
1191                 } while ((cl = *clp) != this->sibling);
1192
1193                 if (this->tparent->children == this) {
1194                         this->tparent->children = this->sibling;
1195                         if (this->sibling == this)
1196                                 this->tparent->children = NULL;
1197                 }
1198         } else {
1199                 BUG_TRAP(this->sibling == this);
1200         }
1201 }
1202
1203 static void cbq_link_class(struct cbq_class *this)
1204 {
1205         struct cbq_sched_data *q = qdisc_priv(this->qdisc);
1206         unsigned h = cbq_hash(this->classid);
1207         struct cbq_class *parent = this->tparent;
1208
1209         this->sibling = this;
1210         this->next = q->classes[h];
1211         q->classes[h] = this;
1212
1213         if (parent == NULL)
1214                 return;
1215
1216         if (parent->children == NULL) {
1217                 parent->children = this;
1218         } else {
1219                 this->sibling = parent->children->sibling;
1220                 parent->children->sibling = this;
1221         }
1222 }
1223
1224 static unsigned int cbq_drop(struct Qdisc* sch)
1225 {
1226         struct cbq_sched_data *q = qdisc_priv(sch);
1227         struct cbq_class *cl, *cl_head;
1228         int prio;
1229         unsigned int len;
1230
1231         for (prio = TC_CBQ_MAXPRIO; prio >= 0; prio--) {
1232                 if ((cl_head = q->active[prio]) == NULL)
1233                         continue;
1234
1235                 cl = cl_head;
1236                 do {
1237                         if (cl->q->ops->drop && (len = cl->q->ops->drop(cl->q))) {
1238                                 sch->q.qlen--;
1239                                 if (!cl->q->q.qlen)
1240                                         cbq_deactivate_class(cl);
1241                                 return len;
1242                         }
1243                 } while ((cl = cl->next_alive) != cl_head);
1244         }
1245         return 0;
1246 }
1247
1248 static void
1249 cbq_reset(struct Qdisc* sch)
1250 {
1251         struct cbq_sched_data *q = qdisc_priv(sch);
1252         struct cbq_class *cl;
1253         int prio;
1254         unsigned h;
1255
1256         q->activemask = 0;
1257         q->pmask = 0;
1258         q->tx_class = NULL;
1259         q->tx_borrowed = NULL;
1260         qdisc_watchdog_cancel(&q->watchdog);
1261         hrtimer_cancel(&q->delay_timer);
1262         q->toplevel = TC_CBQ_MAXLEVEL;
1263         q->now = psched_get_time();
1264         q->now_rt = q->now;
1265
1266         for (prio = 0; prio <= TC_CBQ_MAXPRIO; prio++)
1267                 q->active[prio] = NULL;
1268
1269         for (h = 0; h < 16; h++) {
1270                 for (cl = q->classes[h]; cl; cl = cl->next) {
1271                         qdisc_reset(cl->q);
1272
1273                         cl->next_alive = NULL;
1274                         cl->undertime = PSCHED_PASTPERFECT;
1275                         cl->avgidle = cl->maxidle;
1276                         cl->deficit = cl->quantum;
1277                         cl->cpriority = cl->priority;
1278                 }
1279         }
1280         sch->q.qlen = 0;
1281 }
1282
1283
1284 static int cbq_set_lss(struct cbq_class *cl, struct tc_cbq_lssopt *lss)
1285 {
1286         if (lss->change&TCF_CBQ_LSS_FLAGS) {
1287                 cl->share = (lss->flags&TCF_CBQ_LSS_ISOLATED) ? NULL : cl->tparent;
1288                 cl->borrow = (lss->flags&TCF_CBQ_LSS_BOUNDED) ? NULL : cl->tparent;
1289         }
1290         if (lss->change&TCF_CBQ_LSS_EWMA)
1291                 cl->ewma_log = lss->ewma_log;
1292         if (lss->change&TCF_CBQ_LSS_AVPKT)
1293                 cl->avpkt = lss->avpkt;
1294         if (lss->change&TCF_CBQ_LSS_MINIDLE)
1295                 cl->minidle = -(long)lss->minidle;
1296         if (lss->change&TCF_CBQ_LSS_MAXIDLE) {
1297                 cl->maxidle = lss->maxidle;
1298                 cl->avgidle = lss->maxidle;
1299         }
1300         if (lss->change&TCF_CBQ_LSS_OFFTIME)
1301                 cl->offtime = lss->offtime;
1302         return 0;
1303 }
1304
1305 static void cbq_rmprio(struct cbq_sched_data *q, struct cbq_class *cl)
1306 {
1307         q->nclasses[cl->priority]--;
1308         q->quanta[cl->priority] -= cl->weight;
1309         cbq_normalize_quanta(q, cl->priority);
1310 }
1311
1312 static void cbq_addprio(struct cbq_sched_data *q, struct cbq_class *cl)
1313 {
1314         q->nclasses[cl->priority]++;
1315         q->quanta[cl->priority] += cl->weight;
1316         cbq_normalize_quanta(q, cl->priority);
1317 }
1318
1319 static int cbq_set_wrr(struct cbq_class *cl, struct tc_cbq_wrropt *wrr)
1320 {
1321         struct cbq_sched_data *q = qdisc_priv(cl->qdisc);
1322
1323         if (wrr->allot)
1324                 cl->allot = wrr->allot;
1325         if (wrr->weight)
1326                 cl->weight = wrr->weight;
1327         if (wrr->priority) {
1328                 cl->priority = wrr->priority-1;
1329                 cl->cpriority = cl->priority;
1330                 if (cl->priority >= cl->priority2)
1331                         cl->priority2 = TC_CBQ_MAXPRIO-1;
1332         }
1333
1334         cbq_addprio(q, cl);
1335         return 0;
1336 }
1337
1338 static int cbq_set_overlimit(struct cbq_class *cl, struct tc_cbq_ovl *ovl)
1339 {
1340         switch (ovl->strategy) {
1341         case TC_CBQ_OVL_CLASSIC:
1342                 cl->overlimit = cbq_ovl_classic;
1343                 break;
1344         case TC_CBQ_OVL_DELAY:
1345                 cl->overlimit = cbq_ovl_delay;
1346                 break;
1347         case TC_CBQ_OVL_LOWPRIO:
1348                 if (ovl->priority2-1 >= TC_CBQ_MAXPRIO ||
1349                     ovl->priority2-1 <= cl->priority)
1350                         return -EINVAL;
1351                 cl->priority2 = ovl->priority2-1;
1352                 cl->overlimit = cbq_ovl_lowprio;
1353                 break;
1354         case TC_CBQ_OVL_DROP:
1355                 cl->overlimit = cbq_ovl_drop;
1356                 break;
1357         case TC_CBQ_OVL_RCLASSIC:
1358                 cl->overlimit = cbq_ovl_rclassic;
1359                 break;
1360         default:
1361                 return -EINVAL;
1362         }
1363         cl->penalty = ovl->penalty;
1364         return 0;
1365 }
1366
1367 #ifdef CONFIG_NET_CLS_POLICE
1368 static int cbq_set_police(struct cbq_class *cl, struct tc_cbq_police *p)
1369 {
1370         cl->police = p->police;
1371
1372         if (cl->q->handle) {
1373                 if (p->police == TC_POLICE_RECLASSIFY)
1374                         cl->q->reshape_fail = cbq_reshape_fail;
1375                 else
1376                         cl->q->reshape_fail = NULL;
1377         }
1378         return 0;
1379 }
1380 #endif
1381
1382 static int cbq_set_fopt(struct cbq_class *cl, struct tc_cbq_fopt *fopt)
1383 {
1384         cbq_change_defmap(cl, fopt->split, fopt->defmap, fopt->defchange);
1385         return 0;
1386 }
1387
1388 static int cbq_init(struct Qdisc *sch, struct rtattr *opt)
1389 {
1390         struct cbq_sched_data *q = qdisc_priv(sch);
1391         struct rtattr *tb[TCA_CBQ_MAX];
1392         struct tc_ratespec *r;
1393
1394         if (rtattr_parse_nested(tb, TCA_CBQ_MAX, opt) < 0 ||
1395             tb[TCA_CBQ_RTAB-1] == NULL || tb[TCA_CBQ_RATE-1] == NULL ||
1396             RTA_PAYLOAD(tb[TCA_CBQ_RATE-1]) < sizeof(struct tc_ratespec))
1397                 return -EINVAL;
1398
1399         if (tb[TCA_CBQ_LSSOPT-1] &&
1400             RTA_PAYLOAD(tb[TCA_CBQ_LSSOPT-1]) < sizeof(struct tc_cbq_lssopt))
1401                 return -EINVAL;
1402
1403         r = RTA_DATA(tb[TCA_CBQ_RATE-1]);
1404
1405         if ((q->link.R_tab = qdisc_get_rtab(r, tb[TCA_CBQ_RTAB-1])) == NULL)
1406                 return -EINVAL;
1407
1408         q->link.refcnt = 1;
1409         q->link.sibling = &q->link;
1410         q->link.classid = sch->handle;
1411         q->link.qdisc = sch;
1412         if (!(q->link.q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops,
1413                                             sch->handle)))
1414                 q->link.q = &noop_qdisc;
1415
1416         q->link.priority = TC_CBQ_MAXPRIO-1;
1417         q->link.priority2 = TC_CBQ_MAXPRIO-1;
1418         q->link.cpriority = TC_CBQ_MAXPRIO-1;
1419         q->link.ovl_strategy = TC_CBQ_OVL_CLASSIC;
1420         q->link.overlimit = cbq_ovl_classic;
1421         q->link.allot = psched_mtu(sch->dev);
1422         q->link.quantum = q->link.allot;
1423         q->link.weight = q->link.R_tab->rate.rate;
1424
1425         q->link.ewma_log = TC_CBQ_DEF_EWMA;
1426         q->link.avpkt = q->link.allot/2;
1427         q->link.minidle = -0x7FFFFFFF;
1428
1429         qdisc_watchdog_init(&q->watchdog, sch);
1430         hrtimer_init(&q->delay_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
1431         q->delay_timer.function = cbq_undelay;
1432         q->toplevel = TC_CBQ_MAXLEVEL;
1433         q->now = psched_get_time();
1434         q->now_rt = q->now;
1435
1436         cbq_link_class(&q->link);
1437
1438         if (tb[TCA_CBQ_LSSOPT-1])
1439                 cbq_set_lss(&q->link, RTA_DATA(tb[TCA_CBQ_LSSOPT-1]));
1440
1441         cbq_addprio(q, &q->link);
1442         return 0;
1443 }
1444
1445 static __inline__ int cbq_dump_rate(struct sk_buff *skb, struct cbq_class *cl)
1446 {
1447         unsigned char *b = skb_tail_pointer(skb);
1448
1449         RTA_PUT(skb, TCA_CBQ_RATE, sizeof(cl->R_tab->rate), &cl->R_tab->rate);
1450         return skb->len;
1451
1452 rtattr_failure:
1453         nlmsg_trim(skb, b);
1454         return -1;
1455 }
1456
1457 static __inline__ int cbq_dump_lss(struct sk_buff *skb, struct cbq_class *cl)
1458 {
1459         unsigned char *b = skb_tail_pointer(skb);
1460         struct tc_cbq_lssopt opt;
1461
1462         opt.flags = 0;
1463         if (cl->borrow == NULL)
1464                 opt.flags |= TCF_CBQ_LSS_BOUNDED;
1465         if (cl->share == NULL)
1466                 opt.flags |= TCF_CBQ_LSS_ISOLATED;
1467         opt.ewma_log = cl->ewma_log;
1468         opt.level = cl->level;
1469         opt.avpkt = cl->avpkt;
1470         opt.maxidle = cl->maxidle;
1471         opt.minidle = (u32)(-cl->minidle);
1472         opt.offtime = cl->offtime;
1473         opt.change = ~0;
1474         RTA_PUT(skb, TCA_CBQ_LSSOPT, sizeof(opt), &opt);
1475         return skb->len;
1476
1477 rtattr_failure:
1478         nlmsg_trim(skb, b);
1479         return -1;
1480 }
1481
1482 static __inline__ int cbq_dump_wrr(struct sk_buff *skb, struct cbq_class *cl)
1483 {
1484         unsigned char *b = skb_tail_pointer(skb);
1485         struct tc_cbq_wrropt opt;
1486
1487         opt.flags = 0;
1488         opt.allot = cl->allot;
1489         opt.priority = cl->priority+1;
1490         opt.cpriority = cl->cpriority+1;
1491         opt.weight = cl->weight;
1492         RTA_PUT(skb, TCA_CBQ_WRROPT, sizeof(opt), &opt);
1493         return skb->len;
1494
1495 rtattr_failure:
1496         nlmsg_trim(skb, b);
1497         return -1;
1498 }
1499
1500 static __inline__ int cbq_dump_ovl(struct sk_buff *skb, struct cbq_class *cl)
1501 {
1502         unsigned char *b = skb_tail_pointer(skb);
1503         struct tc_cbq_ovl opt;
1504
1505         opt.strategy = cl->ovl_strategy;
1506         opt.priority2 = cl->priority2+1;
1507         opt.pad = 0;
1508         opt.penalty = cl->penalty;
1509         RTA_PUT(skb, TCA_CBQ_OVL_STRATEGY, sizeof(opt), &opt);
1510         return skb->len;
1511
1512 rtattr_failure:
1513         nlmsg_trim(skb, b);
1514         return -1;
1515 }
1516
1517 static __inline__ int cbq_dump_fopt(struct sk_buff *skb, struct cbq_class *cl)
1518 {
1519         unsigned char *b = skb_tail_pointer(skb);
1520         struct tc_cbq_fopt opt;
1521
1522         if (cl->split || cl->defmap) {
1523                 opt.split = cl->split ? cl->split->classid : 0;
1524                 opt.defmap = cl->defmap;
1525                 opt.defchange = ~0;
1526                 RTA_PUT(skb, TCA_CBQ_FOPT, sizeof(opt), &opt);
1527         }
1528         return skb->len;
1529
1530 rtattr_failure:
1531         nlmsg_trim(skb, b);
1532         return -1;
1533 }
1534
1535 #ifdef CONFIG_NET_CLS_POLICE
1536 static __inline__ int cbq_dump_police(struct sk_buff *skb, struct cbq_class *cl)
1537 {
1538         unsigned char *b = skb_tail_pointer(skb);
1539         struct tc_cbq_police opt;
1540
1541         if (cl->police) {
1542                 opt.police = cl->police;
1543                 opt.__res1 = 0;
1544                 opt.__res2 = 0;
1545                 RTA_PUT(skb, TCA_CBQ_POLICE, sizeof(opt), &opt);
1546         }
1547         return skb->len;
1548
1549 rtattr_failure:
1550         nlmsg_trim(skb, b);
1551         return -1;
1552 }
1553 #endif
1554
1555 static int cbq_dump_attr(struct sk_buff *skb, struct cbq_class *cl)
1556 {
1557         if (cbq_dump_lss(skb, cl) < 0 ||
1558             cbq_dump_rate(skb, cl) < 0 ||
1559             cbq_dump_wrr(skb, cl) < 0 ||
1560             cbq_dump_ovl(skb, cl) < 0 ||
1561 #ifdef CONFIG_NET_CLS_POLICE
1562             cbq_dump_police(skb, cl) < 0 ||
1563 #endif
1564             cbq_dump_fopt(skb, cl) < 0)
1565                 return -1;
1566         return 0;
1567 }
1568
1569 static int cbq_dump(struct Qdisc *sch, struct sk_buff *skb)
1570 {
1571         struct cbq_sched_data *q = qdisc_priv(sch);
1572         unsigned char *b = skb_tail_pointer(skb);
1573         struct rtattr *rta;
1574
1575         rta = (struct rtattr*)b;
1576         RTA_PUT(skb, TCA_OPTIONS, 0, NULL);
1577         if (cbq_dump_attr(skb, &q->link) < 0)
1578                 goto rtattr_failure;
1579         rta->rta_len = skb_tail_pointer(skb) - b;
1580         return skb->len;
1581
1582 rtattr_failure:
1583         nlmsg_trim(skb, b);
1584         return -1;
1585 }
1586
1587 static int
1588 cbq_dump_stats(struct Qdisc *sch, struct gnet_dump *d)
1589 {
1590         struct cbq_sched_data *q = qdisc_priv(sch);
1591
1592         q->link.xstats.avgidle = q->link.avgidle;
1593         return gnet_stats_copy_app(d, &q->link.xstats, sizeof(q->link.xstats));
1594 }
1595
1596 static int
1597 cbq_dump_class(struct Qdisc *sch, unsigned long arg,
1598                struct sk_buff *skb, struct tcmsg *tcm)
1599 {
1600         struct cbq_class *cl = (struct cbq_class*)arg;
1601         unsigned char *b = skb_tail_pointer(skb);
1602         struct rtattr *rta;
1603
1604         if (cl->tparent)
1605                 tcm->tcm_parent = cl->tparent->classid;
1606         else
1607                 tcm->tcm_parent = TC_H_ROOT;
1608         tcm->tcm_handle = cl->classid;
1609         tcm->tcm_info = cl->q->handle;
1610
1611         rta = (struct rtattr*)b;
1612         RTA_PUT(skb, TCA_OPTIONS, 0, NULL);
1613         if (cbq_dump_attr(skb, cl) < 0)
1614                 goto rtattr_failure;
1615         rta->rta_len = skb_tail_pointer(skb) - b;
1616         return skb->len;
1617
1618 rtattr_failure:
1619         nlmsg_trim(skb, b);
1620         return -1;
1621 }
1622
1623 static int
1624 cbq_dump_class_stats(struct Qdisc *sch, unsigned long arg,
1625         struct gnet_dump *d)
1626 {
1627         struct cbq_sched_data *q = qdisc_priv(sch);
1628         struct cbq_class *cl = (struct cbq_class*)arg;
1629
1630         cl->qstats.qlen = cl->q->q.qlen;
1631         cl->xstats.avgidle = cl->avgidle;
1632         cl->xstats.undertime = 0;
1633
1634         if (cl->undertime != PSCHED_PASTPERFECT)
1635                 cl->xstats.undertime = cl->undertime - q->now;
1636
1637         if (gnet_stats_copy_basic(d, &cl->bstats) < 0 ||
1638             gnet_stats_copy_rate_est(d, &cl->rate_est) < 0 ||
1639             gnet_stats_copy_queue(d, &cl->qstats) < 0)
1640                 return -1;
1641
1642         return gnet_stats_copy_app(d, &cl->xstats, sizeof(cl->xstats));
1643 }
1644
1645 static int cbq_graft(struct Qdisc *sch, unsigned long arg, struct Qdisc *new,
1646                      struct Qdisc **old)
1647 {
1648         struct cbq_class *cl = (struct cbq_class*)arg;
1649
1650         if (cl) {
1651                 if (new == NULL) {
1652                         if ((new = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops,
1653                                                      cl->classid)) == NULL)
1654                                 return -ENOBUFS;
1655                 } else {
1656 #ifdef CONFIG_NET_CLS_POLICE
1657                         if (cl->police == TC_POLICE_RECLASSIFY)
1658                                 new->reshape_fail = cbq_reshape_fail;
1659 #endif
1660                 }
1661                 sch_tree_lock(sch);
1662                 *old = xchg(&cl->q, new);
1663                 qdisc_tree_decrease_qlen(*old, (*old)->q.qlen);
1664                 qdisc_reset(*old);
1665                 sch_tree_unlock(sch);
1666
1667                 return 0;
1668         }
1669         return -ENOENT;
1670 }
1671
1672 static struct Qdisc *
1673 cbq_leaf(struct Qdisc *sch, unsigned long arg)
1674 {
1675         struct cbq_class *cl = (struct cbq_class*)arg;
1676
1677         return cl ? cl->q : NULL;
1678 }
1679
1680 static void cbq_qlen_notify(struct Qdisc *sch, unsigned long arg)
1681 {
1682         struct cbq_class *cl = (struct cbq_class *)arg;
1683
1684         if (cl->q->q.qlen == 0)
1685                 cbq_deactivate_class(cl);
1686 }
1687
1688 static unsigned long cbq_get(struct Qdisc *sch, u32 classid)
1689 {
1690         struct cbq_sched_data *q = qdisc_priv(sch);
1691         struct cbq_class *cl = cbq_class_lookup(q, classid);
1692
1693         if (cl) {
1694                 cl->refcnt++;
1695                 return (unsigned long)cl;
1696         }
1697         return 0;
1698 }
1699
1700 static void cbq_destroy_class(struct Qdisc *sch, struct cbq_class *cl)
1701 {
1702         struct cbq_sched_data *q = qdisc_priv(sch);
1703
1704         BUG_TRAP(!cl->filters);
1705
1706         tcf_destroy_chain(cl->filter_list);
1707         qdisc_destroy(cl->q);
1708         qdisc_put_rtab(cl->R_tab);
1709         gen_kill_estimator(&cl->bstats, &cl->rate_est);
1710         if (cl != &q->link)
1711                 kfree(cl);
1712 }
1713
1714 static void
1715 cbq_destroy(struct Qdisc* sch)
1716 {
1717         struct cbq_sched_data *q = qdisc_priv(sch);
1718         struct cbq_class *cl;
1719         unsigned h;
1720
1721 #ifdef CONFIG_NET_CLS_POLICE
1722         q->rx_class = NULL;
1723 #endif
1724         /*
1725          * Filters must be destroyed first because we don't destroy the
1726          * classes from root to leafs which means that filters can still
1727          * be bound to classes which have been destroyed already. --TGR '04
1728          */
1729         for (h = 0; h < 16; h++) {
1730                 for (cl = q->classes[h]; cl; cl = cl->next) {
1731                         tcf_destroy_chain(cl->filter_list);
1732                         cl->filter_list = NULL;
1733                 }
1734         }
1735         for (h = 0; h < 16; h++) {
1736                 struct cbq_class *next;
1737
1738                 for (cl = q->classes[h]; cl; cl = next) {
1739                         next = cl->next;
1740                         cbq_destroy_class(sch, cl);
1741                 }
1742         }
1743 }
1744
1745 static void cbq_put(struct Qdisc *sch, unsigned long arg)
1746 {
1747         struct cbq_class *cl = (struct cbq_class*)arg;
1748
1749         if (--cl->refcnt == 0) {
1750 #ifdef CONFIG_NET_CLS_POLICE
1751                 struct cbq_sched_data *q = qdisc_priv(sch);
1752
1753                 spin_lock_bh(&sch->dev->queue_lock);
1754                 if (q->rx_class == cl)
1755                         q->rx_class = NULL;
1756                 spin_unlock_bh(&sch->dev->queue_lock);
1757 #endif
1758
1759                 cbq_destroy_class(sch, cl);
1760         }
1761 }
1762
1763 static int
1764 cbq_change_class(struct Qdisc *sch, u32 classid, u32 parentid, struct rtattr **tca,
1765                  unsigned long *arg)
1766 {
1767         int err;
1768         struct cbq_sched_data *q = qdisc_priv(sch);
1769         struct cbq_class *cl = (struct cbq_class*)*arg;
1770         struct rtattr *opt = tca[TCA_OPTIONS-1];
1771         struct rtattr *tb[TCA_CBQ_MAX];
1772         struct cbq_class *parent;
1773         struct qdisc_rate_table *rtab = NULL;
1774
1775         if (opt==NULL || rtattr_parse_nested(tb, TCA_CBQ_MAX, opt))
1776                 return -EINVAL;
1777
1778         if (tb[TCA_CBQ_OVL_STRATEGY-1] &&
1779             RTA_PAYLOAD(tb[TCA_CBQ_OVL_STRATEGY-1]) < sizeof(struct tc_cbq_ovl))
1780                 return -EINVAL;
1781
1782         if (tb[TCA_CBQ_FOPT-1] &&
1783             RTA_PAYLOAD(tb[TCA_CBQ_FOPT-1]) < sizeof(struct tc_cbq_fopt))
1784                 return -EINVAL;
1785
1786         if (tb[TCA_CBQ_RATE-1] &&
1787             RTA_PAYLOAD(tb[TCA_CBQ_RATE-1]) < sizeof(struct tc_ratespec))
1788                         return -EINVAL;
1789
1790         if (tb[TCA_CBQ_LSSOPT-1] &&
1791             RTA_PAYLOAD(tb[TCA_CBQ_LSSOPT-1]) < sizeof(struct tc_cbq_lssopt))
1792                         return -EINVAL;
1793
1794         if (tb[TCA_CBQ_WRROPT-1] &&
1795             RTA_PAYLOAD(tb[TCA_CBQ_WRROPT-1]) < sizeof(struct tc_cbq_wrropt))
1796                         return -EINVAL;
1797
1798 #ifdef CONFIG_NET_CLS_POLICE
1799         if (tb[TCA_CBQ_POLICE-1] &&
1800             RTA_PAYLOAD(tb[TCA_CBQ_POLICE-1]) < sizeof(struct tc_cbq_police))
1801                         return -EINVAL;
1802 #endif
1803
1804         if (cl) {
1805                 /* Check parent */
1806                 if (parentid) {
1807                         if (cl->tparent && cl->tparent->classid != parentid)
1808                                 return -EINVAL;
1809                         if (!cl->tparent && parentid != TC_H_ROOT)
1810                                 return -EINVAL;
1811                 }
1812
1813                 if (tb[TCA_CBQ_RATE-1]) {
1814                         rtab = qdisc_get_rtab(RTA_DATA(tb[TCA_CBQ_RATE-1]), tb[TCA_CBQ_RTAB-1]);
1815                         if (rtab == NULL)
1816                                 return -EINVAL;
1817                 }
1818
1819                 /* Change class parameters */
1820                 sch_tree_lock(sch);
1821
1822                 if (cl->next_alive != NULL)
1823                         cbq_deactivate_class(cl);
1824
1825                 if (rtab) {
1826                         rtab = xchg(&cl->R_tab, rtab);
1827                         qdisc_put_rtab(rtab);
1828                 }
1829
1830                 if (tb[TCA_CBQ_LSSOPT-1])
1831                         cbq_set_lss(cl, RTA_DATA(tb[TCA_CBQ_LSSOPT-1]));
1832
1833                 if (tb[TCA_CBQ_WRROPT-1]) {
1834                         cbq_rmprio(q, cl);
1835                         cbq_set_wrr(cl, RTA_DATA(tb[TCA_CBQ_WRROPT-1]));
1836                 }
1837
1838                 if (tb[TCA_CBQ_OVL_STRATEGY-1])
1839                         cbq_set_overlimit(cl, RTA_DATA(tb[TCA_CBQ_OVL_STRATEGY-1]));
1840
1841 #ifdef CONFIG_NET_CLS_POLICE
1842                 if (tb[TCA_CBQ_POLICE-1])
1843                         cbq_set_police(cl, RTA_DATA(tb[TCA_CBQ_POLICE-1]));
1844 #endif
1845
1846                 if (tb[TCA_CBQ_FOPT-1])
1847                         cbq_set_fopt(cl, RTA_DATA(tb[TCA_CBQ_FOPT-1]));
1848
1849                 if (cl->q->q.qlen)
1850                         cbq_activate_class(cl);
1851
1852                 sch_tree_unlock(sch);
1853
1854                 if (tca[TCA_RATE-1])
1855                         gen_replace_estimator(&cl->bstats, &cl->rate_est,
1856                                               &sch->dev->queue_lock,
1857                                               tca[TCA_RATE-1]);
1858                 return 0;
1859         }
1860
1861         if (parentid == TC_H_ROOT)
1862                 return -EINVAL;
1863
1864         if (tb[TCA_CBQ_WRROPT-1] == NULL || tb[TCA_CBQ_RATE-1] == NULL ||
1865             tb[TCA_CBQ_LSSOPT-1] == NULL)
1866                 return -EINVAL;
1867
1868         rtab = qdisc_get_rtab(RTA_DATA(tb[TCA_CBQ_RATE-1]), tb[TCA_CBQ_RTAB-1]);
1869         if (rtab == NULL)
1870                 return -EINVAL;
1871
1872         if (classid) {
1873                 err = -EINVAL;
1874                 if (TC_H_MAJ(classid^sch->handle) || cbq_class_lookup(q, classid))
1875                         goto failure;
1876         } else {
1877                 int i;
1878                 classid = TC_H_MAKE(sch->handle,0x8000);
1879
1880                 for (i=0; i<0x8000; i++) {
1881                         if (++q->hgenerator >= 0x8000)
1882                                 q->hgenerator = 1;
1883                         if (cbq_class_lookup(q, classid|q->hgenerator) == NULL)
1884                                 break;
1885                 }
1886                 err = -ENOSR;
1887                 if (i >= 0x8000)
1888                         goto failure;
1889                 classid = classid|q->hgenerator;
1890         }
1891
1892         parent = &q->link;
1893         if (parentid) {
1894                 parent = cbq_class_lookup(q, parentid);
1895                 err = -EINVAL;
1896                 if (parent == NULL)
1897                         goto failure;
1898         }
1899
1900         err = -ENOBUFS;
1901         cl = kzalloc(sizeof(*cl), GFP_KERNEL);
1902         if (cl == NULL)
1903                 goto failure;
1904         cl->R_tab = rtab;
1905         rtab = NULL;
1906         cl->refcnt = 1;
1907         if (!(cl->q = qdisc_create_dflt(sch->dev, &pfifo_qdisc_ops, classid)))
1908                 cl->q = &noop_qdisc;
1909         cl->classid = classid;
1910         cl->tparent = parent;
1911         cl->qdisc = sch;
1912         cl->allot = parent->allot;
1913         cl->quantum = cl->allot;
1914         cl->weight = cl->R_tab->rate.rate;
1915
1916         sch_tree_lock(sch);
1917         cbq_link_class(cl);
1918         cl->borrow = cl->tparent;
1919         if (cl->tparent != &q->link)
1920                 cl->share = cl->tparent;
1921         cbq_adjust_levels(parent);
1922         cl->minidle = -0x7FFFFFFF;
1923         cbq_set_lss(cl, RTA_DATA(tb[TCA_CBQ_LSSOPT-1]));
1924         cbq_set_wrr(cl, RTA_DATA(tb[TCA_CBQ_WRROPT-1]));
1925         if (cl->ewma_log==0)
1926                 cl->ewma_log = q->link.ewma_log;
1927         if (cl->maxidle==0)
1928                 cl->maxidle = q->link.maxidle;
1929         if (cl->avpkt==0)
1930                 cl->avpkt = q->link.avpkt;
1931         cl->overlimit = cbq_ovl_classic;
1932         if (tb[TCA_CBQ_OVL_STRATEGY-1])
1933                 cbq_set_overlimit(cl, RTA_DATA(tb[TCA_CBQ_OVL_STRATEGY-1]));
1934 #ifdef CONFIG_NET_CLS_POLICE
1935         if (tb[TCA_CBQ_POLICE-1])
1936                 cbq_set_police(cl, RTA_DATA(tb[TCA_CBQ_POLICE-1]));
1937 #endif
1938         if (tb[TCA_CBQ_FOPT-1])
1939                 cbq_set_fopt(cl, RTA_DATA(tb[TCA_CBQ_FOPT-1]));
1940         sch_tree_unlock(sch);
1941
1942         if (tca[TCA_RATE-1])
1943                 gen_new_estimator(&cl->bstats, &cl->rate_est,
1944                                   &sch->dev->queue_lock, tca[TCA_RATE-1]);
1945
1946         *arg = (unsigned long)cl;
1947         return 0;
1948
1949 failure:
1950         qdisc_put_rtab(rtab);
1951         return err;
1952 }
1953
1954 static int cbq_delete(struct Qdisc *sch, unsigned long arg)
1955 {
1956         struct cbq_sched_data *q = qdisc_priv(sch);
1957         struct cbq_class *cl = (struct cbq_class*)arg;
1958         unsigned int qlen;
1959
1960         if (cl->filters || cl->children || cl == &q->link)
1961                 return -EBUSY;
1962
1963         sch_tree_lock(sch);
1964
1965         qlen = cl->q->q.qlen;
1966         qdisc_reset(cl->q);
1967         qdisc_tree_decrease_qlen(cl->q, qlen);
1968
1969         if (cl->next_alive)
1970                 cbq_deactivate_class(cl);
1971
1972         if (q->tx_borrowed == cl)
1973                 q->tx_borrowed = q->tx_class;
1974         if (q->tx_class == cl) {
1975                 q->tx_class = NULL;
1976                 q->tx_borrowed = NULL;
1977         }
1978 #ifdef CONFIG_NET_CLS_POLICE
1979         if (q->rx_class == cl)
1980                 q->rx_class = NULL;
1981 #endif
1982
1983         cbq_unlink_class(cl);
1984         cbq_adjust_levels(cl->tparent);
1985         cl->defmap = 0;
1986         cbq_sync_defmap(cl);
1987
1988         cbq_rmprio(q, cl);
1989         sch_tree_unlock(sch);
1990
1991         if (--cl->refcnt == 0)
1992                 cbq_destroy_class(sch, cl);
1993
1994         return 0;
1995 }
1996
1997 static struct tcf_proto **cbq_find_tcf(struct Qdisc *sch, unsigned long arg)
1998 {
1999         struct cbq_sched_data *q = qdisc_priv(sch);
2000         struct cbq_class *cl = (struct cbq_class *)arg;
2001
2002         if (cl == NULL)
2003                 cl = &q->link;
2004
2005         return &cl->filter_list;
2006 }
2007
2008 static unsigned long cbq_bind_filter(struct Qdisc *sch, unsigned long parent,
2009                                      u32 classid)
2010 {
2011         struct cbq_sched_data *q = qdisc_priv(sch);
2012         struct cbq_class *p = (struct cbq_class*)parent;
2013         struct cbq_class *cl = cbq_class_lookup(q, classid);
2014
2015         if (cl) {
2016                 if (p && p->level <= cl->level)
2017                         return 0;
2018                 cl->filters++;
2019                 return (unsigned long)cl;
2020         }
2021         return 0;
2022 }
2023
2024 static void cbq_unbind_filter(struct Qdisc *sch, unsigned long arg)
2025 {
2026         struct cbq_class *cl = (struct cbq_class*)arg;
2027
2028         cl->filters--;
2029 }
2030
2031 static void cbq_walk(struct Qdisc *sch, struct qdisc_walker *arg)
2032 {
2033         struct cbq_sched_data *q = qdisc_priv(sch);
2034         unsigned h;
2035
2036         if (arg->stop)
2037                 return;
2038
2039         for (h = 0; h < 16; h++) {
2040                 struct cbq_class *cl;
2041
2042                 for (cl = q->classes[h]; cl; cl = cl->next) {
2043                         if (arg->count < arg->skip) {
2044                                 arg->count++;
2045                                 continue;
2046                         }
2047                         if (arg->fn(sch, (unsigned long)cl, arg) < 0) {
2048                                 arg->stop = 1;
2049                                 return;
2050                         }
2051                         arg->count++;
2052                 }
2053         }
2054 }
2055
2056 static struct Qdisc_class_ops cbq_class_ops = {
2057         .graft          =       cbq_graft,
2058         .leaf           =       cbq_leaf,
2059         .qlen_notify    =       cbq_qlen_notify,
2060         .get            =       cbq_get,
2061         .put            =       cbq_put,
2062         .change         =       cbq_change_class,
2063         .delete         =       cbq_delete,
2064         .walk           =       cbq_walk,
2065         .tcf_chain      =       cbq_find_tcf,
2066         .bind_tcf       =       cbq_bind_filter,
2067         .unbind_tcf     =       cbq_unbind_filter,
2068         .dump           =       cbq_dump_class,
2069         .dump_stats     =       cbq_dump_class_stats,
2070 };
2071
2072 static struct Qdisc_ops cbq_qdisc_ops = {
2073         .next           =       NULL,
2074         .cl_ops         =       &cbq_class_ops,
2075         .id             =       "cbq",
2076         .priv_size      =       sizeof(struct cbq_sched_data),
2077         .enqueue        =       cbq_enqueue,
2078         .dequeue        =       cbq_dequeue,
2079         .requeue        =       cbq_requeue,
2080         .drop           =       cbq_drop,
2081         .init           =       cbq_init,
2082         .reset          =       cbq_reset,
2083         .destroy        =       cbq_destroy,
2084         .change         =       NULL,
2085         .dump           =       cbq_dump,
2086         .dump_stats     =       cbq_dump_stats,
2087         .owner          =       THIS_MODULE,
2088 };
2089
2090 static int __init cbq_module_init(void)
2091 {
2092         return register_qdisc(&cbq_qdisc_ops);
2093 }
2094 static void __exit cbq_module_exit(void)
2095 {
2096         unregister_qdisc(&cbq_qdisc_ops);
2097 }
2098 module_init(cbq_module_init)
2099 module_exit(cbq_module_exit)
2100 MODULE_LICENSE("GPL");