From xiphmont@gmail.com Thu Sep 28 00:28:21 2006 Message-ID: <806dafc20609280028i3ef3e5a1he7cdb2ee863ee189@mail.gmail.com> Date: Thu, 28 Sep 2006 03:28:18 -0400 From: "Christopher \"Monty\" Montgomery" To: linux-usb-devel@lists.sourceforge.net Subject: [PATCH 13/15] USB: ehci-hcd: add FSTN support Cc: greg@kroah.com, david-b@pacbell.net, xiphmont@gmail.com Content-Disposition: inline patch 13: Adds low-level scheduler mechanisms for linking, unlinking, manipulating and maintaining FSTNs. Turns on shadow budget logic to allow/use FSTNs in budgeting. Signed-off-by: Christopher "Monty" Montgomery Cc: David Brownell Signed-off-by: Greg Kroah-Hartman --- drivers/usb/host/ehci-hcd.c | 5 drivers/usb/host/ehci-mem.c | 104 ++++++++++--- drivers/usb/host/ehci-sched.c | 313 +++++++++++++++++++++++++++++++++++++++--- drivers/usb/host/ehci.h | 5 4 files changed, 385 insertions(+), 42 deletions(-) --- gregkh-2.6.orig/drivers/usb/host/ehci-hcd.c +++ gregkh-2.6/drivers/usb/host/ehci-hcd.c @@ -440,6 +440,11 @@ static int ehci_init(struct usb_hcd *hcd if ((retval = ehci_mem_init(ehci, GFP_KERNEL)) < 0) return retval; + /* a periodic schedule that supports FSTNs always has a single + static restore FSTN that matches all possible save + FSTNs. */ + periodic_init_link_restore_fstn(ehci); + /* controllers may cache some of the periodic schedule ... */ hcc_params = readl(&ehci->caps->hcc_params); if (HCC_ISOC_CACHE(hcc_params)) // full frame cache --- gregkh-2.6.orig/drivers/usb/host/ehci-mem.c +++ gregkh-2.6/drivers/usb/host/ehci-mem.c @@ -62,6 +62,26 @@ static inline void ehci_qtd_free (struct dma_pool_free (ehci->qtd_pool, qtd, qtd->qtd_dma); } +static struct ehci_fstn *ehci_fstn_alloc (struct ehci_hcd *ehci, gfp_t flags) +{ + struct ehci_fstn *fstn = NULL; + dma_addr_t dma; + + if(ehci->fstn_pool){ /* alloced only if HC actually supports fstn */ + fstn = dma_pool_alloc (ehci->fstn_pool, flags, &dma); + if (fstn != NULL) { + memset (fstn, 0, sizeof *fstn); + fstn->fstn_dma = dma; + } + } + return fstn; +} + +static inline void ehci_fstn_free (struct ehci_hcd *ehci, + struct ehci_fstn *fstn) +{ + dma_pool_free (ehci->fstn_pool, fstn, fstn->fstn_dma); +} static void qh_destroy (struct kref *kref) { @@ -144,6 +164,10 @@ static void ehci_mem_cleanup (struct ehc dma_pool_destroy (ehci->qtd_pool); ehci->qtd_pool = NULL; + if (ehci->fstn_pool) + dma_pool_destroy (ehci->fstn_pool); + ehci->fstn_pool = NULL; + if (ehci->qh_pool) { dma_pool_destroy (ehci->qh_pool); ehci->qh_pool = NULL; @@ -163,6 +187,8 @@ static void ehci_mem_cleanup (struct ehc ehci->periodic, ehci->periodic_dma); ehci->periodic = NULL; + if(ehci->periodic_save_fstns) + kfree(ehci->periodic_save_fstns); if(ehci->budget) kfree(ehci->budget); ehci->budget = NULL; @@ -171,7 +197,8 @@ static void ehci_mem_cleanup (struct ehc ehci->budget_pool = NULL; /* shadow periodic table */ - kfree(ehci->pshadow); + if(ehci->pshadow) + kfree(ehci->pshadow); ehci->pshadow = NULL; } @@ -181,45 +208,70 @@ static int ehci_mem_init (struct ehci_hc int i; /* QTDs for control/bulk/intr transfers */ - ehci->qtd_pool = dma_pool_create ("ehci_qtd", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_qtd), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->qtd_pool = + dma_pool_create ("ehci_qtd", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_qtd), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->qtd_pool) { goto fail; } + /* does this HC support FSTNs? Don't allocate FSTN structs if not. */ + if( HC_VERSION(readl (&ehci->caps->hc_capbase)) >= 96){ + /* EHCI 0.96+ */ + ehci->fstn_pool = + dma_pool_create ("ehci_fstn", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_fstn), + 32, + 4096); + /* do not give up on failure; + scheduling is merely constrained */ + if(ehci->fstn_pool) + ehci->periodic_save_fstns = + kcalloc(PERIODIC_QH_MAX_PERIOD, + sizeof(*ehci->periodic_save_fstns), + flags); + /* lazy-alloc the rest */ + + } + /* QHs for control/bulk/intr transfers */ - ehci->qh_pool = dma_pool_create ("ehci_qh", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_qh), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->qh_pool = + dma_pool_create ("ehci_qh", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_qh), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->qh_pool) { goto fail; } + ehci->async = ehci_qh_alloc (ehci, flags); if (!ehci->async) { goto fail; } /* ITD for high speed ISO transfers */ - ehci->itd_pool = dma_pool_create ("ehci_itd", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_itd), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->itd_pool = + dma_pool_create ("ehci_itd", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_itd), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->itd_pool) { goto fail; } /* SITD for full/low speed split ISO transfers */ - ehci->sitd_pool = dma_pool_create ("ehci_sitd", - ehci_to_hcd(ehci)->self.controller, - sizeof (struct ehci_sitd), - 32 /* byte alignment (for hw parts) */, - 4096 /* can't cross 4K */); + ehci->sitd_pool = + dma_pool_create ("ehci_sitd", + ehci_to_hcd(ehci)->self.controller, + sizeof (struct ehci_sitd), + 32 /* byte alignment (for hw parts) */, + 4096 /* can't cross 4K */); if (!ehci->sitd_pool) { goto fail; } @@ -227,8 +279,8 @@ static int ehci_mem_init (struct ehci_hc /* Hardware periodic table */ ehci->periodic = (__le32 *) dma_alloc_coherent (ehci_to_hcd(ehci)->self.controller, - ehci->periodic_size * sizeof(__le32), - &ehci->periodic_dma, 0); + ehci->periodic_size * sizeof(__le32), + &ehci->periodic_dma, 0); if (ehci->periodic == NULL) { goto fail; } @@ -241,9 +293,9 @@ static int ehci_mem_init (struct ehci_hc goto fail; } ehci->budget_pool = - kmem_cache_create ("ehci_budget", - sizeof(struct ehci_shadow_budget), - 0,0,NULL,NULL); + kmem_cache_create ("ehci_budget", + sizeof(struct ehci_shadow_budget), + 0,0,NULL,NULL); if (!ehci->budget_pool) { goto fail; } --- gregkh-2.6.orig/drivers/usb/host/ehci-sched.c +++ gregkh-2.6/drivers/usb/host/ehci-sched.c @@ -61,6 +61,10 @@ * For isochronous, an "iso_stream" head serves the same role as a QH. * It keeps track of every ITD (or SITD) that's scheduled. * + * This scheduler supports and aggressively uses both FSTNs in order + * to allow interrupt QH transfers to span H-frame boundaries. This + * support is necessary for efficient FS bus bandwidth usage through a + * 2.0 hub's TT mechanism. * * The transfer ordering in the shadow schedule looks like: * @@ -98,7 +102,10 @@ * * Transaction ordering is determined by the budgeting code and set in * the budget. Transactions are linked into the actual hardware - * schedule in the order specified by the budget. + * schedule in the order specified by the budget. FSTN positioning is + * static; it follows the recommendations of the EHCI 1.0 spec and + * places one save-state FSTN at the lowest level in each branch of + * the INTR tree and one restore FSTN at the head of level one. * */ @@ -110,6 +117,13 @@ MODULE_PARM_DESC (sched_verbose, "debugging information to syslog. Incurs large latencies in" " near-realtime code; default is 0 (off)"); +/* enable/disable use of FSTNs */ +static unsigned use_fstn = 1; +module_param (use_fstn, uint, S_IRUGO); +MODULE_PARM_DESC (use_fstn, + "use FSTNs: allow use of FSTN to schedule full speed INTR " + "transfers across frame boundaries; default is 1 (on)"); + /* set limit on periodic load per TT uframe */ static unsigned fs_bytes_per_uframe = 188; module_param (fs_bytes_per_uframe, uint, S_IRUGO); @@ -267,8 +281,9 @@ static void print_schedule_frame (char * if(fstn->hw_prev == EHCI_LIST_END) printk("[FSTN restore 0x%p]",fstn); else - printk("[FSTN save 0x%p]", - fstn); + printk("[FSTN save 0x%p <- 0x%p]", + fstn, + fstn->fstn_prev); } break; case Q_TYPE_QH: @@ -930,10 +945,14 @@ static int budget_calc_fs_frame(struct e /* USB 2.0 11.18.4.3.b */ cmask = (0x1c << start_frame); - /* don't allow spanning yet */ - if(cmask&(~0xff)) + /* if we need an FSTN are they even + available? */ + if(cmask&(~0xff)){ + if(use_fstn==0) return -1; - + if(ehci->fstn_pool==NULL) + return -1; + } }else{ /* sitd */ if(budget->owner.iso->bEndpointAddress & USB_DIR_IN){ @@ -1382,10 +1401,256 @@ static int disable_periodic (struct ehci } /*-------------------------------------------------------------------------*/ +/* FSTN machinery + + FSTN decision making and manipulation is self contained and has + nothing to do with the budget, only the schedule. The master restore + FSTN is set up along with the ehci initialization; save-place FSTNs + are manipulated and adjusted after any changes to the QH tree in the + hardware schedule. */ + +/* periodic_init_link_restore_fstn - Place one restore FSTN at the + * beginning of node level 1 of the hardware schedule interrupt tree + * [EHCI 1.0 4.12.2.2.2]. Called once by the ehci initialization code + * and not again. + * + * @ehci: pointer to ehci host controller device structure. + */ +static void periodic_init_link_restore_fstn(struct ehci_hcd *ehci) +{ + int i; + + if(use_fstn){ + ehci->periodic_restore_fstn = + ehci_fstn_alloc (ehci, GFP_ATOMIC); + + if(ehci->periodic_restore_fstn){ + ehci->periodic_restore_fstn->hw_next = EHCI_LIST_END; + ehci->periodic_restore_fstn->hw_prev = EHCI_LIST_END; + + for(i=0; iperiodic_size; i++){ + union ehci_shadow *here = &ehci->pshadow [i]; + __le32 *hw_p = &ehci->periodic [i]; + + here->fstn = ehci->periodic_restore_fstn; + *hw_p = FSTN_NEXT (here->fstn->fstn_dma); + wmb (); + + } + } + } +} + +/* periodic_position_save_fstn - determine the insertion point within + * the specified frame for a save-place FSTN. Any given frame + * contains only one save-state FSTN, which sits at the beginning of + * the lowest level (highest interval) of the hardware schedule + * interrupt tree. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @hw_p: returns pointer to insertion point in hardware schedule + */ +static union ehci_shadow * +periodic_position_save_fstn(struct ehci_hcd *ehci, + unsigned frame, + __le32 **hw_p) +{ + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + __le32 type; + + *hw_p = &ehci->periodic [frame & (ehci->periodic_size-1)]; + type = Q_NEXT_TYPE (**hw_p); + + while (here->ptr){ + + if(type == Q_TYPE_FSTN || type == Q_TYPE_QH) + break; + + *hw_p = here->hw_next; + here = periodic_next_shadow (here, type); + type = Q_NEXT_TYPE (**hw_p); + } + + return here; +} + +/* periodic_find_save_fstn - search a frame in the shadow/hardware + * schedule for a save FSTN, returning the insertion point if found or + * NULL if not found. Any given frame will contain at most a single + * save-state FSTN. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @hw_p: returns pointer to insertion point in hardware schedule + */ +static union ehci_shadow * +periodic_find_save_fstn(struct ehci_hcd *ehci, + unsigned frame, + __le32 **hw_p) +{ + union ehci_shadow *here; + int fframe = frame & (PERIODIC_QH_MAX_PERIOD-1); + struct ehci_fstn *fstn; + + if(ehci->periodic_save_fstns == NULL) + goto out; /* no FSTN support in HC */ + fstn = ehci->periodic_save_fstns[fframe]; + if(fstn == NULL) + goto out; /* this FSTN not yet alloced, couldn't be linked */ + + here = periodic_position_save_fstn(ehci, frame, hw_p); + if(here->ptr == fstn){ + return here; + } + +out: + return NULL; +} + +/* _periodic_unlink_fstn - unconditionally unlinks the save-state FSTN + * from the specified frame. noop if the frame does not have an + * active save-state FSTN. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + */ +static void _periodic_unlink_fstn (struct ehci_hcd *ehci, unsigned frame) +{ + __le32 *hw_p; + union ehci_shadow *here = periodic_find_save_fstn(ehci, frame, &hw_p); + + if (!here) + return; + + /* inactivate */ + here->fstn->hw_prev = EHCI_LIST_END; + here->fstn->fstn_prev = NULL; + + /* unlink */ + *hw_p = *here->hw_next; + *here = here->fstn->fstn_next; + +} + +/* _periodic_link_fstn - allocs [if needed] and links save-state + * fstn into frame in appropraite position, also patching FSTN + * backpointer. If this frame already contains an active FSTN, only + * patches backpointer. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @back_qh: pointer to qh which the FSTN's backpointer should reference + */ +static void _periodic_link_fstn (struct ehci_hcd *ehci, + int frame, + struct ehci_qh *back_qh) +{ + + if(ehci->periodic_save_fstns){ + int fframe = frame & (PERIODIC_QH_MAX_PERIOD-1); + struct ehci_fstn *fstn = ehci->periodic_save_fstns[fframe]; + __le32 *hw_p; + union ehci_shadow *here = + periodic_position_save_fstn(ehci, frame, &hw_p); + + /* is the FSTN already alloced? */ + if(!fstn){ + fstn = ehci_fstn_alloc (ehci, GFP_ATOMIC); + if(!fstn){ + ehci_err(ehci, + "Unable to allocate FSTN\n"); + return; + } + + ehci->periodic_save_fstns[fframe] = fstn; + } + + /* patch the FSTN */ + fstn->hw_prev = QH_NEXT(back_qh->qh_dma); + fstn->fstn_prev = back_qh; + + /* already linked? */ + if(here->ptr != fstn){ + /* splice right to save-place fstn */ + fstn->fstn_next = *here; + fstn->hw_next = *hw_p; + wmb (); + + /* splice it to left; it's now live */ + here->fstn = fstn; + *hw_p = FSTN_NEXT (fstn->fstn_dma); + } + wmb (); + } +} + +/* _periodic_inspect_fstn_frame - toplevel machinery for managing all + * changes to FSTN state in the schedule; called whenever changes are + * made to the QH structure of a frame, automagically updates FSTN + * state in the hardware/shadow schedule to correctly reflect QH frame + * spanning. Links/unlinks/patches FSTNs in the following frame as + * needed. + * + * @ehci: pointer to ehci host controller device structure + * @frame: frame number in shadow/hardware schedule + * @insert: if changes are to be affected to FSTN preceeding linking + * new QH into live schedule, *insert designates new QH + * insertion point + * @new_qh: points to new qh to be linked into schedule following FSTN updates; + * allows valid FSTN to preceed activating the QH; NULL if this is an + * unlink update. + */ +static void _periodic_inspect_fstn_frame (struct ehci_hcd *ehci, + int frame, + union ehci_shadow *insert, + union ehci_shadow *new_qh) +{ + __le32 type = + Q_NEXT_TYPE (ehci->periodic [frame & (ehci->periodic_size-1)]); + __le32 save_type = type; + union ehci_shadow *here = + &ehci->pshadow [frame & (ehci->periodic_size-1)]; + + if(here == insert){ + here = new_qh; + type = Q_TYPE_QH; + } + + /* search frame for first spanning QH */ + while (here->ptr){ + if(type == Q_TYPE_QH){ + if(BUDGET_WRAP_P(here->qh->budget)){ + _periodic_link_fstn (ehci, frame+1, here->qh); + return; + } + } + + /* advance */ + if(here == new_qh){ + here = insert; + type = save_type; + }else{ + save_type = Q_NEXT_TYPE (*here->hw_next); + here = periodic_next_shadow (here, type); + if(here == insert){ + here = new_qh; + type = Q_TYPE_QH; + }else + type = save_type; + } + } + + /* no QH found, make sure there's no FSTN in following frame */ + _periodic_unlink_fstn (ehci, frame+1); +} + +/*-------------------------------------------------------------------------*/ /* Periodic interrupt (QH) endpoint machinery */ /* periodic_qh_unlink_frame - unlink the passed in QH from the - * specified frame + * specified frame and update frame FSTN state * * @ehci: pointer to ehci host controller device structure * @frame: frame number in shadow/hardware schedule @@ -1406,9 +1671,10 @@ static void periodic_qh_unlink_frame (st /*... the old "next" pointers from ptr (and if a qh, the fstns) may still be in use, the caller updates them. */ - } + _periodic_inspect_fstn_frame(ehci,frame,NULL,NULL); + } /* periodic_qh_deschedule - toplevel entry point for removing a given @@ -1471,7 +1737,7 @@ static void periodic_qh_deschedule(struc } /* periodic_qh_link - link the passed in QH into all relevant frames of the - * hardware schedule; assumes QH is already budgeted. + * hardware schedule and update FSTN state; assumes QH is already budgeted. * * @ehci: pointer to ehci host controller device structure * @qh: QH to link @@ -1505,6 +1771,14 @@ static int periodic_qh_link (struct ehci continue; } + /* perform FSTN inspection/adjustment before qh is + * linked */ + { + union ehci_shadow temp; + temp.qh=qh; + _periodic_inspect_fstn_frame (ehci, i, here, &temp); + } + /* already linked in? */ if(here->ptr != qh){ @@ -2323,7 +2597,7 @@ itd_link_urb ( next_uframe += stream->interval; stream->depth += stream->interval; - next_uframe %= mod; + next_uframe &= (mod-1); packet++; /* link completed itds into the schedule */ @@ -2931,9 +3205,16 @@ static int scan_frame(struct ehci_hcd *e q = *q_p; - modified = qh_completions (ehci, temp.qh); - if (unlikely (list_empty (&temp.qh->qtd_list))) - periodic_qh_deschedule (ehci, temp.qh); + /* process completions for this frame only if + * we're certain all completions for a + * preceeding spanning frame have first been + * processed; that is true iff clock was past + * uframe 1 when we started */ + if (uframes > 2) { + modified = qh_completions(ehci, temp.qh); + if (unlikely(list_empty(&temp.qh->qtd_list))) + periodic_qh_deschedule(ehci, temp.qh); + } qh_put (temp.qh); break; @@ -3069,7 +3350,7 @@ scan_periodic (struct ehci_hcd *ehci) clock = readl (&ehci->regs->frame_index); else clock = now_uframe + mod - 1; - clock %= mod; + clock &= (mod-1); for (;;) { unsigned uframes; @@ -3106,7 +3387,7 @@ scan_periodic (struct ehci_hcd *ehci) break; ehci->next_uframe = now_uframe; - now = readl (&ehci->regs->frame_index) % mod; + now = readl (&ehci->regs->frame_index) & (mod-1); if (now_uframe == now) break; @@ -3114,7 +3395,7 @@ scan_periodic (struct ehci_hcd *ehci) clock = now; } else { now_uframe++; - now_uframe %= mod; + now_uframe &= (mod-1); if(now_uframe == 0){ --- gregkh-2.6.orig/drivers/usb/host/ehci.h +++ gregkh-2.6/drivers/usb/host/ehci.h @@ -75,11 +75,14 @@ struct ehci_hcd { /* one per controlle struct ehci_shadow_budget **budget; /* pointer to the shadow budget of bandwidth placeholders */ + struct ehci_fstn *periodic_restore_fstn; + struct ehci_fstn **periodic_save_fstns; /* [PERIODIC_QH_MAX_PERIOD] */ /* per root hub port */ unsigned long reset_done [EHCI_MAX_ROOT_PORTS]; /* per-HC memory pools (could be per-bus, but ...) */ struct dma_pool *qh_pool; /* qh per active urb */ + struct dma_pool *fstn_pool; /* a qh has [up to] two fstns */ struct dma_pool *qtd_pool; /* one or more per qh */ struct dma_pool *itd_pool; /* itd per iso urb */ struct dma_pool *sitd_pool; /* sitd per split iso urb */ @@ -365,6 +368,7 @@ struct ehci_qtd { /* next async queue entry, or pointer to interrupt/periodic QH */ #define QH_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_QH) +#define FSTN_NEXT(dma) (cpu_to_le32(((u32)dma)&~0x01f)|Q_TYPE_FSTN) /* for periodic/async schedules and qtd lists, mark end of list */ #define EHCI_LIST_END __constant_cpu_to_le32(1) /* "null pointer" to hw */ @@ -665,6 +669,7 @@ struct ehci_fstn { /* the rest is HCD-private */ dma_addr_t fstn_dma; union ehci_shadow fstn_next; /* ptr to periodic q entry */ + struct ehci_qh *fstn_prev; /* ptr to backlinked qh */ } __attribute__ ((aligned (32))); /*-------------------------------------------------------------------------*/