diff --git a/block/blk-cgroup.c b/block/blk-cgroup.c index aa860f3..1f8e8e1 100644 --- a/block/blk-cgroup.c +++ b/block/blk-cgroup.c @@ -1678,7 +1678,7 @@ static void blkiocg_attach_task(struct cgroup *cgrp, struct task_struct *tsk) task_lock(tsk); ioc = tsk->io_context; if (ioc) - ioc->cgroup_changed = 1; + ioc_cgroup_changed(ioc); task_unlock(tsk); } diff --git a/block/blk-ioc.c b/block/blk-ioc.c index 167c923..3971852 100644 --- a/block/blk-ioc.c +++ b/block/blk-ioc.c @@ -187,6 +187,51 @@ void copy_io_context(struct io_context **pdst, struct io_context **psrc) } EXPORT_SYMBOL(copy_io_context); +void ioc_set_changed(struct io_context *ioc, int which) +{ + struct cfq_io_context *cic; + struct hlist_node *n; + + hlist_for_each_entry(cic, n, &ioc->cic_list, cic_list) + set_bit(which, &cic->changed); +} + +/** + * ioc_ioprio_changed - notify ioprio change + * @ioc: io_context of interest + * @ioprio: new ioprio + * + * @ioc's ioprio has changed to @ioprio. Set %CIC_IOPRIO_CHANGED for all + * cic's. iosched is responsible for checking the bit and applying it on + * request issue path. + */ +void ioc_ioprio_changed(struct io_context *ioc, int ioprio) +{ + unsigned long flags; + + spin_lock_irqsave(&ioc->lock, flags); + ioc->ioprio = ioprio; + ioc_set_changed(ioc, CIC_IOPRIO_CHANGED); + spin_unlock_irqrestore(&ioc->lock, flags); +} + +/** + * ioc_cgroup_changed - notify cgroup change + * @ioc: io_context of interest + * + * @ioc's cgroup has changed. Set %CIC_CGROUP_CHANGED for all cic's. + * iosched is responsible for checking the bit and applying it on request + * issue path. + */ +void ioc_cgroup_changed(struct io_context *ioc) +{ + unsigned long flags; + + spin_lock_irqsave(&ioc->lock, flags); + ioc_set_changed(ioc, CIC_CGROUP_CHANGED); + spin_unlock_irqrestore(&ioc->lock, flags); +} + static int __init blk_ioc_init(void) { iocontext_cachep = kmem_cache_create("blkdev_ioc", diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c index 775e8d9..d5320fb 100644 --- a/block/cfq-iosched.c +++ b/block/cfq-iosched.c @@ -3065,7 +3065,7 @@ static void cfq_init_prio_data(struct cfq_queue *cfqq, struct io_context *ioc) cfq_clear_cfqq_prio_changed(cfqq); } -static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic) +static void changed_ioprio(struct cfq_io_context *cic) { struct cfq_data *cfqd = cic->key; struct cfq_queue *cfqq; @@ -3093,12 +3093,6 @@ static void changed_ioprio(struct io_context *ioc, struct cfq_io_context *cic) spin_unlock_irqrestore(cfqd->queue->queue_lock, flags); } -static void cfq_ioc_set_ioprio(struct io_context *ioc) -{ - call_for_each_cic(ioc, changed_ioprio); - ioc->ioprio_changed = 0; -} - static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq, pid_t pid, bool is_sync) { @@ -3120,7 +3114,7 @@ static void cfq_init_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq, } #ifdef CONFIG_CFQ_GROUP_IOSCHED -static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic) +static void changed_cgroup(struct cfq_io_context *cic) { struct cfq_queue *sync_cfqq = cic_to_cfqq(cic, 1); struct cfq_data *cfqd = cic->key; @@ -3146,12 +3140,6 @@ static void changed_cgroup(struct io_context *ioc, struct cfq_io_context *cic) spin_unlock_irqrestore(q->queue_lock, flags); } - -static void cfq_ioc_set_cgroup(struct io_context *ioc) -{ - call_for_each_cic(ioc, changed_cgroup); - ioc->cgroup_changed = 0; -} #endif /* CONFIG_CFQ_GROUP_IOSCHED */ static struct cfq_queue * @@ -3375,13 +3363,15 @@ retry: out: smp_read_barrier_depends(); - if (unlikely(ioc->ioprio_changed)) - cfq_ioc_set_ioprio(ioc); - + if (unlikely(cic->changed)) { + if (test_and_clear_bit(CIC_IOPRIO_CHANGED, &cic->changed)) + changed_ioprio(cic); #ifdef CONFIG_CFQ_GROUP_IOSCHED - if (unlikely(ioc->cgroup_changed)) - cfq_ioc_set_cgroup(ioc); + if (test_and_clear_bit(CIC_CGROUP_CHANGED, &cic->changed)) + changed_cgroup(cic); #endif + } + return cic; err_free: cfq_cic_free(cic); diff --git a/fs/ioprio.c b/fs/ioprio.c index 8d25a24..73c6656 100644 --- a/fs/ioprio.c +++ b/fs/ioprio.c @@ -64,8 +64,7 @@ int set_task_ioprio(struct task_struct *task, int ioprio) } while (1); if (!err) { - ioc->ioprio = ioprio; - ioc->ioprio_changed = 1; + ioc_ioprio_changed(ioc, ioprio); } task_unlock(task); diff --git a/include/linux/iocontext.h b/include/linux/iocontext.h index 16ca5a5..2d9bf2b 100644 --- a/include/linux/iocontext.h +++ b/include/linux/iocontext.h @@ -30,6 +30,11 @@ struct as_io_context { sector_t seek_mean; }; +enum { + CIC_IOPRIO_CHANGED, + CIC_CGROUP_CHANGED, +}; + struct cfq_queue; struct cfq_io_context { void *key; @@ -48,6 +53,8 @@ struct cfq_io_context { struct list_head queue_list; struct hlist_node cic_list; + unsigned long changed; + void (*dtor)(struct io_context *); /* destructor */ void (*exit)(struct io_context *); /* called on task exit */ @@ -66,11 +73,6 @@ struct io_context { spinlock_t lock; unsigned short ioprio; - unsigned short ioprio_changed; - -#ifdef CONFIG_BLK_CGROUP - unsigned short cgroup_changed; -#endif /* * For request batching @@ -110,6 +112,8 @@ struct io_context *get_io_context(gfp_t gfp_flags, int node); struct io_context *alloc_io_context(gfp_t gfp_flags, int node); struct io_context *current_io_context(gfp_t gfp_flags, int node); void copy_io_context(struct io_context **pdst, struct io_context **psrc); +void ioc_ioprio_changed(struct io_context *ioc, int ioprio); +void ioc_cgroup_changed(struct io_context *ioc); #else static inline void exit_io_context(struct task_struct *task) {