/*[A[A
 * Copyright (C) 2005-2009 Red Hat, Inc. All rights reserved.
 *
 * Module Author: Heinz Mauelshagen <heinzm@redhat.com>
 *
 * This file is released under the GPL.
 *
 *
 * Linux 2.6 Device Mapper RAID4 and RAID5 target.
 *
 * Supports:
 *	o RAID4 with dedicated and selectable parity device
 *	o RAID5 with rotating parity (left+right, symmetric+asymmetric)
 *	o recovery of out of sync device for initial
 *	  RAID set creation or after dead drive replacement
 *	o run time optimization of xor algorithm used to calculate parity
 *
 *
 * Thanks to MD for:
 *    o the raid address calculation algorithm
 *    o the base of the biovec <-> page list copier.
 *
 *
 * Uses region hash to keep track of how many writes are in flight to
 * regions in order to use dirty log to keep state of regions to recover:
 *
 *    o clean regions (those which are synchronized
 * 	and don't have write io in flight)
 *    o dirty regions (those with write io in flight)
 *
 *
 * On startup, any dirty regions are migrated to the
 * 'nosync' state and are subject to recovery by the daemon.
 *
 * See raid_ctr() for table definition.
 *
 * FIXME: recovery bandwidth
 */ 

static const char *version = "v0.2594b";

#include "dm.h"
#include "dm-memcache.h"
#include "dm-message.h"
#include "dm-raid45.h"

#include <linux/kernel.h>
#include <linux/vmalloc.h>
#include <linux/raid/xor.h>

#include <linux/bio.h>
#include <linux/dm-io.h>
#include <linux/dm-dirty-log.h>
#include "dm-region-hash.h"


/*
 * Configurable parameters
 */

/* Minimum/maximum and default # of selectable stripes. */
#define	STRIPES_MIN		8
#define	STRIPES_MAX		16384
#define	STRIPES_DEFAULT		80

/* Maximum and default chunk size in sectors if not set in constructor. */
#define	CHUNK_SIZE_MIN		8
#define	CHUNK_SIZE_MAX		16384
#define	CHUNK_SIZE_DEFAULT	64

/* Default io size in sectors if not set in constructor. */
#define	IO_SIZE_MIN		CHUNK_SIZE_MIN
#define	IO_SIZE_DEFAULT		IO_SIZE_MIN

/* Recover io size default in sectors. */
#define	RECOVER_IO_SIZE_MIN		64
#define	RECOVER_IO_SIZE_DEFAULT		256

/* Default, minimum and maximum percentage of recover io bandwidth. */
#define	BANDWIDTH_DEFAULT	10
#define	BANDWIDTH_MIN		1
#define	BANDWIDTH_MAX		100

/* # of parallel recovered regions */
#define RECOVERY_STRIPES_MIN	1
#define RECOVERY_STRIPES_MAX	64
#define RECOVERY_STRIPES_DEFAULT	RECOVERY_STRIPES_MIN
/*
 * END Configurable parameters
 */

#define	TARGET	"dm-raid45"
#define	DAEMON	"kraid45d"
#define	DM_MSG_PREFIX	TARGET

#define	SECTORS_PER_PAGE	(PAGE_SIZE >> SECTOR_SHIFT)

/* Amount/size for __xor(). */
#define	XOR_SIZE	PAGE_SIZE

/* Check value in range. */
#define	range_ok(i, min, max)	(i >= min && i <= max)

/* Check argument is power of 2. */
#define POWER_OF_2(a) (!(a & (a - 1)))

/* Structure access macros. */
/* Derive raid_set from stripe_cache pointer. */
#define	RS(x)	container_of(x, struct raid_set, sc)

/* Page reference. */
#define PAGE(stripe, p)  ((stripe)->obj[p].pl->page)

/* Stripe chunk reference. */
#define CHUNK(stripe, p) ((stripe)->chunk + p)

/* Bio list reference. */
#define	BL(stripe, p, rw)	(stripe->chunk[p].bl + rw)
#define	BL_CHUNK(chunk, rw)	(chunk->bl + rw)

/* Page list reference. */
#define	PL(stripe, p)		(stripe->obj[p].pl)
/* END: structure access macros. */

/* Factor out to dm-bio-list.h */
static inline void bio_list_push(struct bio_list *bl, struct bio *bio)
{
	bio->bi_next = bl->head;
	bl->head = bio;

	if (!bl->tail)
		bl->tail = bio;
}

/* Factor out to dm.h */
#define TI_ERR_RET(str, ret) \
	do { ti->error = str; return ret; } while (0);
#define TI_ERR(str)     TI_ERR_RET(str, -EINVAL)

/* Macro to define access IO flags access inline functions. */
#define	BITOPS(name, what, var, flag) \
static inline int TestClear ## name ## what(struct var *v) \
{ return test_and_clear_bit(flag, &v->io.flags); } \
static inline int TestSet ## name ## what(struct var *v) \
{ return test_and_set_bit(flag, &v->io.flags); } \
static inline void Clear ## name ## what(struct var *v) \
{ clear_bit(flag, &v->io.flags); } \
static inline void Set ## name ## what(struct var *v) \
{ set_bit(flag, &v->io.flags); } \
static inline int name ## what(struct var *v) \
{ return test_bit(flag, &v->io.flags); }

/*-----------------------------------------------------------------
 * Stripe cache
 *
 * Cache for all reads and writes to raid sets (operational or degraded)
 *
 * We need to run all data to and from a RAID set through this cache,
 * because parity chunks need to get calculated from data chunks
 * or, in the degraded/resynchronization case, missing chunks need
 * to be reconstructed using the other chunks of the stripe.
 *---------------------------------------------------------------*/
/* A chunk within a stripe (holds bios hanging off). */
/* IO status flags for chunks of a stripe. */
enum chunk_flags {
	CHUNK_DIRTY,		/* Pages of chunk dirty; need writing. */
	CHUNK_ERROR,		/* IO error on any chunk page. */
	CHUNK_IO,		/* Allow/prohibit IO on chunk pages. */
	CHUNK_LOCKED,		/* Chunk pages locked during IO. */
	CHUNK_MUST_IO,		/* Chunk must io. */
	CHUNK_UNLOCK,		/* Enforce chunk unlock. */
	CHUNK_UPTODATE,		/* Chunk pages are uptodate. */
};

#if READ != 0 || WRITE != 1
#error dm-raid45: READ/WRITE != 0/1 used as index!!!
#endif

enum bl_type {
	WRITE_QUEUED = WRITE + 1,
	WRITE_MERGED,
	NR_BL_TYPES,	/* Must be last one! */
};
struct stripe_chunk {
	atomic_t cnt;		/* Reference count. */
	struct stripe *stripe;	/* Backpointer to stripe for endio(). */
	/* Bio lists for reads, writes, and writes merged. */
	struct bio_list bl[NR_BL_TYPES];
	struct {
		unsigned long flags; /* IO status flags. */
	} io;
};

/* Define chunk bit operations. */
BITOPS(Chunk, Dirty,	 stripe_chunk, CHUNK_DIRTY)
BITOPS(Chunk, Error,	 stripe_chunk, CHUNK_ERROR)
BITOPS(Chunk, Io,	 stripe_chunk, CHUNK_IO)
BITOPS(Chunk, Locked,	 stripe_chunk, CHUNK_LOCKED)
BITOPS(Chunk, MustIo,	 stripe_chunk, CHUNK_MUST_IO)
BITOPS(Chunk, Unlock,	 stripe_chunk, CHUNK_UNLOCK)
BITOPS(Chunk, Uptodate,	 stripe_chunk, CHUNK_UPTODATE)

/*
 * Stripe linked list indexes. Keep order, because the stripe
 * and the stripe cache rely on the first 3!
 */
enum list_types {
	LIST_FLUSH,	/* Stripes to flush for io. */
	LIST_ENDIO,	/* Stripes to endio. */
	LIST_LRU,	/* Least recently used stripes. */
	SC_NR_LISTS,	/* # of lists in stripe cache. */
	LIST_HASH = SC_NR_LISTS,	/* Hashed stripes. */
	LIST_RECOVER = LIST_HASH, /* For recovery type stripes only. */
	STRIPE_NR_LISTS,/* To size array in struct stripe. */
};

/* Adressing region recovery. */
struct recover_addr {
	struct dm_region *reg;	/* Actual region to recover. */
	sector_t pos;	/* Position within region to recover. */
	sector_t end;	/* End of region to recover. */
};

/* A stripe: the io object to handle all reads and writes to a RAID set. */
struct stripe {
	atomic_t cnt;			/* Reference count. */
	struct stripe_cache *sc;	/* Backpointer to stripe cache. */

	/*
	 * 4 linked lists:
	 *   o io list to flush io
	 *   o endio list
	 *   o LRU list to put stripes w/o reference count on
	 *   o stripe cache hash
	 */
	struct list_head lists[STRIPE_NR_LISTS];

	sector_t key;	 /* Hash key. */
	region_t region; /* Region stripe is mapped to. */

	struct {
		unsigned long flags;	/* Stripe state flags (see below). */

		/*
		 * Pending ios in flight:
		 *
		 * used to control move of stripe to endio list
		 */
		atomic_t pending;

		/* Sectors to read and write for multi page stripe sets. */
		unsigned size;
	} io;

	/* Address region recovery. */
	struct recover_addr *recover;

	/* Lock on stripe (Future: for clustering). */
	void *lock;

	struct {
		unsigned short parity;	/* Parity chunk index. */
		short recover;		/* Recovery chunk index. */
	} idx;

	/*
	 * This stripe's memory cache object (dm-mem-cache);
	 * i.e. the io chunk pages.
	 */
	struct dm_mem_cache_object *obj;

	/* Array of stripe sets (dynamically allocated). */
	struct stripe_chunk chunk[0];
};

/* States stripes can be in (flags field). */
enum stripe_states {
	STRIPE_ERROR,		/* io error on stripe. */
	STRIPE_MERGED,		/* Writes got merged to be written. */
	STRIPE_RBW,		/* Read-before-write stripe. */
	STRIPE_RECONSTRUCT,	/* Reconstruct of a missing chunk required. */
	STRIPE_RECONSTRUCTED,	/* Reconstructed of a missing chunk. */
	STRIPE_RECOVER,		/* Stripe used for RAID set recovery. */
};

/* Define stripe bit operations. */
BITOPS(Stripe, Error,	      stripe, STRIPE_ERROR)
BITOPS(Stripe, Merged,        stripe, STRIPE_MERGED)
BITOPS(Stripe, RBW,	      stripe, STRIPE_RBW)
BITOPS(Stripe, Reconstruct,   stripe, STRIPE_RECONSTRUCT)
BITOPS(Stripe, Reconstructed, stripe, STRIPE_RECONSTRUCTED)
BITOPS(Stripe, Recover,	      stripe, STRIPE_RECOVER)

/* A stripe hash. */
struct stripe_hash {
	struct list_head *hash;
	unsigned buckets;
	unsigned mask;
	unsigned prime;
	unsigned shift;
};

enum sc_lock_types {
	LOCK_ENDIO,	/* Protect endio list. */
	LOCK_LRU,	/* Protect LRU list. */
	NR_LOCKS,       /* To size array in struct stripe_cache. */
};

/* A stripe cache. */
struct stripe_cache {
	/* Stripe hash. */
	struct stripe_hash hash;

	spinlock_t locks[NR_LOCKS];	/* Locks to protect lists. */

	/* Stripes with io to flush, stripes to endio and LRU lists. */
	struct list_head lists[SC_NR_LISTS];

	/* Slab cache to allocate stripes from. */
	struct {
		struct kmem_cache *cache;	/* Cache itself. */
		char name[32];	/* Unique name. */
	} kc;

	struct dm_io_client *dm_io_client; /* dm-io client resource context. */

	/* dm-mem-cache client resource context. */
	struct dm_mem_cache_client *mem_cache_client;

	int stripes_parm;	    /* # stripes parameter from constructor. */
	atomic_t stripes;	    /* actual # of stripes in cache. */
	atomic_t stripes_to_set;    /* # of stripes to resize cache to. */
	atomic_t stripes_last;	    /* last # of stripes in cache. */
	atomic_t active_stripes;    /* actual # of active stripes in cache. */

	/* REMOVEME: */
	atomic_t active_stripes_max; /* actual # of active stripes in cache. */
};

/* Flag specs for raid_dev */ ;
enum raid_dev_flags {
	DEV_FAILED,	/* Device failed. */
	DEV_IO_QUEUED,	/* Io got queued to device. */
};

/* The raid device in a set. */
struct raid_dev {
	struct dm_dev *dev;
	sector_t start;		/* Offset to map to. */
	struct {	/* Using struct to be able to BITOPS(). */
		unsigned long flags;	/* raid_dev_flags. */
	} io;
};

BITOPS(Dev, Failed,   raid_dev, DEV_FAILED)
BITOPS(Dev, IoQueued, raid_dev, DEV_IO_QUEUED)

/* Flags spec for raid_set. */
enum raid_set_flags {
	RS_CHECK_OVERWRITE,	/* Check for chunk overwrites. */
	RS_DEAD,		/* RAID set inoperational. */
	RS_DEGRADED,		/* Io errors on RAID device. */
	RS_DEVEL_STATS,		/* REMOVEME: display status information. */
	RS_RECOVER,		/* Do recovery. */
	RS_RECOVERY_BANDWIDTH,	/* Allow recovery bandwidth (delayed bios). */
	RS_SC_BUSY,		/* Stripe cache busy -> send an event. */
	RS_SUSPEND,		/* Suspend RAID set. */
};

/* REMOVEME: devel stats counters. */
enum stats_types {
	S_BIOS_READ,
	S_BIOS_ADDED_READ,
	S_BIOS_ENDIO_READ,
	S_BIOS_WRITE,
	S_BIOS_ADDED_WRITE,
	S_BIOS_ENDIO_WRITE,
	S_CAN_MERGE,
	S_CANT_MERGE,
	S_CONGESTED,
	S_DM_IO_READ,
	S_DM_IO_WRITE,
	S_BANDWIDTH,
	S_BARRIER,
	S_BIO_COPY_PL_NEXT,
	S_DEGRADED,
	S_DELAYED_BIOS,
	S_FLUSHS,
	S_HITS_1ST,
	S_IOS_POST,
	S_INSCACHE,
	S_MAX_LOOKUP,
	S_CHUNK_LOCKED,
	S_NO_BANDWIDTH,
	S_NOT_CONGESTED,
	S_NO_RW,
	S_NOSYNC,
	S_OVERWRITE,
	S_PROHIBITCHUNKIO,
	S_RECONSTRUCT_EI,
	S_RECONSTRUCT_DEV,
	S_RECONSTRUCT_SET,
	S_RECONSTRUCTED,
	S_REQUEUE,
	S_STRIPE_ERROR,
	S_SUM_DELAYED_BIOS,
	S_XORS,
	S_NR_STATS,	/* # of stats counters. Must be last! */
};

/* Status type -> string mappings. */
struct stats_map {
	const enum stats_types type;
	const char *str;
};

static struct stats_map stats_map[] = {
	{ S_BIOS_READ, "r=" },
	{ S_BIOS_ADDED_READ, "/" },
	{ S_BIOS_ENDIO_READ, "/" },
	{ S_BIOS_WRITE, " w=" },
	{ S_BIOS_ADDED_WRITE, "/" },
	{ S_BIOS_ENDIO_WRITE, "/" },
	{ S_DM_IO_READ, " rc=" },
	{ S_DM_IO_WRITE, " wc=" },
	{ S_BANDWIDTH, "\nbw=" },
	{ S_NO_BANDWIDTH, " no_bw=" },
	{ S_BARRIER, "\nbarrier=" },
	{ S_BIO_COPY_PL_NEXT, "\nbio_cp_next=" },
	{ S_CAN_MERGE, "\nmerge=" },
	{ S_CANT_MERGE, "/no_merge=" },
	{ S_CHUNK_LOCKED, "\nchunk_locked=" },
	{ S_CONGESTED, "\ncgst=" },
	{ S_NOT_CONGESTED, "/not_cgst=" },
	{ S_DEGRADED, "\ndegraded=" },
	{ S_DELAYED_BIOS, "\ndel_bios=" },
	{ S_SUM_DELAYED_BIOS, "/sum_del_bios=" },
	{ S_FLUSHS, "\nflushs=" },
	{ S_HITS_1ST, "\nhits_1st=" },
	{ S_IOS_POST, " ios_post=" },
	{ S_INSCACHE, " inscache=" },
	{ S_MAX_LOOKUP, " maxlookup=" },
	{ S_NO_RW, "\nno_rw=" },
	{ S_NOSYNC, " nosync=" },
	{ S_OVERWRITE, " ovr=" },
	{ S_PROHIBITCHUNKIO, " prhbt_io=" },
	{ S_RECONSTRUCT_EI, "\nrec_ei=" },
	{ S_RECONSTRUCT_DEV, " rec_dev=" },
	{ S_RECONSTRUCT_SET, " rec_set=" },
	{ S_RECONSTRUCTED, " rec=" },
	{ S_REQUEUE, " requeue=" },
	{ S_STRIPE_ERROR, " stripe_err=" },
	{ S_XORS, " xors=" },
};

/*
 * A RAID set.
 */
#define	dm_rh_client	dm_region_hash
enum count_type { IO_WORK = 0, IO_RECOVER, IO_NR_COUNT };
typedef void (*xor_function_t)(unsigned count, unsigned long **data);
struct raid_set {
	struct dm_target *ti;	/* Target pointer. */

	struct {
		unsigned long flags;	/* State flags. */
		struct mutex in_lock;	/* Protects central input list below. */
		struct bio_list in;	/* Pending ios (central input list). */
		struct bio_list work;	/* ios work set. */
		wait_queue_head_t suspendq;	/* suspend synchronization. */
		atomic_t in_process;	/* counter of queued bios (suspendq). */
		atomic_t in_process_max;/* counter of queued bios max. */

		/* io work. */
		struct workqueue_struct *wq;
		struct delayed_work dws_do_raid;	/* For main worker. */
		struct work_struct ws_do_table_event;	/* For event worker. */
	} io;

	/* Stripe locking abstraction. */
	struct dm_raid45_locking_type *locking;

	struct stripe_cache sc;	/* Stripe cache for this set. */

	/* Xor optimization. */
	struct {
		struct xor_func *f;
		unsigned chunks;
		unsigned speed;
	} xor;

	/* Recovery parameters. */
	struct recover {
		struct dm_dirty_log *dl;	/* Dirty log. */
		struct dm_rh_client *rh;	/* Region hash. */

		struct dm_io_client *dm_io_client; /* recovery dm-io client. */
		/* dm-mem-cache client resource context for recovery stripes. */
		struct dm_mem_cache_client *mem_cache_client;

		struct list_head stripes;	/* List of recovery stripes. */

		region_t nr_regions;
		region_t nr_regions_to_recover;
		region_t nr_regions_recovered;
		unsigned long start_jiffies;
		unsigned long end_jiffies;

		unsigned bandwidth;	 /* Recovery bandwidth [%]. */
		unsigned bandwidth_work; /* Recovery bandwidth [factor]. */
		unsigned bandwidth_parm; /*  " constructor parm. */
		unsigned io_size;        /* recovery io size <= region size. */
		unsigned io_size_parm;   /* recovery io size ctr parameter. */
		unsigned recovery;	 /* Recovery allowed/prohibited. */
		unsigned recovery_stripes; /* # of parallel recovery stripes. */

		/* recovery io throttling. */
		atomic_t io_count[IO_NR_COUNT];	/* counter recover/regular io.*/
		unsigned long last_jiffies;
	} recover;

	/* RAID set parameters. */
	struct {
		struct raid_type *raid_type;	/* RAID type (eg, RAID4). */
		unsigned raid_parms;	/* # variable raid parameters. */

		unsigned chunk_size;	/* Sectors per chunk. */
		unsigned chunk_size_parm;
		unsigned chunk_shift;	/* rsector chunk size shift. */

		unsigned io_size;	/* Sectors per io. */
		unsigned io_size_parm;
		unsigned io_mask;	/* Mask for bio_copy_page_list(). */
		unsigned io_inv_mask;	/* Mask for raid_address(). */

		sector_t sectors_per_dev;	/* Sectors per device. */

		atomic_t failed_devs;		/* Amount of devices failed. */

		/* Index of device to initialize. */
		int dev_to_init;
		int dev_to_init_parm;

		/* Raid devices dynamically allocated. */
		unsigned raid_devs;	/* # of RAID devices below. */
		unsigned data_devs;	/* # of RAID data devices. */

		int ei;		/* index of failed RAID device. */

		/* Index of dedicated parity device (i.e. RAID4). */
		int pi;
		int pi_parm;	/* constructor parm for status output. */
	} set;

	/* REMOVEME: devel stats counters. */
	atomic_t stats[S_NR_STATS];

	/* Dynamically allocated temporary pointers for xor(). */
	unsigned long **data;

	/* Dynamically allocated RAID devices. Alignment? */
	struct raid_dev dev[0];
};

/* Define RAID set bit operations. */
BITOPS(RS, Bandwidth, raid_set, RS_RECOVERY_BANDWIDTH)
BITOPS(RS, CheckOverwrite, raid_set, RS_CHECK_OVERWRITE)
BITOPS(RS, Dead, raid_set, RS_DEAD)
BITOPS(RS, Degraded, raid_set, RS_DEGRADED)
BITOPS(RS, DevelStats, raid_set, RS_DEVEL_STATS)
BITOPS(RS, Recover, raid_set, RS_RECOVER)
BITOPS(RS, ScBusy, raid_set, RS_SC_BUSY)
BITOPS(RS, Suspend, raid_set, RS_SUSPEND)
#undef BITOPS

/*-----------------------------------------------------------------
 * Raid-4/5 set structures.
 *---------------------------------------------------------------*/
/* RAID level definitions. */
enum raid_level {
	raid4,
	raid5,
};

/* Symmetric/Asymmetric, Left/Right parity rotating algorithms. */
enum raid_algorithm {
	none,
	left_asym,
	right_asym,
	left_sym,
	right_sym,
};

struct raid_type {
	const char *name;		/* RAID algorithm. */
	const char *descr;		/* Descriptor text for logging. */
	const unsigned parity_devs;	/* # of parity devices. */
	const unsigned minimal_devs;	/* minimal # of devices in set. */
	const enum raid_level level;		/* RAID level. */
	const enum raid_algorithm algorithm;	/* RAID algorithm. */
};

/* Supported raid types and properties. */
static struct raid_type raid_types[] = {
	{"raid4",    "RAID4 (dedicated parity disk)", 1, 3, raid4, none},
	{"raid5_la", "RAID5 (left asymmetric)",       1, 3, raid5, left_asym},
	{"raid5_ra", "RAID5 (right asymmetric)",      1, 3, raid5, right_asym},
	{"raid5_ls", "RAID5 (left symmetric)",        1, 3, raid5, left_sym},
	{"raid5_rs", "RAID5 (right symmetric)",       1, 3, raid5, right_sym},
};

/* Address as calculated by raid_address(). */
struct raid_address {
	sector_t key;		/* Hash key (address of stripe % chunk_size). */
	unsigned di, pi;	/* Data and parity disks index. */
};

/* REMOVEME: reset statistics counters. */
static void stats_reset(struct raid_set *rs)
{
	unsigned s = S_NR_STATS;

	while (s--)
		atomic_set(rs->stats + s, 0);
}

/*----------------------------------------------------------------
 * RAID set management routines.
 *--------------------------------------------------------------*/
/*
 * Begin small helper functions.
 */
/* No need to be called from region hash indirectly at dm_rh_dec(). */
static void wake_dummy(void *context) {}

/* Return # of io reference. */
static int io_ref(struct raid_set *rs)
{
	return atomic_read(&rs->io.in_process);
}

/* Get an io reference. */
static void io_get(struct raid_set *rs)
{
	int p = atomic_inc_return(&rs->io.in_process);

	if (p > atomic_read(&rs->io.in_process_max))
		atomic_set(&rs->io.in_process_max, p); /* REMOVEME: max. */
}

/* Put the io reference and conditionally wake io waiters. */
static void io_put(struct raid_set *rs)
{
	/* Intel: rebuild data corrupter? */
	if (atomic_dec_and_test(&rs->io.in_process))
		wake_up(&rs->io.suspendq);
	else
		BUG_ON(io_ref(rs) < 0);
}

/* Wait until all io has been processed. */
static void wait_ios(struct raid_set *rs)
{
	wait_event(rs->io.suspendq, !io_ref(rs));
}

/* Queue (optionally delayed) io work. */
static void wake_do_raid_delayed(struct raid_set *rs, unsigned long delay)
{
	queue_delayed_work(rs->io.wq, &rs->io.dws_do_raid, delay);
}

/* Queue io work immediately (called from region hash too). */
static void wake_do_raid(void *context)
{
	struct raid_set *rs = context;

	queue_work(rs->io.wq, &rs->io.dws_do_raid.work);
}

/* Calculate device sector offset. */
static sector_t _sector(struct raid_set *rs, struct bio *bio)
{
	sector_t sector = bio->bi_sector;

	sector_div(sector, rs->set.data_devs);
	return sector;
}

/* Return # of active stripes in stripe cache. */
static int sc_active(struct stripe_cache *sc)
{
	return atomic_read(&sc->active_stripes);
}

/* Stripe cache busy indicator. */
static int sc_busy(struct raid_set *rs)
{
	return sc_active(&rs->sc) >
	       atomic_read(&rs->sc.stripes) - (STRIPES_MIN / 2);
}

/* Set chunks states. */
enum chunk_dirty_type { CLEAN, DIRTY, ERROR };
static void chunk_set(struct stripe_chunk *chunk, enum chunk_dirty_type type)
{
	switch (type) {
	case CLEAN:
		ClearChunkDirty(chunk);
		break;
	case DIRTY:
		SetChunkDirty(chunk);
		break;
	case ERROR:
		SetChunkError(chunk);
		SetStripeError(chunk->stripe);
		return;
	default:
		BUG();
	}

	SetChunkUptodate(chunk);
	SetChunkIo(chunk);
	ClearChunkError(chunk);
}

/* Return region state for a sector. */
static int region_state(struct raid_set *rs, sector_t sector, 
			enum dm_rh_region_states state)
{
	struct dm_rh_client *rh = rs->recover.rh;
	region_t region = dm_rh_sector_to_region(rh, sector);

	return !!(dm_rh_get_state(rh, region, 1) & state);
}

/*
 * Return true in case a chunk should be read/written
 *
 * Conditions to read/write:
 *	o chunk not uptodate
 *	o chunk dirty
 *
 * Conditios to avoid io:
 *	o io already ongoing on chunk
 *	o io explitely prohibited
 */
static int chunk_io(struct stripe_chunk *chunk)
{
	/* 2nd run optimization (flag set below on first run). */
	if (TestClearChunkMustIo(chunk))
		return 1;

	/* Avoid io if prohibited or a locked chunk. */
	if (!ChunkIo(chunk) || ChunkLocked(chunk))
		return 0;

	if (!ChunkUptodate(chunk) || ChunkDirty(chunk)) {
		SetChunkMustIo(chunk); /* 2nd run optimization. */
		return 1;
	}

	return 0;
}

/* Call a function on each chunk needing io unless device failed. */
static unsigned for_each_io_dev(struct stripe *stripe,
			        void (*f_io)(struct stripe *stripe, unsigned p))
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned p, r = 0;

	for (p = 0; p < rs->set.raid_devs; p++) {
		if (chunk_io(CHUNK(stripe, p)) && !DevFailed(rs->dev + p)) {
			f_io(stripe, p);
			r++;
		}
	}

	return r;
}

/*
 * Index of device to calculate parity on.
 *
 * Either the parity device index *or* the selected
 * device to init after a spare replacement.
 */
static int dev_for_parity(struct stripe *stripe, int *sync)
{
	struct raid_set *rs = RS(stripe->sc);
	int r = region_state(rs, stripe->key, DM_RH_NOSYNC | DM_RH_RECOVERING);

	*sync = !r;

	/* Reconstruct a particular device ?. */
	if (r && rs->set.dev_to_init > -1)
		return rs->set.dev_to_init;
	else if (rs->set.raid_type->level == raid4)
		return rs->set.pi;
	else if (!StripeRecover(stripe))
		return stripe->idx.parity;
	else
		return -1;
}

/* RAID set congested function. */
static int rs_congested(void *congested_data, int bdi_bits)
{
	int r;
	unsigned p;
	struct raid_set *rs = congested_data;

	if (sc_busy(rs) || RSSuspend(rs))
		r = 1;
	else for (r = 0, p = rs->set.raid_devs; !r && p--; ) {
		/* If any of our component devices are overloaded. */
		struct request_queue *q = bdev_get_queue(rs->dev[p].dev->bdev);

		r |= bdi_congested(&q->backing_dev_info, bdi_bits);
	}

	/* REMOVEME: statistics. */
	atomic_inc(rs->stats + (r ? S_CONGESTED : S_NOT_CONGESTED));
	return r;
}

/* RAID device degrade check. */
static void rs_check_degrade_dev(struct raid_set *rs,
				       struct stripe *stripe, unsigned p)
{
	if (TestSetDevFailed(rs->dev + p))
		return;

	/* Through an event in case of member device errors. */
	if (atomic_inc_return(&rs->set.failed_devs) >
	    rs->set.raid_type->parity_devs &&
	    !TestSetRSDead(rs)) {
		/* Display RAID set dead message once. */
		unsigned p;
		char buf[BDEVNAME_SIZE];

		DMERR("FATAL: too many devices failed -> RAID set broken");
		for (p = 0; p < rs->set.raid_devs; p++) {
			if (DevFailed(rs->dev + p))
				DMERR("device /dev/%s failed",
				      bdevname(rs->dev[p].dev->bdev, buf));
		}
	}

	/* Only log the first member error. */
	if (!TestSetRSDegraded(rs)) {
		char buf[BDEVNAME_SIZE];

		/* Store index for recovery. */
		rs->set.ei = p;
		DMERR("CRITICAL: %sio error on device /dev/%s "
		      "in region=%llu; DEGRADING RAID set\n",
		      stripe ? "" : "FAKED ",
		      bdevname(rs->dev[p].dev->bdev, buf),
		      (unsigned long long) (stripe ? stripe->key : 0));
		DMERR("further device error messages suppressed");
	}

	schedule_work(&rs->io.ws_do_table_event);
}

/* RAID set degrade check. */
static void rs_check_degrade(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned p = rs->set.raid_devs;

	while (p--) {
		if (ChunkError(CHUNK(stripe, p)))
			rs_check_degrade_dev(rs, stripe, p);
	}
}

/* Lookup a RAID device by name or by major:minor number. */
static int raid_dev_lookup(struct raid_set *rs, struct raid_dev *dev_lookup)
{
	unsigned p;
	struct raid_dev *dev;

	/*
	 * Must be an incremental loop, because the device array
	 * can have empty slots still on calls from raid_ctr()
	 */
	for (dev = rs->dev, p = 0;
	     dev->dev && p < rs->set.raid_devs;
	     dev++, p++) {
		if (dev_lookup->dev->bdev->bd_dev == dev->dev->bdev->bd_dev)
			return p;
	}

	return -ENODEV;
}
/*
 * End small helper functions.
 */

/*
 * Stripe hash functions
 */
/* Initialize/destroy stripe hash. */
static int hash_init(struct stripe_hash *hash, unsigned stripes)
{
	unsigned buckets = 2, max_buckets = stripes >> 1;
	static unsigned hash_primes[] = {
		/* Table of primes for hash_fn/table size optimization. */
		1, 2, 3, 7, 13, 27, 53, 97, 193, 389, 769,
		1543, 3079, 6151, 12289, 24593, 49157, 98317,
	};

	/* Calculate number of buckets (2^^n <= stripes / 2). */
	while (buckets < max_buckets)
		buckets <<= 1;

	/* Allocate stripe hash buckets. */
	hash->hash = vmalloc(buckets * sizeof(*hash->hash));
	if (!hash->hash)
		return -ENOMEM;

	hash->buckets = buckets;
	hash->mask = buckets - 1;
	hash->shift = ffs(buckets);
	if (hash->shift > ARRAY_SIZE(hash_primes))
		hash->shift = ARRAY_SIZE(hash_primes) - 1;

	BUG_ON(hash->shift < 2);
	hash->prime = hash_primes[hash->shift];

	/* Initialize buckets. */
	while (buckets--)
		INIT_LIST_HEAD(hash->hash + buckets);
	return 0;
}

static void hash_exit(struct stripe_hash *hash)
{
	if (hash->hash) {
		vfree(hash->hash);
		hash->hash = NULL;
	}
}

static unsigned hash_fn(struct stripe_hash *hash, sector_t key)
{
	return (unsigned) (((key * hash->prime) >> hash->shift) & hash->mask);
}

static struct list_head *hash_bucket(struct stripe_hash *hash, sector_t key)
{
	return hash->hash + hash_fn(hash, key);
}

/* Insert an entry into a hash. */
static void stripe_insert(struct stripe_hash *hash, struct stripe *stripe)
{
	list_add(stripe->lists + LIST_HASH, hash_bucket(hash, stripe->key));
}

/* Lookup an entry in the stripe hash. */
static struct stripe *stripe_lookup(struct stripe_cache *sc, sector_t key)
{
	unsigned look = 0;
	struct stripe *stripe;
	struct list_head *bucket = hash_bucket(&sc->hash, key);

	list_for_each_entry(stripe, bucket, lists[LIST_HASH]) {
		look++;

		if (stripe->key == key) {
			/* REMOVEME: statisics. */
			if (look > atomic_read(RS(sc)->stats + S_MAX_LOOKUP))
				atomic_set(RS(sc)->stats + S_MAX_LOOKUP, look);
			return stripe;
		}
	}

	return NULL;
}

/* Resize the stripe cache hash on size changes. */
static int sc_hash_resize(struct stripe_cache *sc)
{
	/* Resize indicated ? */
	if (atomic_read(&sc->stripes) != atomic_read(&sc->stripes_last)) {
		int r;
		struct stripe_hash hash;

		r = hash_init(&hash, atomic_read(&sc->stripes));
		if (r)
			return r;

		if (sc->hash.hash) {
			unsigned b = sc->hash.buckets;
			struct list_head *pos, *tmp;

			/* Walk old buckets and insert into new. */
			while (b--) {
				list_for_each_safe(pos, tmp, sc->hash.hash + b)
				    stripe_insert(&hash,
						  list_entry(pos, struct stripe,
							     lists[LIST_HASH]));
			}

		}

		hash_exit(&sc->hash);
		memcpy(&sc->hash, &hash, sizeof(sc->hash));
		atomic_set(&sc->stripes_last, atomic_read(&sc->stripes));
	}

	return 0;
}
/* End hash stripe hash function. */

/* List add, delete, push and pop functions. */
/* Add stripe to flush list. */
#define	DEL_LIST(lh) \
	if (!list_empty(lh)) \
		list_del_init(lh);

/* Delete stripe from hash. */
static void stripe_hash_del(struct stripe *stripe)
{
	DEL_LIST(stripe->lists + LIST_HASH);
}

/* Return stripe reference count. */
static inline int stripe_ref(struct stripe *stripe)
{
	return atomic_read(&stripe->cnt);
}

static void stripe_flush_add(struct stripe *stripe)
{
	struct stripe_cache *sc = stripe->sc;
	struct list_head *lh = stripe->lists + LIST_FLUSH;

	if (!StripeReconstruct(stripe) && list_empty(lh))
		list_add_tail(lh, sc->lists + LIST_FLUSH);
}

/*
 * Add stripe to LRU (inactive) list.
 *
 * Need lock, because of concurrent access from message interface.
 */
static void stripe_lru_add(struct stripe *stripe)
{
	if (!StripeRecover(stripe)) {
		unsigned long flags;
		struct list_head *lh = stripe->lists + LIST_LRU;
		spinlock_t *lock = stripe->sc->locks + LOCK_LRU;

		spin_lock_irqsave(lock, flags);
		if (list_empty(lh))
			list_add_tail(lh, stripe->sc->lists + LIST_LRU);
		spin_unlock_irqrestore(lock, flags);
	}
}

#define POP_LIST(list) \
	do { \
		if (list_empty(sc->lists + (list))) \
			stripe = NULL; \
		else { \
			stripe = list_first_entry(sc->lists + (list), \
						  struct stripe, \
						  lists[(list)]); \
			list_del_init(stripe->lists + (list)); \
		} \
	} while (0);

/* Pop an available stripe off the LRU list. */
static struct stripe *stripe_lru_pop(struct stripe_cache *sc)
{
	struct stripe *stripe;
	spinlock_t *lock = sc->locks + LOCK_LRU;

	spin_lock_irq(lock);
	POP_LIST(LIST_LRU);
	spin_unlock_irq(lock);

	return stripe;
}

/* Pop an available stripe off the io list. */
static struct stripe *stripe_io_pop(struct stripe_cache *sc)
{
	struct stripe *stripe;

	POP_LIST(LIST_FLUSH);
	return stripe;
}

/* Push a stripe safely onto the endio list to be handled by do_endios(). */
static void stripe_endio_push(struct stripe *stripe)
{
	unsigned long flags;
	struct stripe_cache *sc = stripe->sc;
	struct list_head *stripe_list = stripe->lists + LIST_ENDIO,
			 *sc_list = sc->lists + LIST_ENDIO;
	spinlock_t *lock = sc->locks + LOCK_ENDIO;

	/* This runs in parallel with do_endios(). */
	spin_lock_irqsave(lock, flags);
	if (list_empty(stripe_list))
		list_add_tail(stripe_list, sc_list);
	spin_unlock_irqrestore(lock, flags);

	wake_do_raid(RS(sc)); /* Wake myself. */
}

/* Pop a stripe off safely off the endio list. */
static struct stripe *stripe_endio_pop(struct stripe_cache *sc)
{
	struct stripe *stripe;
	spinlock_t *lock = sc->locks + LOCK_ENDIO;

	/* This runs in parallel with endio(). */
	spin_lock_irq(lock);
	POP_LIST(LIST_ENDIO)
	spin_unlock_irq(lock);
	return stripe;
}
#undef POP_LIST

/*
 * Stripe cache locking functions
 */
/* Dummy lock function for single host RAID4+5. */
static void *no_lock(sector_t key, enum dm_lock_type type)
{
	return &no_lock;
}

/* Dummy unlock function for single host RAID4+5. */
static void no_unlock(void *lock_handle)
{
}

/* No locking (for single host RAID 4+5). */
static struct dm_raid45_locking_type locking_none = {
	.lock = no_lock,
	.unlock = no_unlock,
};

/* Lock a stripe (for clustering). */
static int
stripe_lock(struct stripe *stripe, int rw, sector_t key)
{
	stripe->lock = RS(stripe->sc)->locking->lock(key, rw == READ ? DM_RAID45_SHARED : DM_RAID45_EX);
	return stripe->lock ? 0 : -EPERM;
}

/* Unlock a stripe (for clustering). */
static void stripe_unlock(struct stripe *stripe)
{
	RS(stripe->sc)->locking->unlock(stripe->lock);
	stripe->lock = NULL;
}

/* Test io pending on stripe. */
static int stripe_io_ref(struct stripe *stripe)
{
	return atomic_read(&stripe->io.pending);
}

static void stripe_io_get(struct stripe *stripe)
{
	if (atomic_inc_return(&stripe->io.pending) == 1)
		/* REMOVEME: statistics */
		atomic_inc(&stripe->sc->active_stripes);
	else
		BUG_ON(stripe_io_ref(stripe) < 0);
}

static void stripe_io_put(struct stripe *stripe)
{
	if (atomic_dec_and_test(&stripe->io.pending)) {
		if (unlikely(StripeRecover(stripe)))
			/* Don't put recovery stripe on endio list. */
			wake_do_raid(RS(stripe->sc));
		else
			/* Add regular stripe to endio list and wake daemon. */
			stripe_endio_push(stripe);

		/* REMOVEME: statistics */
		atomic_dec(&stripe->sc->active_stripes);
	} else
		BUG_ON(stripe_io_ref(stripe) < 0);
}

/* Take stripe reference out. */
static int stripe_get(struct stripe *stripe)
{
	int r;
	struct list_head *lh = stripe->lists + LIST_LRU;
	spinlock_t *lock = stripe->sc->locks + LOCK_LRU;

	/* Delete stripe from LRU (inactive) list if on. */
	spin_lock_irq(lock);
	DEL_LIST(lh);
	spin_unlock_irq(lock);

	BUG_ON(stripe_ref(stripe) < 0);

	/* Lock stripe on first reference */
	r = (atomic_inc_return(&stripe->cnt) == 1) ?
	    stripe_lock(stripe, WRITE, stripe->key) : 0;

	return r;
}
#undef DEL_LIST

/* Return references on a chunk. */
static int chunk_ref(struct stripe_chunk *chunk)
{
	return atomic_read(&chunk->cnt);
}

/* Take out reference on a chunk. */
static int chunk_get(struct stripe_chunk *chunk)
{
	return atomic_inc_return(&chunk->cnt);
}

/* Drop reference on a chunk. */
static void chunk_put(struct stripe_chunk *chunk)
{
	BUG_ON(atomic_dec_return(&chunk->cnt) < 0);
}

/*
 * Drop reference on a stripe.
 *
 * Move it to list of LRU stripes if zero.
 */
static void stripe_put(struct stripe *stripe)
{
	if (atomic_dec_and_test(&stripe->cnt)) {
		BUG_ON(stripe_io_ref(stripe));
		stripe_unlock(stripe);
	} else
		BUG_ON(stripe_ref(stripe) < 0);
}

/* Helper needed by for_each_io_dev(). */
static void stripe_get_references(struct stripe *stripe, unsigned p)
{

	/*
	 * Another one to reference the stripe in
	 * order to protect vs. LRU list moves.
	 */
	io_get(RS(stripe->sc));	/* Global io references. */
	stripe_get(stripe);
	stripe_io_get(stripe);	/* One for each chunk io. */
}

/* Helper for endio() to put all take references. */
static void stripe_put_references(struct stripe *stripe)
{
	stripe_io_put(stripe);	/* One for each chunk io. */
	stripe_put(stripe);
	io_put(RS(stripe->sc));
}

/*
 * Stripe cache functions.
 */
/*
 * Invalidate all chunks (i.e. their pages)  of a stripe.
 *
 * I only keep state for the whole chunk.
 */
static inline void stripe_chunk_invalidate(struct stripe_chunk *chunk)
{
	chunk->io.flags = 0;
}

static void
stripe_chunks_invalidate(struct stripe *stripe)
{
	unsigned p = RS(stripe->sc)->set.raid_devs;

	while (p--)
		stripe_chunk_invalidate(CHUNK(stripe, p));
}

/* Prepare stripe for (re)use. */
static void stripe_invalidate(struct stripe *stripe)
{
	stripe->io.flags = 0;
	stripe->idx.parity = stripe->idx.recover = -1;
	stripe_chunks_invalidate(stripe);
}

/*
 * Allow io on all chunks of a stripe.
 * If not set, IO will not occur; i.e. it's prohibited.
 *
 * Actual IO submission for allowed chunks depends
 * on their !uptodate or dirty state.
 */
static void stripe_allow_io(struct stripe *stripe)
{
	unsigned p = RS(stripe->sc)->set.raid_devs;

	while (p--)
		SetChunkIo(CHUNK(stripe, p));
}

/* Initialize a stripe. */
static void stripe_init(struct stripe_cache *sc, struct stripe *stripe)
{
	unsigned i, p = RS(sc)->set.raid_devs;

	/* Work all io chunks. */
	while (p--) {
		struct stripe_chunk *chunk = CHUNK(stripe, p);

		atomic_set(&chunk->cnt, 0);
		chunk->stripe = stripe;
		i = ARRAY_SIZE(chunk->bl);
		while (i--)
			bio_list_init(chunk->bl + i);
	}

	stripe->sc = sc;


	i = ARRAY_SIZE(stripe->lists);
	while (i--)
		INIT_LIST_HEAD(stripe->lists + i);

	stripe->io.size = RS(sc)->set.io_size;
	atomic_set(&stripe->cnt, 0);
	atomic_set(&stripe->io.pending, 0);
	stripe_invalidate(stripe);
}

/* Number of pages per chunk. */
static inline unsigned chunk_pages(unsigned sectors)
{
	return dm_div_up(sectors, SECTORS_PER_PAGE);
}

/* Number of pages per stripe. */
static inline unsigned stripe_pages(struct raid_set *rs, unsigned io_size)
{
	return chunk_pages(io_size) * rs->set.raid_devs;
}

/* Initialize part of page_list (recovery). */
static void stripe_zero_pl_part(struct stripe *stripe, int p,
				unsigned start, unsigned count)
{
	unsigned o = start / SECTORS_PER_PAGE, pages = chunk_pages(count);
	/* Get offset into the page_list. */
	struct page_list *pl = pl_elem(PL(stripe, p), o);

	BUG_ON(!pl);
	while (pl && pages--) {
		BUG_ON(!pl->page);
		memset(page_address(pl->page), 0, PAGE_SIZE);
		pl = pl->next;
	}
}

/* Initialize parity chunk of stripe. */
static void stripe_zero_chunk(struct stripe *stripe, int p)
{
	if (p > -1)
		stripe_zero_pl_part(stripe, p, 0, stripe->io.size);
}

/* Return dynamic stripe structure size. */
static size_t stripe_size(struct raid_set *rs)
{
	return sizeof(struct stripe) +
		      rs->set.raid_devs * sizeof(struct stripe_chunk);
}

/* Allocate a stripe and its memory object. */
/* XXX adjust to cope with stripe cache and recovery stripe caches. */
enum grow { SC_GROW, SC_KEEP };
static struct stripe *stripe_alloc(struct stripe_cache *sc,
				   struct dm_mem_cache_client *mc,
				   enum grow grow)
{
	int r;
	struct stripe *stripe;

	stripe = kmem_cache_zalloc(sc->kc.cache, GFP_KERNEL);
	if (stripe) {
		/* Grow the dm-mem-cache by one object. */
		if (grow == SC_GROW) {
			r = dm_mem_cache_grow(mc, 1);
			if (r)
				goto err_free;
		}

		stripe->obj = dm_mem_cache_alloc(mc);
		if (!stripe->obj)
			goto err_shrink;

		stripe_init(sc, stripe);
	}

	return stripe;

err_shrink:
	if (grow == SC_GROW)
		dm_mem_cache_shrink(mc, 1);
err_free:
	kmem_cache_free(sc->kc.cache, stripe);
	return NULL;
}

/*
 * Free a stripes memory object, shrink the
 * memory cache and free the stripe itself.
 */
static void stripe_free(struct stripe *stripe, struct dm_mem_cache_client *mc)
{
	dm_mem_cache_free(mc, stripe->obj);
	dm_mem_cache_shrink(mc, 1);
	kmem_cache_free(stripe->sc->kc.cache, stripe);
}

/* Free the recovery stripe. */
static void stripe_recover_free(struct raid_set *rs)
{
	struct recover *rec = &rs->recover;
	struct dm_mem_cache_client *mc;

	mc = rec->mem_cache_client;
	rec->mem_cache_client = NULL;
	if (mc) {
		struct stripe *stripe;

		while (!list_empty(&rec->stripes)) {
			stripe = list_first_entry(&rec->stripes, struct stripe,
						  lists[LIST_RECOVER]);
			list_del(stripe->lists + LIST_RECOVER);
			kfree(stripe->recover);
			stripe_free(stripe, mc);
		}
	
		dm_mem_cache_client_destroy(mc);
		dm_io_client_destroy(rec->dm_io_client);
		rec->dm_io_client = NULL;
	}
}

/* Grow stripe cache. */
static int sc_grow(struct stripe_cache *sc, unsigned stripes, enum grow grow)
{
	int r = 0;

	/* Try to allocate this many (additional) stripes. */
	while (stripes--) {
		struct stripe *stripe =
			stripe_alloc(sc, sc->mem_cache_client, grow);

		if (likely(stripe)) {
			stripe_lru_add(stripe);
			atomic_inc(&sc->stripes);
		} else {
			r = -ENOMEM;
			break;
		}
	}

	return r ? r : sc_hash_resize(sc);
}

/* Shrink stripe cache. */
static int sc_shrink(struct stripe_cache *sc, unsigned stripes)
{
	int r = 0;

	/* Try to get unused stripe from LRU list. */
	while (stripes--) {
		struct stripe *stripe;

		stripe = stripe_lru_pop(sc);
		if (stripe) {
			/* An LRU stripe may never have ios pending! */
			BUG_ON(stripe_io_ref(stripe));
			BUG_ON(stripe_ref(stripe));
			atomic_dec(&sc->stripes);
			/* Remove from hash if on before deletion. */
			stripe_hash_del(stripe);
			stripe_free(stripe, sc->mem_cache_client);
		} else {
			r = -ENOENT;
			break;
		}
	}

	/* Check if stats are still sane. */
	if (atomic_read(&sc->active_stripes_max) >
	    atomic_read(&sc->stripes))
		atomic_set(&sc->active_stripes_max, 0);

	if (r)
		return r;

	return atomic_read(&sc->stripes) ? sc_hash_resize(sc) : 0;
}

/* Create stripe cache and recovery. */
static int sc_init(struct raid_set *rs, unsigned stripes)
{
	unsigned i, r, rstripes;
	struct stripe_cache *sc = &rs->sc;
	struct stripe *stripe;
	struct recover *rec = &rs->recover;
	struct mapped_device *md;
	struct gendisk *disk;

	/* Initialize lists and locks. */
	i = ARRAY_SIZE(sc->lists);
	while (i--)
		INIT_LIST_HEAD(sc->lists + i);

	INIT_LIST_HEAD(&rec->stripes);

	/* Initialize endio and LRU list locks. */
	i = NR_LOCKS;
	while (i--)
		spin_lock_init(sc->locks + i);

	/* Initialize atomic variables. */
	atomic_set(&sc->stripes, 0);
	atomic_set(&sc->stripes_to_set, 0);
	atomic_set(&sc->active_stripes, 0);
	atomic_set(&sc->active_stripes_max, 0);	/* REMOVEME: statistics. */

	/*
	 * We need a runtime unique # to suffix the kmem cache name
	 * because we'll have one for each active RAID set.
	 */
	md = dm_table_get_md(rs->ti->table);
	disk = dm_disk(md);
	sprintf(sc->kc.name, "%s-%d", TARGET, disk->first_minor);
	dm_put(md);
	sc->kc.cache = kmem_cache_create(sc->kc.name, stripe_size(rs),
					 0, 0, NULL);
	if (!sc->kc.cache)
		return -ENOMEM;

	/* Create memory cache client context for RAID stripe cache. */
	sc->mem_cache_client =
		dm_mem_cache_client_create(stripes, rs->set.raid_devs,
					   chunk_pages(rs->set.io_size));
	if (IS_ERR(sc->mem_cache_client))
		return PTR_ERR(sc->mem_cache_client);

	/* Create memory cache client context for RAID recovery stripe(s). */
	rstripes = rec->recovery_stripes;
	rec->mem_cache_client =
		dm_mem_cache_client_create(rstripes, rs->set.raid_devs,
					   chunk_pages(rec->io_size));
	if (IS_ERR(rec->mem_cache_client))
		return PTR_ERR(rec->mem_cache_client);

	/* Create dm-io client context for IO stripes. */
	sc->dm_io_client =
		dm_io_client_create((stripes > 32 ? 32 : stripes) *
				    rs->set.raid_devs *
				    chunk_pages(rs->set.io_size));
	if (IS_ERR(sc->dm_io_client))
		return PTR_ERR(sc->dm_io_client);

	/* FIXME: intermingeled with stripe cache initialization. */
	/* Create dm-io client context for recovery stripes. */
	rec->dm_io_client =
		dm_io_client_create(rstripes * rs->set.raid_devs *
				    chunk_pages(rec->io_size));
	if (IS_ERR(rec->dm_io_client))
		return PTR_ERR(rec->dm_io_client);

	/* Allocate stripes for set recovery. */
	while (rstripes--) {
		stripe = stripe_alloc(sc, rec->mem_cache_client, SC_KEEP);
		if (!stripe)
			return -ENOMEM;

		stripe->recover = kzalloc(sizeof(*stripe->recover), GFP_KERNEL);
		if (!stripe->recover) {
			stripe_free(stripe, rec->mem_cache_client);
			return -ENOMEM;
		}

		SetStripeRecover(stripe);
		stripe->io.size = rec->io_size;
		list_add_tail(stripe->lists + LIST_RECOVER, &rec->stripes);
		/* Don't add recovery stripes to LRU list! */
	}

	/*
	 * Allocate the stripe objetcs from the
	 * cache and add them to the LRU list.
	 */
	r = sc_grow(sc, stripes, SC_KEEP);
	if (!r)
		atomic_set(&sc->stripes_last, stripes);

	return r;
}

/* Destroy the stripe cache. */
static void sc_exit(struct stripe_cache *sc)
{
	struct raid_set *rs = RS(sc);

	if (sc->kc.cache) {
		stripe_recover_free(rs);
		BUG_ON(sc_shrink(sc, atomic_read(&sc->stripes)));
		kmem_cache_destroy(sc->kc.cache);
		sc->kc.cache = NULL;

		if (sc->mem_cache_client && !IS_ERR(sc->mem_cache_client))
			dm_mem_cache_client_destroy(sc->mem_cache_client);

		if (sc->dm_io_client && !IS_ERR(sc->dm_io_client))
			dm_io_client_destroy(sc->dm_io_client);

		hash_exit(&sc->hash);
	}
}

/*
 * Calculate RAID address
 *
 * Delivers tuple with the index of the data disk holding the chunk
 * in the set, the parity disks index and the start of the stripe
 * within the address space of the set (used as the stripe cache hash key).
 */
/* thx MD. */
static struct raid_address *raid_address(struct raid_set *rs, sector_t sector,
					 struct raid_address *addr)
{
	sector_t stripe, tmp;

	/*
	 * chunk_number = sector / chunk_size
	 * stripe_number = chunk_number / data_devs
	 * di = stripe % data_devs;
	 */
	stripe = sector >> rs->set.chunk_shift;
	addr->di = sector_div(stripe, rs->set.data_devs);

	switch (rs->set.raid_type->level) {
	case raid4:
		addr->pi = rs->set.pi;
		goto check_shift_di;
	case raid5:
		tmp = stripe;
		addr->pi = sector_div(tmp, rs->set.raid_devs);

		switch (rs->set.raid_type->algorithm) {
		case left_asym:		/* Left asymmetric. */
			addr->pi = rs->set.data_devs - addr->pi;
		case right_asym:	/* Right asymmetric. */
check_shift_di:
			if (addr->di >= addr->pi)
				addr->di++;
			break;
		case left_sym:		/* Left symmetric. */
			addr->pi = rs->set.data_devs - addr->pi;
		case right_sym:		/* Right symmetric. */
			addr->di = (addr->pi + addr->di + 1) %
				   rs->set.raid_devs;
			break;
		case none: /* Ain't happen: RAID4 algorithm placeholder. */
			BUG();
		}
	}

	/*
	 * Start offset of the stripes chunk on any single device of the RAID
	 * set, adjusted in case io size differs from chunk size.
	 */
	addr->key = (stripe << rs->set.chunk_shift) +
		    (sector & rs->set.io_inv_mask);
	return addr;
}

/*
 * Copy data across between stripe pages and bio vectors.
 *
 * Pay attention to data alignment in stripe and bio pages.
 */
static void bio_copy_page_list(int rw, struct stripe *stripe,
			       struct page_list *pl, struct bio *bio)
{
	unsigned i, page_offset;
	void *page_addr;
	struct raid_set *rs = RS(stripe->sc);
	struct bio_vec *bv;

	/* Get start page in page list for this sector. */
	i = (bio->bi_sector & rs->set.io_mask) / SECTORS_PER_PAGE;
	pl = pl_elem(pl, i);
	BUG_ON(!pl);
	BUG_ON(!pl->page);

	page_addr = page_address(pl->page);
	page_offset = to_bytes(bio->bi_sector & (SECTORS_PER_PAGE - 1));

	/* Walk all segments and copy data across between bio_vecs and pages. */
	bio_for_each_segment(bv, bio, i) {
		int len = bv->bv_len, size;
		unsigned bio_offset = 0;
		void *bio_addr = __bio_kmap_atomic(bio, i, KM_USER0);
redo:
		size = (page_offset + len > PAGE_SIZE) ?
		       PAGE_SIZE - page_offset : len;

		if (rw == READ)
			memcpy(bio_addr + bio_offset,
			       page_addr + page_offset, size);
		else
			memcpy(page_addr + page_offset,
			       bio_addr + bio_offset, size);

		page_offset += size;
		if (page_offset == PAGE_SIZE) {
			/*
			 * We reached the end of the chunk page ->
			 * need to refer to the next one to copy more data.
			 */
			len -= size;
			if (len) {
				/* Get next page. */
				pl = pl->next;
				BUG_ON(!pl);
				BUG_ON(!pl->page);
				page_addr = page_address(pl->page);
				page_offset = 0;
				bio_offset += size;
				/* REMOVEME: statistics. */
				atomic_inc(rs->stats + S_BIO_COPY_PL_NEXT);
				goto redo;
			}
		}

		__bio_kunmap_atomic(bio_addr, KM_USER0);
	}
}

/*
 * Xor optimization macros.
 */
/* Xor data pointer declaration and initialization macros. */
#define DECLARE_2	unsigned long *d0 = data[0], *d1 = data[1]
#define DECLARE_3	DECLARE_2, *d2 = data[2]
#define DECLARE_4	DECLARE_3, *d3 = data[3]
#define DECLARE_5	DECLARE_4, *d4 = data[4]
#define DECLARE_6	DECLARE_5, *d5 = data[5]
#define DECLARE_7	DECLARE_6, *d6 = data[6]
#define DECLARE_8	DECLARE_7, *d7 = data[7]

/* Xor unrole macros. */
#define D2(n)	d0[n] = d0[n] ^ d1[n]
#define D3(n)	D2(n) ^ d2[n]
#define D4(n)	D3(n) ^ d3[n]
#define D5(n)	D4(n) ^ d4[n]
#define D6(n)	D5(n) ^ d5[n]
#define D7(n)	D6(n) ^ d6[n]
#define D8(n)	D7(n) ^ d7[n]

#define	X_2(macro, offset)	macro(offset); macro(offset + 1);
#define	X_4(macro, offset)	X_2(macro, offset); X_2(macro, offset + 2);
#define	X_8(macro, offset)	X_4(macro, offset); X_4(macro, offset + 4);
#define	X_16(macro, offset)	X_8(macro, offset); X_8(macro, offset + 8);
#define	X_32(macro, offset)	X_16(macro, offset); X_16(macro, offset + 16);
#define	X_64(macro, offset)	X_32(macro, offset); X_32(macro, offset + 32);

/* Define a _xor_#chunks_#xors_per_run() function. */
#define	_XOR(chunks, xors_per_run) \
static void _xor ## chunks ## _ ## xors_per_run(unsigned long **data) \
{ \
	unsigned end = XOR_SIZE / sizeof(data[0]), i; \
	DECLARE_ ## chunks; \
\
	for (i = 0; i < end; i += xors_per_run) { \
		X_ ## xors_per_run(D ## chunks, i); \
	} \
}

/* Define xor functions for 2 - 8 chunks and xors per run. */
#define	MAKE_XOR_PER_RUN(xors_per_run) \
	_XOR(2, xors_per_run); _XOR(3, xors_per_run); \
	_XOR(4, xors_per_run); _XOR(5, xors_per_run); \
	_XOR(6, xors_per_run); _XOR(7, xors_per_run); \
	_XOR(8, xors_per_run);

MAKE_XOR_PER_RUN(8)	/* Define _xor_*_8() functions. */
MAKE_XOR_PER_RUN(16)	/* Define _xor_*_16() functions. */
MAKE_XOR_PER_RUN(32)	/* Define _xor_*_32() functions. */
MAKE_XOR_PER_RUN(64)	/* Define _xor_*_64() functions. */

#define MAKE_XOR(xors_per_run) \
struct { \
	void (*f)(unsigned long **); \
} static xor_funcs ## xors_per_run[] = { \
	{ NULL }, /* NULL pointers to optimize indexing in xor(). */ \
	{ NULL }, \
	{ _xor2_ ## xors_per_run }, \
	{ _xor3_ ## xors_per_run }, \
	{ _xor4_ ## xors_per_run }, \
	{ _xor5_ ## xors_per_run }, \
	{ _xor6_ ## xors_per_run }, \
	{ _xor7_ ## xors_per_run }, \
	{ _xor8_ ## xors_per_run }, \
}; \
\
static void xor_ ## xors_per_run(unsigned n, unsigned long **data) \
{ \
	/* Call respective function for amount of chunks. */ \
	xor_funcs ## xors_per_run[n].f(data); \
}

/* Define xor_8() - xor_64 functions. */
MAKE_XOR(8)
MAKE_XOR(16)
MAKE_XOR(32)
MAKE_XOR(64)

/* Maximum number of chunks, which can be xor'ed in one go. */
#define	XOR_CHUNKS_MAX	(ARRAY_SIZE(xor_funcs8) - 1)

static void xor_blocks_wrapper(unsigned n, unsigned long **data)
{
	BUG_ON(n < 2 || n > MAX_XOR_BLOCKS + 1);
	xor_blocks(n - 1, XOR_SIZE, (void *) data[0], (void **) data + 1);
}

struct xor_func {
	xor_function_t f;
	const char *name;
} static xor_funcs[] = {
	{ xor_8,   "xor_8"  },
	{ xor_16,  "xor_16" },
	{ xor_32,  "xor_32" },
	{ xor_64,  "xor_64" },
	{ xor_blocks_wrapper, "xor_blocks" },
};

/*
 * Check, if chunk has to be xored in/out:
 *
 * o if writes are queued
 * o if writes are merged
 * o if stripe is to be reconstructed
 * o if recovery stripe
 */
static inline int chunk_must_xor(struct stripe_chunk *chunk)
{
	if (ChunkUptodate(chunk)) {
		BUG_ON(!bio_list_empty(BL_CHUNK(chunk, WRITE_QUEUED)) &&
		       !bio_list_empty(BL_CHUNK(chunk, WRITE_MERGED)));

		if (!bio_list_empty(BL_CHUNK(chunk, WRITE_QUEUED)) ||
		    !bio_list_empty(BL_CHUNK(chunk, WRITE_MERGED)))
			return 1;

		if (StripeReconstruct(chunk->stripe) ||
		    StripeRecover(chunk->stripe))
			return 1;
	}

	return 0;
}

/*
 * Calculate crc.
 *
 * This indexes into the chunks of a stripe and their pages.
 *
 * All chunks will be xored into the indexed (@pi)
 * chunk in maximum groups of xor.chunks.
 *
 */
static void xor(struct stripe *stripe, unsigned pi, unsigned sector)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned max_chunks = rs->xor.chunks, n = 1,
		 o = sector / SECTORS_PER_PAGE, /* Offset into the page_list. */
		 p = rs->set.raid_devs;
	unsigned long **d = rs->data;
	xor_function_t xor_f = rs->xor.f->f;

	BUG_ON(sector > stripe->io.size);

	/* Address of parity page to xor into. */
	d[0] = page_address(pl_elem(PL(stripe, pi), o)->page);

	while (p--) {
		/* Preset pointers to data pages. */
		if (p != pi && chunk_must_xor(CHUNK(stripe, p)))
			d[n++] = page_address(pl_elem(PL(stripe, p), o)->page);

		/* If max chunks -> xor. */
		if (n == max_chunks) {
			xor_f(n, d);
			n = 1;
		}
	}

	/* If chunks -> xor. */
	if (n > 1)
		xor_f(n, d);
}

/* Common xor loop through all stripe page lists. */
static void common_xor(struct stripe *stripe, sector_t count,
		       unsigned off, unsigned pi)
{
	unsigned sector;

	BUG_ON(!count);
	for (sector = off; sector < count; sector += SECTORS_PER_PAGE)
		xor(stripe, pi, sector);

	/* Set parity page uptodate and clean. */
	chunk_set(CHUNK(stripe, pi), CLEAN);
	atomic_inc(RS(stripe->sc)->stats + S_XORS); /* REMOVEME: statistics. */
}

/*
 * Calculate parity sectors on intact stripes.
 *
 * Need to calculate raid address for recover stripe, because its
 * chunk sizes differs and is typically larger than io chunk size.
 */
static void parity_xor(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned chunk_size = rs->set.chunk_size, io_size = stripe->io.size,
		 xor_size = chunk_size > io_size ? io_size : chunk_size;
	sector_t off;

	/* This can be the recover stripe with a larger io size. */
	for (off = 0; off < io_size; off += xor_size) {
		/*
		 * Recover stripe is likely bigger than regular io
		 * ones and has no precalculated parity disk index ->
		 * need to calculate RAID address.
		 */
		if (unlikely(StripeRecover(stripe))) {
			struct raid_address addr;

			raid_address(rs,
				     (stripe->key + off) * rs->set.data_devs,
				     &addr);
			stripe->idx.parity = addr.pi;
			stripe_zero_pl_part(stripe, addr.pi, off, xor_size);
		}

		common_xor(stripe, xor_size, off, stripe->idx.parity);
		chunk_set(CHUNK(stripe, stripe->idx.parity), DIRTY);
	}
}

/* Reconstruct missing chunk. */
static void stripe_reconstruct(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	int p = rs->set.raid_devs, pr = stripe->idx.recover;

	BUG_ON(pr < 0);

	/* Check if all but the chunk to be reconstructed are uptodate. */
	while (p--)
		BUG_ON(p != pr && !ChunkUptodate(CHUNK(stripe, p)));

	/* REMOVEME: statistics. */
	atomic_inc(rs->stats + (RSDegraded(rs) ? S_RECONSTRUCT_EI :
						 S_RECONSTRUCT_DEV));
	/* Zero chunk to be reconstructed. */
	stripe_zero_chunk(stripe, pr);
	common_xor(stripe, stripe->io.size, 0, pr);
	stripe->idx.recover = -1;
}

/*
 * Recovery io throttling
 */
/* Conditionally reset io counters. */
static int recover_io_reset(struct raid_set *rs)
{
	unsigned long j = jiffies;

	/* Pay attention to jiffies overflows. */
	if (j > rs->recover.last_jiffies + HZ / 20 ||
	    j < rs->recover.last_jiffies) {
		atomic_set(rs->recover.io_count + IO_WORK, 0);
		atomic_set(rs->recover.io_count + IO_RECOVER, 0);
		rs->recover.last_jiffies = j;
		return 1;
	}

	return 0;
}

/* Count ios. */
static void recover_io_count(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);

	recover_io_reset(rs);
	atomic_inc(rs->recover.io_count +
		   (StripeRecover(stripe) ? IO_RECOVER : IO_WORK));
}

/* Try getting a stripe either from the hash or from the LRU list. */
static struct stripe *stripe_find(struct raid_set *rs,
				  struct raid_address *addr)
{
	int r;
	struct stripe_cache *sc = &rs->sc;
	struct stripe *stripe;

	/* Try stripe from hash. */
	stripe = stripe_lookup(sc, addr->key);
	if (stripe) {
		r = stripe_get(stripe);
		if (r)
			goto get_lock_failed;

		atomic_inc(rs->stats + S_HITS_1ST); /* REMOVEME: statistics. */
	} else {
		/* Not in hash -> try to get an LRU stripe. */
		stripe = stripe_lru_pop(sc);
		if (stripe) {
			/*
			 * An LRU stripe may not be referenced
			 * and may never have ios pending!
			 */
			BUG_ON(stripe_ref(stripe));
			BUG_ON(stripe_io_ref(stripe));

			/* Remove from hash if on before reuse. */
			stripe_hash_del(stripe);

			/* Invalidate before reinserting with changed key. */
			stripe_invalidate(stripe);

			stripe->key = addr->key;
			stripe->region = dm_rh_sector_to_region(rs->recover.rh,
								addr->key);
			stripe->idx.parity = addr->pi;
			r = stripe_get(stripe);
			if (r)
				goto get_lock_failed;

			/* Insert stripe into the stripe hash. */
			stripe_insert(&sc->hash, stripe);
			/* REMOVEME: statistics. */
			atomic_inc(rs->stats + S_INSCACHE);
		}
	}

	return stripe;

get_lock_failed:
	stripe_put(stripe);
	return NULL;
}

/*
 * Process end io
 *
 * I need to do it here because I can't in interrupt
 */
/* End io all bios on a bio list. */
static void bio_list_endio(struct stripe *stripe, struct bio_list *bl,
			   int p, int error)
{
	struct raid_set *rs = RS(stripe->sc);
	struct bio *bio;
	struct page_list *pl = PL(stripe, p);
	struct stripe_chunk *chunk = CHUNK(stripe, p);

	/* Update region counters. */
	while ((bio = bio_list_pop(bl))) {
		if (bio_data_dir(bio) == WRITE)
			/* Drop io pending count for any writes. */
			dm_rh_dec(rs->recover.rh, stripe->region);
		else if (!error)
			/* Copy data accross. */
			bio_copy_page_list(READ, stripe, pl, bio);

		bio_endio(bio, error);

		/* REMOVEME: statistics. */
		atomic_inc(rs->stats + (bio_data_dir(bio) == READ ?
			   S_BIOS_ENDIO_READ : S_BIOS_ENDIO_WRITE));

		chunk_put(chunk);
		stripe_put(stripe);
		io_put(rs);	/* Wake any suspend waiters on last bio. */
	}
}

/*
 * End io all reads/writes on a stripe copying
 * read data accross from stripe to bios and
 * decrementing region counters for writes.
 *
 * Processing of ios depeding on state:
 * o no chunk error -> endio ok
 * o degraded:
 *   - chunk error and read -> ignore to be requeued
 *   - chunk error and write -> endio ok
 * o dead (more than parity_devs failed) and chunk_error-> endio failed
 */
static void stripe_endio(int rw, struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned p = rs->set.raid_devs;
	int write = (rw != READ);

	while (p--) {
		struct stripe_chunk *chunk = CHUNK(stripe, p);
		struct bio_list *bl;

		BUG_ON(ChunkLocked(chunk));

		bl = BL_CHUNK(chunk, rw);
		if (bio_list_empty(bl))
			continue;

		if (unlikely(ChunkError(chunk) || !ChunkUptodate(chunk))) {
			/* RAID set dead. */
			if (unlikely(RSDead(rs)))
				bio_list_endio(stripe, bl, p, -EIO);
			/* RAID set degraded. */
			else if (write)
				bio_list_endio(stripe, bl, p, 0);
		} else {
			BUG_ON(!RSDegraded(rs) && ChunkDirty(chunk));
			bio_list_endio(stripe, bl, p, 0);
		}
	}
}

/* Fail all ios hanging off all bio lists of a stripe. */
static void stripe_fail_io(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned p = rs->set.raid_devs;

	while (p--) {
		struct stripe_chunk *chunk = CHUNK(stripe, p);
		int i = ARRAY_SIZE(chunk->bl);

		/* Fail all bios on all bio lists of the stripe. */
		while (i--) {
			struct bio_list *bl = chunk->bl + i;

			if (!bio_list_empty(bl))
				bio_list_endio(stripe, bl, p, -EIO);
		}
	}

	/* Put stripe on LRU list. */
	BUG_ON(stripe_io_ref(stripe));
	BUG_ON(stripe_ref(stripe));
}

/* Unlock all required chunks. */
static void stripe_chunks_unlock(struct stripe *stripe)
{
	unsigned p = RS(stripe->sc)->set.raid_devs;
	struct stripe_chunk *chunk;

	while (p--) {
		chunk = CHUNK(stripe, p);

		if (TestClearChunkUnlock(chunk))
			ClearChunkLocked(chunk);
	}
}

/*
 * Queue reads and writes to a stripe by hanging
 * their bios off the stripesets read/write lists.
 */
static int stripe_queue_bio(struct raid_set *rs, struct bio *bio,
			    struct bio_list *reject)
{
	struct raid_address addr;
	struct stripe *stripe;

	stripe = stripe_find(rs, raid_address(rs, bio->bi_sector, &addr));
	if (stripe) {
		int r = 0, rw = bio_data_dir(bio);

		/* Distinguish reads and writes. */
		bio_list_add(BL(stripe, addr.di, rw), bio);
	
		if (rw == READ)
			/* REMOVEME: statistics. */
			atomic_inc(rs->stats + S_BIOS_ADDED_READ);
		else {
			/* Inrement pending write count on region. */
			dm_rh_inc(rs->recover.rh, stripe->region);
			r = 1;

			/* REMOVEME: statistics. */
			atomic_inc(rs->stats + S_BIOS_ADDED_WRITE);
		}

		/*
		 * Put on io (flush) list in case of
		 * initial bio queued to chunk.
		 */
		if (chunk_get(CHUNK(stripe, addr.di)) == 1)
			stripe_flush_add(stripe);

		return r;
	}

	/* Got no stripe from cache or failed to lock it -> reject bio. */
	bio_list_add(reject, bio);
	atomic_inc(rs->stats + S_IOS_POST); /* REMOVEME: statistics. */
	return 0;
}

/*
 * Handle all stripes by handing them to the daemon, because we can't
 * map their chunk pages to copy the data in interrupt context.
 *
 * We don't want to handle them here either, while interrupts are disabled.
 */

/* Read/write endio function for dm-io (interrupt context). */
static void endio(unsigned long error, void *context)
{
	struct stripe_chunk *chunk = context;

	if (unlikely(error)) {
		chunk_set(chunk, ERROR);
		/* REMOVEME: statistics. */
		atomic_inc(RS(chunk->stripe->sc)->stats + S_STRIPE_ERROR);
	} else
		chunk_set(chunk, CLEAN);

	/*
	 * For recovery stripes, I need to reset locked locked
	 * here, because those aren't processed in do_endios().
	 */
	if (unlikely(StripeRecover(chunk->stripe)))
		ClearChunkLocked(chunk);
	else
		SetChunkUnlock(chunk);

	/* Indirectly puts stripe on cache's endio list via stripe_io_put(). */
	stripe_put_references(chunk->stripe);
}

/* Read/Write a chunk asynchronously. */
static void stripe_chunk_rw(struct stripe *stripe, unsigned p)
{
	struct stripe_cache *sc = stripe->sc;
	struct raid_set *rs = RS(sc);
	struct dm_mem_cache_object *obj = stripe->obj + p;
	struct page_list *pl = obj->pl;
	struct stripe_chunk *chunk = CHUNK(stripe, p);
	struct raid_dev *dev = rs->dev + p;
	struct dm_io_region io = {
		.bdev = dev->dev->bdev,
		.sector = stripe->key,
		.count = stripe->io.size,
	};
	struct dm_io_request control = {
		.bi_rw = ChunkDirty(chunk) ? WRITE : READ,
		.mem = {
			.type = DM_IO_PAGE_LIST,
			.ptr.pl = pl,
			.offset = 0,
		},
		.notify = {
			.fn = endio,
			.context = chunk,
		},
		.client = StripeRecover(stripe) ? rs->recover.dm_io_client :
						  sc->dm_io_client,
	};

	BUG_ON(ChunkLocked(chunk));
	BUG_ON(!ChunkUptodate(chunk) && ChunkDirty(chunk));
	BUG_ON(ChunkUptodate(chunk) && !ChunkDirty(chunk));

	/*
	 * Don't rw past end of device, which can happen, because
	 * typically sectors_per_dev isn't divisible by io_size.
	 */
	if (unlikely(io.sector + io.count > rs->set.sectors_per_dev))
		io.count = rs->set.sectors_per_dev - io.sector;

	BUG_ON(!io.count);
	io.sector += dev->start;	/* Add <offset>. */
	if (RSRecover(rs))
		recover_io_count(stripe);	/* Recovery io accounting. */

	/* REMOVEME: statistics. */
	atomic_inc(rs->stats + (ChunkDirty(chunk) ? S_DM_IO_WRITE :
						    S_DM_IO_READ));
	SetChunkLocked(chunk);
	SetDevIoQueued(dev);
	BUG_ON(dm_io(&control, 1, &io, NULL));
}

/*
 * Write dirty or read not uptodate page lists of a stripe.
 */
static int stripe_chunks_rw(struct stripe *stripe)
{
	int r;
	struct raid_set *rs = RS(stripe->sc);

	/*
	 * Increment the pending count on the stripe
	 * first, so that we don't race in endio().
	 *
	 * An inc (IO) is needed for any chunk unless !ChunkIo(chunk):
	 *
	 * o not uptodate
	 * o dirtied by writes merged
	 * o dirtied by parity calculations
	 */
	r = for_each_io_dev(stripe, stripe_get_references);
	if (r) {
		/* Io needed: chunks are either not uptodate or dirty. */
		int max;	/* REMOVEME: */
		struct stripe_cache *sc = &rs->sc;

		/* Submit actual io. */
		for_each_io_dev(stripe, stripe_chunk_rw);

		/* REMOVEME: statistics */
		max = sc_active(sc);
		if (atomic_read(&sc->active_stripes_max) < max)
			atomic_set(&sc->active_stripes_max, max);

		atomic_inc(rs->stats + S_FLUSHS);
		/* END REMOVEME: statistics */
	}

	return r;
}

/* Merge in all writes hence dirtying respective chunks. */
static void stripe_merge_writes(struct stripe *stripe)
{
	unsigned p = RS(stripe->sc)->set.raid_devs;

	while (p--) {
		struct stripe_chunk *chunk = CHUNK(stripe, p);
		struct bio_list *write = BL_CHUNK(chunk, WRITE_QUEUED);
	
		if (!bio_list_empty(write)) {
			struct bio *bio;
			struct page_list *pl = stripe->obj[p].pl;

			/*
			 * We can play with the lists without holding a lock,
			 * because it is just us accessing them anyway.
			 */
			bio_list_for_each(bio, write)
				bio_copy_page_list(WRITE, stripe, pl, bio);

			bio_list_merge(BL_CHUNK(chunk, WRITE_MERGED), write);
			bio_list_init(write);
			chunk_set(chunk, DIRTY);
		}
	}
}

/* Queue all writes to get merged. */
static int stripe_queue_writes(struct stripe *stripe)
{
	int r = 0;
	unsigned p = RS(stripe->sc)->set.raid_devs;

	while (p--) {
		struct stripe_chunk *chunk = CHUNK(stripe, p);
		struct bio_list *write = BL_CHUNK(chunk, WRITE);

		if (!bio_list_empty(write)) {
			bio_list_merge(BL_CHUNK(chunk, WRITE_QUEUED), write);
			bio_list_init(write);
SetChunkIo(chunk);
			r = 1;
		}
	}

	return r;
}


/* Check, if a chunk gets completely overwritten. */
static int stripe_check_chunk_overwrite(struct stripe *stripe, unsigned p)
{
	unsigned sectors = 0;
	struct bio *bio;
	struct bio_list *bl = BL(stripe, p, WRITE_QUEUED);

	bio_list_for_each(bio, bl)
		sectors += bio_sectors(bio);

	BUG_ON(sectors > RS(stripe->sc)->set.io_size);
	return sectors == RS(stripe->sc)->set.io_size;
}

/*
 * Avoid io on broken/reconstructed drive in order to
 * reconstruct date on endio.
 *
 * (*1*) We set StripeReconstruct() in here, so that _do_endios()
 *	 will trigger a reconstruct call before resetting it.
 */
static int stripe_chunk_set_io_flags(struct stripe *stripe, int pr)
{
	struct stripe_chunk *chunk = CHUNK(stripe, pr);

	/*
	 * Allow io on all chunks but the indexed one,
	 * because we're either degraded or prohibit it
	 * on the one for later reconstruction.
	 */
	/* Includes ClearChunkIo(), ClearChunkUptodate(). */
	stripe_chunk_invalidate(chunk);
	stripe->idx.recover = pr;
	SetStripeReconstruct(stripe);

	/* REMOVEME: statistics. */
	atomic_inc(RS(stripe->sc)->stats + S_PROHIBITCHUNKIO);
	return -EPERM;
}

/* Chunk locked/uptodate and device failed tests. */
static struct stripe_chunk *
stripe_chunk_check(struct stripe *stripe, unsigned p, unsigned *chunks_uptodate)
{
	struct raid_set *rs = RS(stripe->sc);
	struct stripe_chunk *chunk = CHUNK(stripe, p);

	/* Can't access active chunks. */
	if (ChunkLocked(chunk)) {
		/* REMOVEME: statistics. */
		atomic_inc(rs->stats + S_CHUNK_LOCKED);
		return NULL;
	}

	/* Can't access broken devive. */
	if (ChunkError(chunk) || DevFailed(rs->dev + p))
		return NULL;

	/* Can access uptodate chunks. */
	if (ChunkUptodate(chunk)) {
		(*chunks_uptodate)++;
		return NULL;
	}

	return chunk;
}

/*
 * Degraded/reconstruction mode.
 *
 * Check stripe state to figure which chunks don't need IO.
 *
 * Returns 0 for fully operational, -EPERM for degraded/resynchronizing.
 */
static int stripe_check_reconstruct(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);

	if (RSDead(rs)) {
		ClearStripeReconstruct(stripe);
		ClearStripeReconstructed(stripe);
		stripe_allow_io(stripe);
		return 0;
	}

	/* Avoid further reconstruction setting, when already set. */
	if (StripeReconstruct(stripe)) {
		/* REMOVEME: statistics. */
		atomic_inc(rs->stats + S_RECONSTRUCT_SET);
		return -EBUSY;
	}

	/* Initially allow io on all chunks. */
	stripe_allow_io(stripe);

	/* Return if stripe is already reconstructed. */
	if (StripeReconstructed(stripe)) {
		atomic_inc(rs->stats + S_RECONSTRUCTED);
		return 0;
	}

	/*
	 * Degraded/reconstruction mode (device failed) ->
	 * avoid io on the failed device.
	 */
	if (unlikely(RSDegraded(rs))) {
		/* REMOVEME: statistics. */
		atomic_inc(rs->stats + S_DEGRADED);
		/* Allow IO on all devices but the dead one. */
		BUG_ON(rs->set.ei < 0);
		return stripe_chunk_set_io_flags(stripe, rs->set.ei);
	} else {
		int sync, pi = dev_for_parity(stripe, &sync);

		/*
		 * Reconstruction mode (ie. a particular (replaced) device or
		 * some (rotating) parity chunk is being resynchronized) ->
		 *   o make sure all needed chunks are read in
		 *   o writes are allowed to go through
		 */
		if (!sync) {
			/* REMOVEME: statistics. */
			atomic_inc(rs->stats + S_NOSYNC);
			/* Allow IO on all devs but the one to reconstruct. */
			return stripe_chunk_set_io_flags(stripe, pi);
		}
	}

	return 0;
}

/*
 * Check, if stripe is ready to merge writes.
 * I.e. if all chunks present to allow to merge bios.
 *
 * We prohibit io on:
 *
 * o chunks without bios
 * o chunks which get completely written over
 */
static int stripe_merge_possible(struct stripe *stripe, int nosync)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned chunks_overwrite = 0, chunks_prohibited = 0,
		 chunks_uptodate = 0, p = rs->set.raid_devs;

	/* Walk all chunks. */
	while (p--) {
		struct stripe_chunk *chunk;

		/* Prohibit io on broken devices. */
		if (DevFailed(rs->dev + p)) {
			chunk = CHUNK(stripe, p);
			goto prohibit_io;
		}

		/* We can't optimize any further if no chunk. */
		chunk = stripe_chunk_check(stripe, p, &chunks_uptodate);
		if (!chunk || nosync)
			continue;

		/*
		 * We have a chunk, which is not uptodate.
		 *
		 * If this is not parity and we don't have
		 * reads queued, we can optimize further.
		 */
		if (p != stripe->idx.parity &&
		    bio_list_empty(BL_CHUNK(chunk, READ)) &&
		    bio_list_empty(BL_CHUNK(chunk, WRITE_MERGED))) {
			if (bio_list_empty(BL_CHUNK(chunk, WRITE_QUEUED)))
				goto prohibit_io;
			else if (RSCheckOverwrite(rs) &&
				 stripe_check_chunk_overwrite(stripe, p))
				/* Completely overwritten chunk. */
				chunks_overwrite++;
		}

		/* Allow io for chunks with bios and overwritten ones. */
		SetChunkIo(chunk);
		continue;

prohibit_io:
		/* No io for broken devices or for chunks w/o bios. */
		ClearChunkIo(chunk);
		chunks_prohibited++;
		/* REMOVEME: statistics. */
		atomic_inc(RS(stripe->sc)->stats + S_PROHIBITCHUNKIO);
	}

	/* All data chunks will get written over. */
	if (chunks_overwrite == rs->set.data_devs)
		atomic_inc(rs->stats + S_OVERWRITE); /* REMOVEME: statistics.*/
	else if (chunks_uptodate + chunks_prohibited < rs->set.raid_devs) {
		/* We don't have enough chunks to merge. */
		atomic_inc(rs->stats + S_CANT_MERGE); /* REMOVEME: statistics.*/
		return -EPERM;
	}

	/*
	 * If we have all chunks up to date or overwrite them, we
	 * just zero the parity chunk and let stripe_rw() recreate it.
	 */
	if (chunks_uptodate == rs->set.raid_devs ||
	    chunks_overwrite == rs->set.data_devs) {
		stripe_zero_chunk(stripe, stripe->idx.parity);
		BUG_ON(StripeReconstruct(stripe));
		SetStripeReconstruct(stripe);	/* Enforce xor in caller. */
	} else {
		/*
		 * With less chunks, we xor parity out.
		 *
		 * (*4*) We rely on !StripeReconstruct() in chunk_must_xor(),
		 *	 so that only chunks with queued or merged writes 
		 *	 are being xored.
		 */
		parity_xor(stripe);
	}

	/*
	 * We do have enough chunks to merge.
	 * All chunks are uptodate or get written over.
	 */
	atomic_inc(rs->stats + S_CAN_MERGE); /* REMOVEME: statistics. */
	return 0;
}

/*
 * Avoid reading chunks in case we're fully operational.
 *
 * We prohibit io on any chunks without bios but the parity chunk.
 */
static void stripe_avoid_reads(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	unsigned dummy = 0, p = rs->set.raid_devs;

	/* Walk all chunks. */
	while (p--) {
		struct stripe_chunk *chunk =
			stripe_chunk_check(stripe, p, &dummy);

		if (!chunk)
			continue;

		/* If parity or any bios pending -> allow io. */
		if (chunk_ref(chunk) || p == stripe->idx.parity)
			SetChunkIo(chunk);
		else {
			ClearChunkIo(chunk);
			/* REMOVEME: statistics. */
			atomic_inc(RS(stripe->sc)->stats + S_PROHIBITCHUNKIO);
		}
	}
}

/*
 * Read/write a stripe.
 *
 * All stripe read/write activity goes through this function
 * unless recovery, which has to call stripe_chunk_rw() directly.
 *
 * Make sure we don't try already merged stripes in order
 * to avoid data corruption.
 *
 * Check the state of the RAID set and if degraded (or
 * resynchronizing for reads), read in all other chunks but
 * the one on the dead/resynchronizing device in order to be
 * able to reconstruct the missing one in _do_endios().
 *
 * Can be called on active stripes in order
 * to dispatch new io on inactive chunks.
 *
 * States to cover:
 *   o stripe to read and/or write
 *   o stripe with error to reconstruct
 */
static void stripe_rw(struct stripe *stripe)
{
	int nosync, r;
	struct raid_set *rs = RS(stripe->sc);

	/*
 	 * Check, if a chunk needs to be reconstructed
 	 * because of a degraded set or a region out of sync.
 	 */
	nosync = stripe_check_reconstruct(stripe);
	switch (nosync) {
	case -EBUSY:
		return; /* Wait for stripe reconstruction to finish. */
	case -EPERM:
		goto io;
	}

	/*
	 * If we don't have merged writes pending, we can schedule
	 * queued writes to be merged next without corrupting data.
	 */
	if (!StripeMerged(stripe)) {
		r = stripe_queue_writes(stripe);
		if (r)
			/* Writes got queued -> flag RBW. */
			SetStripeRBW(stripe);
	}

	/*
	 * Merge all writes hanging off uptodate/overwritten
	 * chunks of the stripe.
	 */
	if (StripeRBW(stripe)) {
		r = stripe_merge_possible(stripe, nosync);
		if (!r) { /* Merge possible. */
			struct stripe_chunk *chunk;

			/*
			 * I rely on valid parity in order
			 * to xor a fraction of chunks out
			 * of parity and back in.
			 */
			stripe_merge_writes(stripe);	/* Merge writes in. */
			parity_xor(stripe);		/* Update parity. */
			ClearStripeReconstruct(stripe);	/* Reset xor enforce. */
			SetStripeMerged(stripe);	/* Writes merged. */
			ClearStripeRBW(stripe);		/* Disable RBW. */

			/*
			 * REMOVEME: sanity check on parity chunk
			 * 	     states after writes got merged.
			 */
			chunk = CHUNK(stripe, stripe->idx.parity);
			BUG_ON(ChunkLocked(chunk));
			BUG_ON(!ChunkUptodate(chunk));
			BUG_ON(!ChunkDirty(chunk));
			BUG_ON(!ChunkIo(chunk));
		}
	} else if (!nosync && !StripeMerged(stripe))
		/* Read avoidance if not degraded/resynchronizing/merged. */
		stripe_avoid_reads(stripe);

io:
	/* Now submit any reads/writes for non-uptodate or dirty chunks. */
	r = stripe_chunks_rw(stripe);
	if (!r) {
		/*
		 * No io submitted because of chunk io
		 * prohibited or locked chunks/failed devices
		 * -> push to end io list for processing.
		 */
		stripe_endio_push(stripe);
		atomic_inc(rs->stats + S_NO_RW); /* REMOVEME: statistics. */
	}
}

/*
 * Recovery functions
 */
/* Read a stripe off a raid set for recovery. */
static int stripe_recover_read(struct stripe *stripe, int pi)
{
	BUG_ON(stripe_io_ref(stripe));

	/* Invalidate all chunks so that they get read in. */
	stripe_chunks_invalidate(stripe);
	stripe_allow_io(stripe); /* Allow io on all recovery chunks. */

	/*
	 * If we are reconstructing a perticular device, we can avoid
 	 * reading the respective chunk in, because we're going to
	 * reconstruct it anyway.
	 *
	 * We can't do that for resynchronization of rotating parity,
	 * because the recovery stripe chunk size is typically larger
	 * than the sets chunk size.
	 */
	if (pi > -1)
		ClearChunkIo(CHUNK(stripe, pi));

	return stripe_chunks_rw(stripe);
}

/* Write a stripe to a raid set for recovery. */
static int stripe_recover_write(struct stripe *stripe, int pi)
{
	BUG_ON(stripe_io_ref(stripe));

	/*
	 * If this is a reconstruct of a particular device, then
	 * reconstruct the respective chunk, else create parity chunk.
	 */
	if (pi > -1) {
		stripe_zero_chunk(stripe, pi);
		common_xor(stripe, stripe->io.size, 0, pi);
		chunk_set(CHUNK(stripe, pi), DIRTY);
	} else
		parity_xor(stripe);

	return stripe_chunks_rw(stripe);
}

/* Read/write a recovery stripe. */
static int stripe_recover_rw(struct stripe *stripe)
{
	int r = 0, sync = 0;

	/* Read/write flip-flop. */
	if (TestClearStripeRBW(stripe)) {
		SetStripeMerged(stripe);
		stripe->key = stripe->recover->pos;
		r = stripe_recover_read(stripe, dev_for_parity(stripe, &sync));
		BUG_ON(!r);
	} else if (TestClearStripeMerged(stripe)) {
		r = stripe_recover_write(stripe, dev_for_parity(stripe, &sync));
		BUG_ON(!r);
	}

	BUG_ON(sync);
	return r;
}

/* Recover bandwidth available ?. */
static int recover_bandwidth(struct raid_set *rs)
{
	int r, work;

	/* On reset or when bios delayed -> allow recovery. */
	r = recover_io_reset(rs);
	if (r || RSBandwidth(rs))
		goto out;

	work = atomic_read(rs->recover.io_count + IO_WORK);
	if (work) {
		/* Pay attention to larger recover stripe size. */
		int recover = atomic_read(rs->recover.io_count + IO_RECOVER) *
					  rs->recover.io_size / rs->set.io_size;

		/*
		 * Don't use more than given bandwidth
		 * of the work io for recovery.
		 */
		if (recover > work / rs->recover.bandwidth_work) {
			/* REMOVEME: statistics. */
			atomic_inc(rs->stats + S_NO_BANDWIDTH);
			return 0;
		}
	}

out:
	atomic_inc(rs->stats + S_BANDWIDTH);	/* REMOVEME: statistics. */
	return 1;
}

/* Try to get a region to recover. */
static int stripe_recover_get_region(struct stripe *stripe)
{
	struct raid_set *rs = RS(stripe->sc);
	struct recover *rec = &rs->recover;
	struct recover_addr *addr = stripe->recover;
	struct dm_dirty_log *dl = rec->dl;
	struct dm_rh_client *rh = rec->rh;

	BUG_ON(!dl);
	BUG_ON(!rh);

	/* Return, that we have region first to finish it during suspension. */
	if (addr->reg)
		return 1;

	if (RSSuspend(rs))
		return -EPERM;

	if (dl->type->get_sync_count(dl) >= rec->nr_regions)
		return -ENOENT;

	/* If we don't have enough bandwidth, we don't proceed recovering. */
	if (!recover_bandwidth(rs))
		return -EAGAIN;

	/* Start quiescing a region. */
	dm_rh_recovery_prepare(rh);
	addr->reg = dm_rh_recovery_start(rh);
	if (!addr->reg)
		return -EAGAIN;

	addr->pos = dm_rh_region_to_sector(rh, dm_rh_get_region_key(addr->reg));
	addr->end = addr->pos + dm_rh_get_region_size(rh);

	/*
	 * Take one global io reference out for the
	 * whole region, which is going to be released
	 * when the region is completely done with.
	 */
	io_get(rs);
	return 0;
}

/* Update region hash state. */
enum recover_type { REC_FAILURE = 0, REC_SUCCESS = 1 };
static void recover_rh_update(struct stripe *stripe, enum recover_type success)
{
	struct recover_addr *addr = stripe->recover;
	struct raid_set *rs = RS(stripe->sc);
	struct recover *rec = &rs->recover;

	if (!addr->reg) {
		DMERR("%s- Called w/o region", __func__);
		return;
	}

	dm_rh_recovery_end(addr->reg, success);
	if (success)
		rec->nr_regions_recovered++;

	addr->reg = NULL;

	/*
	 * Completely done with this region ->
	 * release the 1st io reference.
	 */
	io_put(rs);
}

/* Set start of recovery state. */
static void set_start_recovery(struct raid_set *rs)
{
	/* Initialize recovery. */
	rs->recover.start_jiffies = jiffies;
	rs->recover.end_jiffies = 0;
}

/* Set end of recovery state. */
static void set_end_recovery(struct raid_set *rs)
{
	ClearRSRecover(rs);
	rs->set.dev_to_init = -1;

	/* Check for jiffies overrun. */
	rs->recover.end_jiffies = jiffies;
	if (rs->recover.end_jiffies < rs->recover.start_jiffies)
		rs->recover.end_jiffies = ~0;
}

/* Handle recovery on one recovery stripe. */
static int _do_recovery(struct stripe *stripe)
{
	int r;
	struct raid_set *rs = RS(stripe->sc);
	struct recover_addr *addr = stripe->recover;

	/* If recovery is active -> return. */
	if (stripe_io_ref(stripe))
		return 1;

	/* IO error is fatal for recovery -> stop it. */
	if (unlikely(StripeError(stripe)))
		goto err;

	/* Recovery end required. */
	if (!RSRecover(rs))
		goto err;

	/* Get a region to recover. */
	r = stripe_recover_get_region(stripe);
	switch (r) {
	case 0:	/* Got a new region: flag initial read before write. */
		SetStripeRBW(stripe);
	case 1:	/* Have a region in the works. */
		break;
	case -EAGAIN:
		/* No bandwidth/quiesced region yet, try later. */
		if (!io_ref(rs))
			wake_do_raid_delayed(rs, HZ / 4);
	case -EPERM:
		/* Suspend. */
		return 1;
	case -ENOENT:	/* No more regions to recover. */
		schedule_work(&rs->io.ws_do_table_event);
		return 0;
	default:
		BUG();
	}

	/* Read/write a recover stripe. */
	r = stripe_recover_rw(stripe);
	if (r)
		/* IO initiated. */
		return 1;

	/* Read and write finished-> update recovery position within region. */
	addr->pos += stripe->io.size;

	/* If we're at end of region, update region hash. */
	if (addr->pos >= addr->end ||
	    addr->pos >= rs->set.sectors_per_dev)
		recover_rh_update(stripe, REC_SUCCESS);
	else
		/* Prepare to read next region segment. */
		SetStripeRBW(stripe);

	/* Schedule myself for another round... */
	wake_do_raid(rs);
	return 1;

err:
	/* FIXME: rather try recovering other regions on error? */
	rs_check_degrade(stripe);
	recover_rh_update(stripe, REC_FAILURE);

	/* Check state of partially recovered array. */
	if (RSDegraded(rs) && !RSDead(rs) &&
	    rs->set.dev_to_init != -1 &&
	    rs->set.ei != rs->set.dev_to_init)
		/* Broken drive != drive to recover -> FATAL. */
		SetRSDead(rs);

	if (StripeError(stripe)) {
		char buf[BDEVNAME_SIZE];

		DMERR("stopping recovery due to "
		      "ERROR on /dev/%s, stripe at offset %llu",
		      bdevname(rs->dev[rs->set.ei].dev->bdev, buf),
		      (unsigned long long) stripe->key);

	}

	/* Make sure, that all quiesced regions get released. */
	while (addr->reg) {
		dm_rh_recovery_end(addr->reg, -EIO);
		addr->reg = dm_rh_recovery_start(rs->recover.rh);
	}

	return 0;
}

/* Called by main io daemon to recover regions. */
static void do_recovery(struct raid_set *rs)
{
	if (RSRecover(rs)) {
		int r = 0;
		struct stripe *stripe;

		list_for_each_entry(stripe, &rs->recover.stripes,
				    lists[LIST_RECOVER])
			r += _do_recovery(stripe);

		if (!r) {
			set_end_recovery(rs);
			stripe_recover_free(rs);
		}
	}
}

/*
 * END recovery functions
 */

/* End io process all stripes handed in by endio() callback. */
static void _do_endios(struct raid_set *rs, struct stripe *stripe,
		       struct list_head *flush_list)
{
	/* First unlock all required chunks. */
	stripe_chunks_unlock(stripe);

	/*
	 * If an io error on a stripe occured, degrade the RAID set
	 * and try to endio as many bios as possible. If any bios can't
	 * be endio processed, requeue the stripe (stripe_ref() != 0).
	 */
	if (TestClearStripeError(stripe)) {
		/*
		 * FIXME: if read, rewrite the failed chunk after reconstruction
		 *        in order to trigger disk bad sector relocation.
		 */
		rs_check_degrade(stripe); /* Resets ChunkError(). */
		ClearStripeReconstruct(stripe);
		ClearStripeReconstructed(stripe);
	}

	/* Got to reconstruct a missing chunk. */
	if (StripeReconstruct(stripe)) {
		/*
		 * (*2*) We use StripeReconstruct() to allow for
		 *	 all chunks to be xored into the reconstructed
		 *	 one (see chunk_must_xor()).
		 */
		stripe_reconstruct(stripe);

		/*
		 * (*3*) Now we reset StripeReconstruct() and flag
		 * 	 StripeReconstructed() to show to stripe_rw(),
		 * 	 that we have reconstructed a missing chunk.
		 */
		ClearStripeReconstruct(stripe);
		SetStripeReconstructed(stripe);

		/* FIXME: reschedule to be written in case of read. */
		// if (!StripeRBW(stripe)) {
		// 	chunk_set(CHUNK(stripe, pr), DIRTY);
		// 	stripe_chunks_rw(stripe);
		// }
	}

	/*
	 * Now that we eventually got a complete stripe, we
	 * can process the rest of the end ios on reads.
	 */
	stripe_endio(READ, stripe);

	/* End io all merged writes. */
	if (TestClearStripeMerged(stripe))
		stripe_endio(WRITE_MERGED, stripe);

	/* If RAID set is dead -> fail any ios to dead drives. */
	if (RSDead(rs)) {
		DMERR_LIMIT("RAID set dead: failing ios to dead devices");
		stripe_fail_io(stripe);
	}

	/*
	 * We have stripe references still,
	 * beacuse of read befeore writes or IO errors ->
	 * got to put on flush list for processing.
	 */
	if (stripe_ref(stripe)) {
		BUG_ON(!list_empty(stripe->lists + LIST_LRU));
		list_add_tail(stripe->lists + LIST_FLUSH, flush_list);
		atomic_inc(rs->stats + S_REQUEUE); /* REMOVEME: statistics. */
	} else
		stripe_lru_add(stripe);
}

/* Pop any endio stripes off of the endio list and belabour them. */
static void do_endios(struct raid_set *rs)
{
	struct stripe_cache *sc = &rs->sc;
	struct stripe *stripe;
	/* IO flush list for sorted requeued stripes. */
	struct list_head flush_list;

	INIT_LIST_HEAD(&flush_list);

	while ((stripe = stripe_endio_pop(sc))) {
		/* Avoid endio on stripes with newly io'ed chunks. */
		if (!stripe_io_ref(stripe))
			_do_endios(rs, stripe, &flush_list);
	}

	/*
	 * Insert any requeued stripes in the proper
	 * order at the beginning of the io (flush) list.
	 */
	list_splice(&flush_list, sc->lists + LIST_FLUSH);
}

/* Flush any stripes on the io list. */
static void do_flush(struct raid_set *rs)
{
	struct stripe *stripe;

	while ((stripe = stripe_io_pop(&rs->sc)))
		stripe_rw(stripe); /* Read/write stripe. */
}

/* Stripe cache resizing. */
static void do_sc_resize(struct raid_set *rs)
{
	unsigned set = atomic_read(&rs->sc.stripes_to_set);

	if (set) {
		unsigned cur = atomic_read(&rs->sc.stripes);
		int r = (set > cur) ? sc_grow(&rs->sc, set - cur, SC_GROW) :
				      sc_shrink(&rs->sc, cur - set);

		/* Flag end of resizeing if ok. */
		if (!r)
			atomic_set(&rs->sc.stripes_to_set, 0);
	}
}

/*
 * Process all ios
 *
 * We do different things with the io depending
 * on the state of the region that it is in:
 *
 * o reads: hang off stripe cache or postpone if full
 *
 * o writes:
 *
 *  CLEAN/DIRTY/NOSYNC:	increment pending and hang io off stripe's stripe set.
 *			In case stripe cache is full or busy, postpone the io.
 *
 *  RECOVERING:		delay the io until recovery of the region completes.
 *
 */
static void do_ios(struct raid_set *rs, struct bio_list *ios)
{
	int r;
	unsigned flush = 0, delay = 0;
	sector_t sector;
	struct dm_rh_client *rh = rs->recover.rh;
	struct bio *bio;
	struct bio_list reject;

	bio_list_init(&reject);

	/*
	 * Classify each io:
	 *    o delay writes to recovering regions (let reads go through)
	 *    o queue io to all other regions
	 */
	while ((bio = bio_list_pop(ios))) {
		/*
		 * In case we get a barrier bio, push it back onto
		 * the input queue unless all work queues are empty
		 * and the stripe cache is inactive.
		 */
		if (unlikely(bio_rw_flagged(bio, BIO_RW_BARRIER))) {
			/* REMOVEME: statistics. */
			atomic_inc(rs->stats + S_BARRIER);
			if (delay ||
			    !list_empty(rs->sc.lists + LIST_FLUSH) ||
			    !bio_list_empty(&reject) ||
			    sc_active(&rs->sc)) {
				bio_list_push(ios, bio);
				break;
			}
		}

		/* Check for recovering regions. */
		sector = _sector(rs, bio);
		r = region_state(rs, sector, DM_RH_RECOVERING);
		if (unlikely(r && bio_data_dir(bio) == WRITE)) {
			delay++;
			/* Wait writing to recovering regions. */
			dm_rh_delay_by_region(rh, bio,
					      dm_rh_sector_to_region(rh,
								     sector));
			/* REMOVEME: statistics.*/
			atomic_inc(rs->stats + S_DELAYED_BIOS);
			atomic_inc(rs->stats + S_SUM_DELAYED_BIOS);

			/* Force bandwidth tests in recovery. */
			SetRSBandwidth(rs);
		} else {
			/*
			 * Process ios to non-recovering regions by queueing
			 * them to stripes (does dm_rh_inc()) for writes).
			 */
			flush += stripe_queue_bio(rs, bio, &reject);
		}
	}

	if (flush) {
		/* FIXME: better error handling. */
		r = dm_rh_flush(rh); /* Writes got queued -> flush dirty log. */
		if (r)
			DMERR_LIMIT("dirty log flush");
	}

	/* Merge any rejected bios back to the head of the input list. */
	bio_list_merge_head(ios, &reject);
}

/* Unplug: let any queued io role on the sets devices. */
static void do_unplug(struct raid_set *rs)
{
	struct raid_dev *dev = rs->dev + rs->set.raid_devs;

	while (dev-- > rs->dev) {
		/* Only call any device unplug function, if io got queued. */
		if (TestClearDevIoQueued(dev))
			blk_unplug(bdev_get_queue(dev->dev->bdev));
	}
}

/* Send an event in case we're getting too busy. */
static void do_busy_event(struct raid_set *rs)
{
	if (sc_busy(rs)) {
		if (!TestSetRSScBusy(rs))
			schedule_work(&rs->io.ws_do_table_event);
	}

	ClearRSScBusy(rs);
}

/* Throw an event. */
static void do_table_event(struct work_struct *ws)
{
	struct raid_set *rs = container_of(ws, struct raid_set,
					   io.ws_do_table_event);
	dm_table_event(rs->ti->table);
}


/*-----------------------------------------------------------------
 * RAID daemon
 *---------------------------------------------------------------*/
/*
 * o belabour all end ios
 * o update the region hash states
 * o optionally shrink the stripe cache
 * o optionally do recovery
 * o unplug any component raid devices with queued bios
 * o grab the input queue
 * o work an all requeued or new ios and perform stripe cache flushs
 * o unplug any component raid devices with queued bios
 * o check, if the stripe cache gets too busy and throw an event if so
 */
static void do_raid(struct work_struct *ws)
{
	struct raid_set *rs = container_of(ws, struct raid_set,
					   io.dws_do_raid.work);
	struct bio_list *ios = &rs->io.work, *ios_in = &rs->io.in;

	/*
	 * We always need to end io, so that ios can get errored in
	 * case the set failed and the region counters get decremented
	 * before we update region hash states and go any further.
	 */
	do_endios(rs);
	dm_rh_update_states(rs->recover.rh, 1);

	/*
	 * Now that we've end io'd, which may have put stripes on the LRU list
	 * to allow for shrinking, we resize the stripe cache if requested.
	 */
	do_sc_resize(rs);

	/* Try to recover regions. */
	do_recovery(rs);
	do_unplug(rs);		/* Unplug the sets device queues. */

	/* Quickly grab all new ios queued and add them to the work list. */
	mutex_lock(&rs->io.in_lock);
	bio_list_merge(ios, ios_in);
	bio_list_init(ios_in);
	mutex_unlock(&rs->io.in_lock);

	if (!bio_list_empty(ios))
		do_ios(rs, ios); /* Got ios to work into the cache. */

	do_flush(rs);		/* Flush any stripes on io list. */
	do_unplug(rs);		/* Unplug the sets device queues. */
	do_busy_event(rs);	/* Check if we got too busy. */
}

/*
 * Callback for region hash to dispatch
 * delayed bios queued to recovered regions
 * (gets called via dm_rh_update_states()).
 */
static void dispatch_delayed_bios(void *context, struct bio_list *bl)
{
	struct raid_set *rs = context;
	struct bio *bio;

	/* REMOVEME: statistics; decrement pending delayed bios counter. */
	bio_list_for_each(bio, bl)
		atomic_dec(rs->stats + S_DELAYED_BIOS);

	/* Merge region hash private list to work list. */
	bio_list_merge_head(&rs->io.work, bl);
	bio_list_init(bl);
	ClearRSBandwidth(rs);
}

/*************************************************************
 * Constructor helpers
 *************************************************************/
/* Calculate MB/sec. */
static unsigned mbpers(struct raid_set *rs, unsigned speed)
{
	return to_bytes(speed * rs->set.data_devs *
			rs->recover.io_size * HZ >> 10) >> 10;
}

/*
 * Discover fastest xor algorithm and # of chunks combination.
 */
/* Calculate speed for algorithm and # of chunks. */
static unsigned xor_speed(struct stripe *stripe)
{
	unsigned r = 0;
	unsigned long j;

	/* Wait for next tick. */
	for (j = jiffies; j == jiffies; )
		;

	/* Do xors for a full tick. */
	for (j = jiffies; j == jiffies; ) {
		mb();
		common_xor(stripe, stripe->io.size, 0, 0);
		mb();
		r++;
	}

	return r;
}

/* Optimize xor algorithm for this RAID set. */
static unsigned xor_optimize(struct raid_set *rs)
{
	unsigned chunks_max = 2, p = rs->set.raid_devs, speed_max = 0;
	struct xor_func *f = ARRAY_END(xor_funcs), *f_max = NULL;
	struct stripe *stripe;

	BUG_ON(list_empty(&rs->recover.stripes));
	stripe = list_first_entry(&rs->recover.stripes, struct stripe,
				  lists[LIST_RECOVER]);

	/* Must set uptodate so that xor() will belabour chunks. */
	while (p--)
		SetChunkUptodate(CHUNK(stripe, p));

	/* Try all xor functions. */
	while (f-- > xor_funcs) {
		unsigned speed;

		/* Set actual xor function for common_xor(). */
		rs->xor.f = f;
		rs->xor.chunks = (f->f == xor_blocks_wrapper ?
				  (MAX_XOR_BLOCKS + 1) : XOR_CHUNKS_MAX) + 1;

		while (rs->xor.chunks-- > 2) {
			speed = xor_speed(stripe);
			if (speed > speed_max) {
				speed_max = speed;
				chunks_max = rs->xor.chunks;
				f_max = f;
			}
		}
	}

	/* Memorize optimum parameters. */
	rs->xor.f = f_max;
	rs->xor.chunks = chunks_max;
	return speed_max;
}

/*
 * Allocate a RAID context (a RAID set)
 */
/* Structure for variable RAID parameters. */
struct variable_parms {
	int bandwidth;
	int bandwidth_parm;
	int chunk_size;
	int chunk_size_parm;
	int io_size;
	int io_size_parm;
	int stripes;
	int stripes_parm;
	int recover_io_size;
	int recover_io_size_parm;
	int raid_parms;
	int recovery;
	int recovery_stripes;
	int recovery_stripes_parm;
};

static struct raid_set *
context_alloc(struct raid_type *raid_type, struct variable_parms *p,
	      unsigned raid_devs, sector_t sectors_per_dev,
	      struct dm_target *ti, unsigned dl_parms, char **argv)
{
	int r;
	size_t len;
	sector_t region_size, ti_len;
	struct raid_set *rs = NULL;
	struct dm_dirty_log *dl;
	struct recover *rec;

	/*
	 * Create the dirty log
	 *
	 * We need to change length for the dirty log constructor,
	 * because we want an amount of regions for all stripes derived
	 * from the single device size, so that we can keep region
	 * size = 2^^n independant of the number of devices
	 */
	ti_len = ti->len;
	ti->len = sectors_per_dev;
	dl = dm_dirty_log_create(argv[0], ti, dl_parms, argv + 2);
	ti->len = ti_len;
	if (!dl)
		goto bad_dirty_log;

	/* Chunk size *must* be smaller than region size. */
	region_size = dl->type->get_region_size(dl);
	if (p->chunk_size > region_size)
		goto bad_chunk_size;

	/* Recover io size *must* be smaller than region size as well. */
	if (p->recover_io_size > region_size)
		goto bad_recover_io_size;

	/* Size and allocate the RAID set structure. */
	len = sizeof(*rs->data) + sizeof(*rs->dev);
	if (dm_array_too_big(sizeof(*rs), len, raid_devs))
		goto bad_array;

	len = sizeof(*rs) + raid_devs * len;
	rs = kzalloc(len, GFP_KERNEL);
	if (!rs)
		goto bad_alloc;

	rec = &rs->recover;
	atomic_set(&rs->io.in_process, 0);
	atomic_set(&rs->io.in_process_max, 0);
	rec->io_size = p->recover_io_size;

	/* Pointer to data array. */
	rs->data = (unsigned long **)
		   ((void *) rs->dev + raid_devs * sizeof(*rs->dev));
	rec->dl = dl;
	rs->set.raid_devs = raid_devs;
	rs->set.data_devs = raid_devs - raid_type->parity_devs;
	rs->set.raid_type = raid_type;

	rs->set.raid_parms = p->raid_parms;
	rs->set.chunk_size_parm = p->chunk_size_parm;
	rs->set.io_size_parm = p->io_size_parm;
	rs->sc.stripes_parm = p->stripes_parm;
	rec->io_size_parm = p->recover_io_size_parm;
	rec->bandwidth_parm = p->bandwidth_parm;
	rec->recovery = p->recovery;
	rec->recovery_stripes = p->recovery_stripes;

	/*
	 * Set chunk and io size and respective shifts
	 * (used to avoid divisions)
	 */
	rs->set.chunk_size = p->chunk_size;
	rs->set.chunk_shift = ffs(p->chunk_size) - 1;

	rs->set.io_size = p->io_size;
	rs->set.io_mask = p->io_size - 1;
	/* Mask to adjust address key in case io_size != chunk_size. */
	rs->set.io_inv_mask = (p->chunk_size - 1) & ~rs->set.io_mask;

	rs->set.sectors_per_dev = sectors_per_dev;

	rs->set.ei = -1;	/* Indicate no failed device. */
	atomic_set(&rs->set.failed_devs, 0);

	rs->ti = ti;

	atomic_set(rec->io_count + IO_WORK, 0);
	atomic_set(rec->io_count + IO_RECOVER, 0);

	/* Initialize io lock and queues. */
	mutex_init(&rs->io.in_lock);
	bio_list_init(&rs->io.in);
	bio_list_init(&rs->io.work);

	init_waitqueue_head(&rs->io.suspendq);	/* Suspend waiters (dm-io). */

	rec->nr_regions = dm_sector_div_up(sectors_per_dev, region_size);
	rec->rh = dm_region_hash_create(rs, dispatch_delayed_bios,
			wake_dummy, wake_do_raid, 0, p->recovery_stripes,
			dl, region_size, rec->nr_regions);
	if (IS_ERR(rec->rh))
		goto bad_rh;

	/* Initialize stripe cache. */
	r = sc_init(rs, p->stripes);
	if (r)
		goto bad_sc;

	/* REMOVEME: statistics. */
	stats_reset(rs);
	ClearRSDevelStats(rs);	/* Disnable development status. */
	return rs;

bad_dirty_log:
	TI_ERR_RET("Error creating dirty log", ERR_PTR(-ENOMEM));

bad_chunk_size:
	dm_dirty_log_destroy(dl);
	TI_ERR_RET("Chunk size larger than region size", ERR_PTR(-EINVAL));

bad_recover_io_size:
	dm_dirty_log_destroy(dl);
	TI_ERR_RET("Recover stripe io size larger than region size",
			ERR_PTR(-EINVAL));

bad_array:
	dm_dirty_log_destroy(dl);
	TI_ERR_RET("Arry too big", ERR_PTR(-EINVAL));

bad_alloc:
	dm_dirty_log_destroy(dl);
	TI_ERR_RET("Cannot allocate raid context", ERR_PTR(-ENOMEM));

bad_rh:
	dm_dirty_log_destroy(dl);
	ti->error = DM_MSG_PREFIX "Error creating dirty region hash";
	goto free_rs;

bad_sc:
	dm_region_hash_destroy(rec->rh); /* Destroys dirty log too. */
	sc_exit(&rs->sc);
	ti->error = DM_MSG_PREFIX "Error creating stripe cache";
free_rs:
	kfree(rs);
	return ERR_PTR(-ENOMEM);
}

/* Free a RAID context (a RAID set). */
static void context_free(struct raid_set *rs, unsigned p)
{
	while (p--)
		dm_put_device(rs->ti, rs->dev[p].dev);

	sc_exit(&rs->sc);
	dm_region_hash_destroy(rs->recover.rh); /* Destroys dirty log too. */
	kfree(rs);
}

/* Create work queue and initialize delayed work. */
static int rs_workqueue_init(struct raid_set *rs)
{
	struct dm_target *ti = rs->ti;

	rs->io.wq = create_singlethread_workqueue(DAEMON);
	if (!rs->io.wq)
		TI_ERR_RET("failed to create " DAEMON, -ENOMEM);

	INIT_DELAYED_WORK(&rs->io.dws_do_raid, do_raid);
	INIT_WORK(&rs->io.ws_do_table_event, do_table_event);
	return 0;
}

/* Return pointer to raid_type structure for raid name. */
static struct raid_type *get_raid_type(char *name)
{
	struct raid_type *r = ARRAY_END(raid_types);

	while (r-- > raid_types) {
		if (!strcmp(r->name, name))
			return r;
	}

	return NULL;
}

/* FIXME: factor out to dm core. */
static int multiple(sector_t a, sector_t b, sector_t *n)
{
	sector_t r = a;

	sector_div(r, b);
	*n = r;
	return a == r * b;
}

/* Log RAID set information to kernel log. */
static void rs_log(struct raid_set *rs, unsigned speed)
{
	unsigned p;
	char buf[BDEVNAME_SIZE];

	for (p = 0; p < rs->set.raid_devs; p++)
		DMINFO("/dev/%s is raid disk %u%s",
				bdevname(rs->dev[p].dev->bdev, buf), p,
				(p == rs->set.pi) ? " (parity)" : "");

	DMINFO("%d/%d/%d sectors chunk/io/recovery size, %u stripes\n"
	       "algorithm \"%s\", %u chunks with %uMB/s\n"
	       "%s set with net %u/%u devices",
	       rs->set.chunk_size, rs->set.io_size, rs->recover.io_size,
	       atomic_read(&rs->sc.stripes),
	       rs->xor.f->name, rs->xor.chunks, mbpers(rs, speed),
	       rs->set.raid_type->descr, rs->set.data_devs, rs->set.raid_devs);
}

/* Get all devices and offsets. */
static int dev_parms(struct raid_set *rs, char **argv, int *p)
{
	struct dm_target *ti = rs->ti;

	for (*p = 0; *p < rs->set.raid_devs; (*p)++, argv += 2) {
		int r;
		unsigned long long tmp;
		struct raid_dev *dev = rs->dev + *p;

		/* Get offset and device. */
		if (sscanf(argv[1], "%llu", &tmp) != 1 ||
		    tmp > rs->set.sectors_per_dev)
			TI_ERR("Invalid RAID device offset parameter");

		dev->start = tmp;
		r = dm_get_device(ti, *argv, dev->start,
				  rs->set.sectors_per_dev,
				  dm_table_get_mode(ti->table), &dev->dev);
		if (r)
			TI_ERR_RET("RAID device lookup failure", r);

		r = raid_dev_lookup(rs, dev);
		if (r != -ENODEV && r < *p) {
			(*p)++;	/* Ensure dm_put_device() on actual device. */
			TI_ERR_RET("Duplicate RAID device", -ENXIO);
		}
	}

	return 0;
}

/* Set recovery bandwidth. */
static void
recover_set_bandwidth(struct raid_set *rs, unsigned bandwidth)
{
	rs->recover.bandwidth = bandwidth;
	rs->recover.bandwidth_work = 100 / bandwidth;
}

/* Handle variable number of RAID parameters. */
static int get_raid_variable_parms(struct dm_target *ti, char **argv, 
				   struct variable_parms *vp)
{
	int p, value;
	struct {
		int action; /* -1: skip, 0: no pwer2 check, 1: power2 check */
		char *errmsg;
		int min, max;
		int *var, *var2, *var3;
	} argctr[] = {
		{ 1,
		  "Invalid chunk size; must be -1 or 2^^n and <= 16384",
 		  IO_SIZE_MIN, CHUNK_SIZE_MAX,
		  &vp->chunk_size_parm, &vp->chunk_size, &vp->io_size },
		{ 0,
		  "Invalid number of stripes: must be -1 or >= 8 and <= 16384",
		  STRIPES_MIN, STRIPES_MAX,
		  &vp->stripes_parm, &vp->stripes, NULL },
		{ 1,
		  "Invalid io size; must -1 or >= 8, 2^^n and less equal "
		  "min(BIO_MAX_SECTORS/2, chunk size)",
		  IO_SIZE_MIN, 0, /* Needs to be updated in loop below. */
		  &vp->io_size_parm, &vp->io_size, NULL },
		{ 1,
		  "Invalid recovery io size; must be -1 or "
		  "2^^n and less equal BIO_MAX_SECTORS/2",
		  RECOVER_IO_SIZE_MIN, BIO_MAX_SECTORS / 2,
		  &vp->recover_io_size_parm, &vp->recover_io_size, NULL },
		{ 0,
		  "Invalid recovery bandwidth percentage; "
		  "must be -1 or > 0 and <= 100",
		  BANDWIDTH_MIN, BANDWIDTH_MAX,
		  &vp->bandwidth_parm, &vp->bandwidth, NULL },
		/* Handle sync argument seperately in loop. */
		{ -1,
		  "Invalid recovery switch; must be \"sync\" or \"nosync\"" },
		{ 0,
		  "Invalid number of recovery stripes;"
		  "must be -1, > 0 and <= 16384",
		  RECOVERY_STRIPES_MIN, RECOVERY_STRIPES_MAX,
		  &vp->recovery_stripes_parm, &vp->recovery_stripes, NULL },
	}, *varp;

	/* Fetch # of variable raid parameters. */
	if (sscanf(*(argv++), "%d", &vp->raid_parms) != 1 ||
	    !range_ok(vp->raid_parms, 0, 7))
		TI_ERR("Bad variable raid parameters number");

	/* Preset variable RAID parameters. */
	vp->chunk_size = CHUNK_SIZE_DEFAULT;
	vp->io_size = IO_SIZE_DEFAULT;
	vp->stripes = STRIPES_DEFAULT;
	vp->recover_io_size = RECOVER_IO_SIZE_DEFAULT;
	vp->bandwidth = BANDWIDTH_DEFAULT;
	vp->recovery = 1;
	vp->recovery_stripes = RECOVERY_STRIPES_DEFAULT;

	/* Walk the array of argument constraints for all given ones. */
	for (p = 0, varp = argctr; p < vp->raid_parms; p++, varp++) {
	     	BUG_ON(varp >= ARRAY_END(argctr));

		/* Special case for "[no]sync" string argument. */
		if (varp->action < 0) {
			if (!strcmp(*argv, "sync"))
				;
			else if (!strcmp(*argv, "nosync"))
				vp->recovery = 0;
			else
				TI_ERR(varp->errmsg);

			argv++;
			continue;
		}

		/*
		 * Special case for io_size depending
		 * on previously set chunk size.
		 */
		if (p == 2)
			varp->max = min(BIO_MAX_SECTORS / 2, vp->chunk_size);

		if (sscanf(*(argv++), "%d", &value) != 1 ||
		    (value != -1 &&
		     ((varp->action && !POWER_OF_2(value)) ||
		      !range_ok(value, varp->min, varp->max))))
			TI_ERR(varp->errmsg);

		*varp->var = value;
		if (value != -1) {
			if (varp->var2)
				*varp->var2 = value;
			if (varp->var3)
				*varp->var3 = value;
		}
	}

	return 0;
}

/* Parse optional locking parameters. */
static int get_raid_locking_parms(struct dm_target *ti, char **argv,
				  int *locking_parms,
				  struct dm_raid45_locking_type **locking_type)
{
	if (!strnicmp(argv[0], "locking", strlen(argv[0]))) {
		char *lckstr = argv[1];
		size_t lcksz = strlen(lckstr);

		if (!strnicmp(lckstr, "none", lcksz)) {
			*locking_type = &locking_none;
			*locking_parms = 2;
		} else if (!strnicmp(lckstr, "cluster", lcksz)) {
			DMERR("locking type \"%s\" not yet implemented",
			      lckstr);
			return -EINVAL;
		} else {
			DMERR("unknown locking type \"%s\"", lckstr);
			return -EINVAL;
		}
	}

	*locking_parms = 0;
	*locking_type = &locking_none;
	return 0;
}

/* Set backing device read ahead properties of RAID set. */
static void rs_set_read_ahead(struct raid_set *rs,
			      unsigned sectors, unsigned stripes)
{
	unsigned ra_pages = dm_div_up(sectors, SECTORS_PER_PAGE);
	struct mapped_device *md = dm_table_get_md(rs->ti->table);
	struct backing_dev_info *bdi = &dm_disk(md)->queue->backing_dev_info;

	/* Set read-ahead for the RAID set and the component devices. */
	if (ra_pages) {
		unsigned p = rs->set.raid_devs;

		bdi->ra_pages = stripes * ra_pages * rs->set.data_devs;

		while (p--) {
			struct request_queue *q =
				bdev_get_queue(rs->dev[p].dev->bdev);

			q->backing_dev_info.ra_pages = ra_pages;
		}
	}

	dm_put(md);
}

/* Set congested function. */
static void rs_set_congested_fn(struct raid_set *rs)
{
	struct mapped_device *md = dm_table_get_md(rs->ti->table);
	struct backing_dev_info *bdi = &dm_disk(md)->queue->backing_dev_info;

	/* Set congested function and data. */
	bdi->congested_fn = rs_congested;
	bdi->congested_data = rs;
	dm_put(md);
}

/*
 * Construct a RAID4/5 mapping:
 *
 * log_type #log_params <log_params> \
 * raid_type [#parity_dev] #raid_variable_params <raid_params> \
 * [locking "none"/"cluster"]
 * #raid_devs #dev_to_initialize [<dev_path> <offset>]{3,}
 *
 * log_type = "core"/"disk",
 * #log_params = 1-3 (1-2 for core dirty log type, 3 for disk dirty log only)
 * log_params = [dirty_log_path] region_size [[no]sync])
 *
 * raid_type = "raid4", "raid5_la", "raid5_ra", "raid5_ls", "raid5_rs"
 *
 * #parity_dev = N if raid_type = "raid4"
 * o N = -1: pick default = last device
 * o N >= 0 and < #raid_devs: parity device index
 *
 * #raid_variable_params = 0-7; raid_params (-1 = default):
 *   [chunk_size [#stripes [io_size [recover_io_size \
 *    [%recovery_bandwidth [recovery_switch [#recovery_stripes]]]]]]]
 *   o chunk_size (unit to calculate drive addresses; must be 2^^n, > 8
 *     and <= CHUNK_SIZE_MAX)
 *   o #stripes is number of stripes allocated to stripe cache
 *     (must be > 1 and < STRIPES_MAX)
 *   o io_size (io unit size per device in sectors; must be 2^^n and > 8)
 *   o recover_io_size (io unit size per device for recovery in sectors;
 must be 2^^n, > SECTORS_PER_PAGE and <= region_size)
 *   o %recovery_bandwith is the maximum amount spend for recovery during
 *     application io (1-100%)
 *   o recovery switch = [sync|nosync]
 *   o #recovery_stripes is the number of recovery stripes used for
 *     parallel recovery of the RAID set
 * If raid_variable_params = 0, defaults will be used.
 * Any raid_variable_param can be set to -1 to apply a default
 *
 * #raid_devs = N (N >= 3)
 *
 * #dev_to_initialize = N
 * -1: initialize parity on all devices
 * >= 0 and < #raid_devs: initialize raid_path; used to force reconstruction
 * of a failed devices content after replacement
 *
 * <dev_path> = device_path (eg, /dev/sdd1)
 * <offset>   = begin at offset on <dev_path>
 *
 */
#define	MIN_PARMS	13
static int raid_ctr(struct dm_target *ti, unsigned argc, char **argv)
{
	int dev_to_init, dl_parms, i, locking_parms,
	    parity_parm, pi = -1, r, raid_devs;
	unsigned speed;
	sector_t tmp, sectors_per_dev;
	struct dm_raid45_locking_type *locking;
	struct raid_set *rs;
	struct raid_type *raid_type;
	struct variable_parms parms;

	/* Ensure minimum number of parameters. */
	if (argc < MIN_PARMS)
		TI_ERR("Not enough parameters");

	/* Fetch # of dirty log parameters. */
	if (sscanf(argv[1], "%d", &dl_parms) != 1 ||
	    !range_ok(dl_parms, 1, 4711)) /* ;-) */
		TI_ERR("Bad dirty log parameters number");

	/* Check raid_type. */
	raid_type = get_raid_type(argv[dl_parms + 2]);
	if (!raid_type)
		TI_ERR("Bad raid type");

	/* In case of RAID4, parity drive is selectable. */
	parity_parm = !!(raid_type->level == raid4);

	/* Handle variable number of RAID parameters. */
	r = get_raid_variable_parms(ti, argv + dl_parms + parity_parm + 3,
				    &parms);
	if (r)
		return r;

	/* Handle any locking parameters. */
	r = get_raid_locking_parms(ti,
				   argv + dl_parms + parity_parm +
				   parms.raid_parms + 4,
				   &locking_parms, &locking);
	if (r)
		return r;

	/* # of raid devices. */
	i = dl_parms + parity_parm + parms.raid_parms + locking_parms + 4;
	if (sscanf(argv[i], "%d", &raid_devs) != 1 ||
	    raid_devs < raid_type->minimal_devs)
		TI_ERR("Invalid number of raid devices");

	/* In case of RAID4, check parity drive index is in limits. */
	if (raid_type->level == raid4) {
		/* Fetch index of parity device. */
		if (sscanf(argv[dl_parms + 3], "%d", &pi) != 1 ||
		    (pi != -1 && !range_ok(pi, 0, raid_devs - 1)))
			TI_ERR("Invalid RAID4 parity device index");
	}

	/*
	 * Index of device to initialize starts at 0
	 *
	 * o -1 -> don't initialize a selected device;
	 *         initialize parity conforming to algorithm
	 * o 0..raid_devs-1 -> initialize respective device
	 *   (used for reconstruction of a replaced device)
	 */
	if (sscanf(argv[dl_parms + parity_parm + parms.raid_parms +
		   locking_parms + 5], "%d", &dev_to_init) != 1 ||
	    !range_ok(dev_to_init, -1, raid_devs - 1))
		TI_ERR("Invalid number for raid device to initialize");

	/* Check # of raid device arguments. */
	if (argc - dl_parms - parity_parm - parms.raid_parms - 6 !=
	    2 * raid_devs)
		TI_ERR("Wrong number of raid device/offset arguments");

	/*
	 * Check that the table length is devisable
	 * w/o rest by (raid_devs - parity_devs)
	 */
	if (!multiple(ti->len, raid_devs - raid_type->parity_devs,
		      &sectors_per_dev))
		TI_ERR("Target length not divisible by number of data devices");

	/*
	 * Check that the device size is
	 * devisable w/o rest by chunk size
	 */
	if (!multiple(sectors_per_dev, parms.chunk_size, &tmp))
		TI_ERR("Device length not divisible by chunk_size");

	/****************************************************************
	 * Now that we checked the constructor arguments ->
	 * let's allocate the RAID set
	 ****************************************************************/
	rs = context_alloc(raid_type, &parms, raid_devs, sectors_per_dev,
			   ti, dl_parms, argv);
	if (IS_ERR(rs))
		return PTR_ERR(rs);


	rs->set.dev_to_init = rs->set.dev_to_init_parm = dev_to_init;
	rs->set.pi = rs->set.pi_parm = pi;

	/* Set RAID4 parity drive index. */
	if (raid_type->level == raid4)
		rs->set.pi = (pi == -1) ? rs->set.data_devs : pi;

	recover_set_bandwidth(rs, parms.bandwidth);

	/* Use locking type to lock stripe access. */
	rs->locking = locking;

	/* Get the device/offset tupels. */
	argv += dl_parms + 6 + parity_parm + parms.raid_parms;
	r = dev_parms(rs, argv, &i);
	if (r)
		goto err;

	/* Set backing device information (eg. read ahead). */
	rs_set_read_ahead(rs, 2 * rs->set.chunk_size, 4 /* stripes */);
	rs_set_congested_fn(rs); /* Set congested function. */
	SetRSCheckOverwrite(rs); /* Allow chunk overwrite checks. */
	speed = xor_optimize(rs); /* Select best xor algorithm. */

	/* Set for recovery of any nosync regions. */
	if (parms.recovery)
		SetRSRecover(rs);
	else {
		/*
		 * Need to free recovery stripe(s) here in case
		 * of nosync, because xor_optimize uses one.
		 */
		set_start_recovery(rs);
		set_end_recovery(rs);
		stripe_recover_free(rs);
	}

	/*
	 * Make sure that dm core only hands maximum io size
	 * length down and pays attention to io boundaries.
	 */
	ti->split_io = rs->set.io_size;
	ti->private = rs;

	/* Initialize work queue to handle this RAID set's io. */
	r = rs_workqueue_init(rs);
	if (r)
		goto err;

	rs_log(rs, speed); /* Log information about RAID set. */
	return 0;

err:
	context_free(rs, i);
	return r;
}

/*
 * Destruct a raid mapping
 */
static void raid_dtr(struct dm_target *ti)
{
	struct raid_set *rs = ti->private;

	destroy_workqueue(rs->io.wq);
	context_free(rs, rs->set.raid_devs);
}

/* Raid mapping function. */
static int raid_map(struct dm_target *ti, struct bio *bio,
		    union map_info *map_context)
{
	/* I don't want to waste stripe cache capacity. */
	if (bio_rw(bio) == READA)
		return -EIO;
	else {
		struct raid_set *rs = ti->private;

		/*
		 * Get io reference to be waiting for to drop
		 * to zero on device suspension/destruction.
		 */
		io_get(rs);
		bio->bi_sector -= ti->begin;	/* Remap sector. */

		/* Queue io to RAID set. */
		mutex_lock(&rs->io.in_lock);
		bio_list_add(&rs->io.in, bio);
		mutex_unlock(&rs->io.in_lock);

		/* Wake daemon to process input list. */
		wake_do_raid(rs);

		/* REMOVEME: statistics. */
		atomic_inc(rs->stats + (bio_data_dir(bio) == READ ?
				        S_BIOS_READ : S_BIOS_WRITE));
		return DM_MAPIO_SUBMITTED;	/* Handle later. */
	}
}

/* Device suspend. */
static void raid_presuspend(struct dm_target *ti)
{
	struct raid_set *rs = ti->private;
	struct dm_dirty_log *dl = rs->recover.dl;

	SetRSSuspend(rs);

	if (RSRecover(rs))
		dm_rh_stop_recovery(rs->recover.rh);

	cancel_delayed_work(&rs->io.dws_do_raid);
	flush_workqueue(rs->io.wq);
	wait_ios(rs);	/* Wait for completion of all ios being processed. */

	if (dl->type->presuspend && dl->type->presuspend(dl))
		/* FIXME: need better error handling. */
		DMWARN("log presuspend failed");
}

static void raid_postsuspend(struct dm_target *ti)
{
	struct raid_set *rs = ti->private;
	struct dm_dirty_log *dl = rs->recover.dl;

	if (dl->type->postsuspend && dl->type->postsuspend(dl))
		/* FIXME: need better error handling. */
		DMWARN("log postsuspend failed");

}

/* Device resume. */
static void raid_resume(struct dm_target *ti)
{
	struct raid_set *rs = ti->private;
	struct recover *rec = &rs->recover;
	struct dm_dirty_log *dl = rec->dl;

	if (dl->type->resume && dl->type->resume(dl))
		/* Resume dirty log. */
		/* FIXME: need better error handling. */
		DMWARN("log resume failed");

	rec->nr_regions_to_recover =
		rec->nr_regions - dl->type->get_sync_count(dl);

	/* Restart any unfinished recovery. */
	if (RSRecover(rs)) {
		set_start_recovery(rs);
		dm_rh_start_recovery(rec->rh);
	}

	ClearRSSuspend(rs);
	wake_do_raid(rs);
}

/* Return stripe cache size. */
static unsigned sc_size(struct raid_set *rs)
{
	return to_sector(atomic_read(&rs->sc.stripes) *
			 (sizeof(struct stripe) +
			  (sizeof(struct stripe_chunk) +
			   (sizeof(struct page_list) +
			    to_bytes(rs->set.io_size) *
			    rs->set.raid_devs)) +
			  (rs->recover.end_jiffies ?
			   0 : rs->recover.recovery_stripes *
			   to_bytes(rs->set.raid_devs * rs->recover.io_size))));
}

/* REMOVEME: status output for development. */
static void raid_devel_stats(struct dm_target *ti, char *result,
			     unsigned *size, unsigned maxlen)
{
	unsigned sz = *size;
	unsigned long j;
	char buf[BDEVNAME_SIZE], *p;
	struct stats_map *sm;
	struct raid_set *rs = ti->private;
	struct recover *rec = &rs->recover;
	struct timespec ts;

	DMEMIT("%s %s %u\n", version, rs->xor.f->name, rs->xor.chunks);
	DMEMIT("act_ios=%d ", io_ref(rs));
	DMEMIT("act_ios_max=%d\n", atomic_read(&rs->io.in_process_max));
	DMEMIT("act_stripes=%d ", sc_active(&rs->sc));
	DMEMIT("act_stripes_max=%d\n",
	       atomic_read(&rs->sc.active_stripes_max));

	for (sm = stats_map; sm < ARRAY_END(stats_map); sm++)
		DMEMIT("%s%d", sm->str, atomic_read(rs->stats + sm->type));

	DMEMIT(" checkovr=%s\n", RSCheckOverwrite(rs) ? "on" : "off");
	DMEMIT("sc=%u/%u/%u/%u/%u/%u/%u\n", rs->set.chunk_size,
	       atomic_read(&rs->sc.stripes), rs->set.io_size,
	       rec->recovery_stripes, rec->io_size, rs->sc.hash.buckets,
	       sc_size(rs));

	j = (rec->end_jiffies ? rec->end_jiffies : jiffies) -
	    rec->start_jiffies;
	jiffies_to_timespec(j, &ts);
	sprintf(buf, "%ld.%ld", ts.tv_sec, ts.tv_nsec);
	p = strchr(buf, '.');
	p[3] = 0;

	DMEMIT("rg=%llu/%llu/%llu/%u %s\n",
	       (unsigned long long) rec->nr_regions_recovered,
	       (unsigned long long) rec->nr_regions_to_recover,
	       (unsigned long long) rec->nr_regions, rec->bandwidth, buf);

	*size = sz;
}

static int raid_status(struct dm_target *ti, status_type_t type,
		       char *result, unsigned maxlen)
{
	unsigned p, sz = 0;
	char buf[BDEVNAME_SIZE];
	struct raid_set *rs = ti->private;
	int raid_parms[] = {
		rs->set.chunk_size_parm,
		rs->sc.stripes_parm,
		rs->set.io_size_parm,
		rs->recover.io_size_parm,
		rs->recover.bandwidth_parm,
		-2,
		rs->recover.recovery_stripes,
	};

	switch (type) {
	case STATUSTYPE_INFO:
		/* REMOVEME: statistics. */
		if (RSDevelStats(rs))
			raid_devel_stats(ti, result, &sz, maxlen);

		DMEMIT("%u ", rs->set.raid_devs);

		for (p = 0; p < rs->set.raid_devs; p++)
			DMEMIT("%s ",
			       format_dev_t(buf, rs->dev[p].dev->bdev->bd_dev));

		DMEMIT("1 ");
		for (p = 0; p < rs->set.raid_devs; p++) {
			DMEMIT("%c", !DevFailed(rs->dev + p) ? 'A' : 'D');

			if (p == rs->set.pi)
				DMEMIT("p");

			if (rs->set.dev_to_init == p)
				DMEMIT("i");
		}

		break;
	case STATUSTYPE_TABLE:
		sz = rs->recover.dl->type->status(rs->recover.dl, type,
						  result, maxlen);
		DMEMIT("%s %u ", rs->set.raid_type->name,
		       rs->set.raid_parms);

		for (p = 0; p < rs->set.raid_parms; p++) {
			if (raid_parms[p] > -2)
				DMEMIT("%d ", raid_parms[p]);
			else
				DMEMIT("%s ", rs->recover.recovery ?
					      "sync" : "nosync");
		}

		DMEMIT("%u %d ", rs->set.raid_devs, rs->set.dev_to_init);

		for (p = 0; p < rs->set.raid_devs; p++)
			DMEMIT("%s %llu ",
			       format_dev_t(buf, rs->dev[p].dev->bdev->bd_dev),
			       (unsigned long long) rs->dev[p].start);
	}

	return 0;
}

/*
 * Message interface
 */
enum raid_msg_actions {
	act_bw,			/* Recovery bandwidth switch. */
	act_dev,		/* Device failure switch. */
	act_overwrite,		/* Stripe overwrite check. */
	act_stats,		/* Development statistics switch. */
	act_sc,			/* Stripe cache switch. */

	act_on,			/* Set entity on. */
	act_off,		/* Set entity off. */
	act_reset,		/* Reset entity. */

	act_set = act_on,	/* Set # absolute. */
	act_grow = act_off,	/* Grow # by an amount. */
	act_shrink = act_reset,	/* Shrink # by an amount. */
};

/* Turn a delta into an absolute value. */
static int _absolute(unsigned long action, int act, int r)
{
	/* Make delta absolute. */
	if (test_bit(act_set, &action))
		;
	else if (test_bit(act_grow, &action))
		r += act;
	else if (test_bit(act_shrink, &action))
		r = act - r;
	else
		r = -EINVAL;

	return r;
}

 /* Change recovery io bandwidth. */
static int bandwidth_change(struct dm_msg *msg, void *context)
{
	struct raid_set *rs = context;
	int act = rs->recover.bandwidth;
	int bandwidth = DM_MSG_INT_ARG(msg);

	if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) {
		/* Make delta bandwidth absolute. */
		bandwidth = _absolute(msg->action, act, bandwidth);

		/* Check range. */
		if (range_ok(bandwidth, BANDWIDTH_MIN, BANDWIDTH_MAX)) {
			recover_set_bandwidth(rs, bandwidth);
			return 0;
		}
	}

	set_bit(dm_msg_ret_arg, &msg->ret);
	set_bit(dm_msg_ret_inval, &msg->ret);
	return -EINVAL;
}

/* Set/reset development feature flags. */
static int devel_flags(struct dm_msg *msg, void *context)
{
	struct raid_set *rs = context;

	if (test_bit(act_on, &msg->action))
		return test_and_set_bit(msg->spec->parm,
					&rs->io.flags) ? -EPERM : 0;
	else if (test_bit(act_off, &msg->action))
		return test_and_clear_bit(msg->spec->parm,
					  &rs->io.flags) ? 0 : -EPERM;
	else if (test_bit(act_reset, &msg->action)) {
		if (test_bit(act_stats, &msg->action)) {
			stats_reset(rs);
			goto on;
		} else if (test_bit(act_overwrite, &msg->action)) {
on:
			set_bit(msg->spec->parm, &rs->io.flags);
			return 0;
		}
	}

	return -EINVAL;
}

/* Resize the stripe cache. */
static int sc_resize(struct dm_msg *msg, void *context)
{
	int act, stripes;
	struct raid_set *rs = context;

	/* Deny permission in case the daemon is still resizing!. */
	if (atomic_read(&rs->sc.stripes_to_set))
		return -EPERM;

	stripes = DM_MSG_INT_ARG(msg);
	if (stripes > 0) {
		act = atomic_read(&rs->sc.stripes);

		/* Make delta stripes absolute. */
		stripes = _absolute(msg->action, act, stripes);

		/*
		 * Check range and that the # of stripes changes.
		 * We leave the resizing to the wroker.
		 */
		if (range_ok(stripes, STRIPES_MIN, STRIPES_MAX) &&
		    stripes != atomic_read(&rs->sc.stripes)) {
			atomic_set(&rs->sc.stripes_to_set, stripes);
			wake_do_raid(rs);
			return 0;
		}
	}

	set_bit(dm_msg_ret_arg, &msg->ret);
	set_bit(dm_msg_ret_inval, &msg->ret);
	return -EINVAL;
}

/* Parse the RAID message action. */
/*
 * 'ba[ndwidth] {se[t],g[row],sh[rink]} #'	# e.g 'ba se 50'
 * "o[verwrite]  {on,of[f],r[eset]}'		# e.g. 'o of'
 * 'sta[tistics] {on,of[f],r[eset]}'		# e.g. 'stat of'
 * 'str[ipecache] {se[t],g[row],sh[rink]} #'	# e.g. 'stripe set 1024'
 *
 */
static int raid_message(struct dm_target *ti, unsigned argc, char **argv)
{
	/* Variables to store the parsed parameters im. */
	static int i[2];
	static unsigned long *i_arg[] = {
		(unsigned long *) i + 0,
		(unsigned long *) i + 1,
	};

	/* Declare all message option strings. */
	static char *str_sgs[] = { "set", "grow", "shrink" };
	static char *str_oor[] = { "on", "off", "reset" };

	/* Declare all actions. */
	static unsigned long act_sgs[] = { act_set, act_grow, act_shrink };
	static unsigned long act_oor[] = { act_on, act_off, act_reset };

	/* Bandwidth option. */
	static struct dm_message_option bw_opt = { 3, str_sgs, act_sgs };
	static struct dm_message_argument bw_args = {
		1, i_arg, { dm_msg_int_t }
	};

	static struct dm_message_argument null_args = {
		0, NULL, { dm_msg_int_t }
	};

	/* Overwrite and statistics option. */
	static struct dm_message_option ovr_stats_opt = { 3, str_oor, act_oor };

	/* Sripecache option. */
	static struct dm_message_option stripe_opt = { 3, str_sgs, act_sgs };

	/* Declare messages. */
	static struct dm_msg_spec specs[] = {
		{ "bandwidth", act_bw, &bw_opt, &bw_args,
		  0, bandwidth_change },
		{ "overwrite", act_overwrite, &ovr_stats_opt, &null_args,
		  RS_CHECK_OVERWRITE, devel_flags },
		{ "statistics", act_stats, &ovr_stats_opt, &null_args,
		  RS_DEVEL_STATS, devel_flags },
		{ "stripecache", act_sc, &stripe_opt, &bw_args,
		  0, sc_resize },
	};

	/* The message for the parser. */
	struct dm_msg msg = {
		.num_specs = ARRAY_SIZE(specs),
		.specs = specs,
	};

	return dm_message_parse(TARGET, &msg, ti->private, argc, argv);
}
/*
 * END message interface
 */

static struct target_type raid_target = {
	.name = "raid45",
	.version = {1, 0, 0},
	.module = THIS_MODULE,
	.ctr = raid_ctr,
	.dtr = raid_dtr,
	.map = raid_map,
	.presuspend = raid_presuspend,
	.postsuspend = raid_postsuspend,
	.resume = raid_resume,
	.status = raid_status,
	.message = raid_message,
};

static void init_exit(const char *bad_msg, const char *good_msg, int r)
{
	if (r)
		DMERR("Failed to %sregister target [%d]", bad_msg, r);
	else
		DMINFO("%s %s", good_msg, version);
}

static int __init dm_raid_init(void)
{
	int r = dm_register_target(&raid_target);

	init_exit("", "initialized", r);
	return r;
}

static void __exit dm_raid_exit(void)
{
	dm_unregister_target(&raid_target);
	init_exit("un", "exit", 0);
}

/* Module hooks. */
module_init(dm_raid_init);
module_exit(dm_raid_exit);

MODULE_DESCRIPTION(DM_NAME " raid4/5 target");
MODULE_AUTHOR("Heinz Mauelshagen <hjm@redhat.com>");
MODULE_LICENSE("GPL");
MODULE_ALIAS("dm-raid4");
MODULE_ALIAS("dm-raid5");