- 2 x 1TB hard drives contributing one partition each to a software RAID 1 array and contributing one swap partition each to the system.
- 2 x 3TB hard drives contributing two partitions each to software RAID 1 Arrays.
The system:
# cat /etc/issue /etc/debian_version Debian GNU/Linux 6.0 \n \l 6.0.7 # uname -r 2.6.32-5-amd64
Install the Multiple Devices MD --Linux Software RAID Array-- Manager
# apt-get install mdadm
Check out the disk used by the System.
# fdisk -l Disk /dev/sda: 1000.2 GB, 1000204886016 bytes 255 heads, 63 sectors/track, 121601 cylinders Units = cylinders of 16065 * 512 = 8225280 bytes Sector size (logical/physical): 512 bytes / 512 bytes I/O size (minimum/optimal): 512 bytes / 512 bytes Disk identifier: 0x000a8440 Device Boot Start End Blocks Id System /dev/sda1 * 1 121133 972993536 83 Linux /dev/sda2 121133 121602 3766273 5 Extended /dev/sda5 121133 121602 3766272 82 Linux swap / Solaris
OK, shut it down and add the other ~ 1TB disk
Look at the disks
# sfdisk -l Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sda1 * 0+ 121132- 121133- 972993536 83 Linux /dev/sda2 121132+ 121601- 469- 3766273 5 Extended /dev/sda3 0 - 0 0 0 Empty /dev/sda4 0 - 0 0 0 Empty /dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track sfdisk: ERROR: sector 0 does not have an msdos signature /dev/sdb: unrecognized partition table type No partitions found
Copy the partition table of /dev/sda to /dev/sdb
# sfdisk -d /dev/sda | sfdisk --force /dev/sdb
List all logical drives on all hard drives
# sfdisk -l Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sda1 * 0+ 121132- 121133- 972993536 83 Linux /dev/sda2 121132+ 121601- 469- 3766273 5 Extended /dev/sda3 0 - 0 0 0 Empty /dev/sda4 0 - 0 0 0 Empty /dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sdb1 * 0+ 121132- 121133- 972993536 83 Linux /dev/sdb2 121132+ 121601- 469- 3766273 5 Extended /dev/sdb3 0 - 0 0 0 Empty /dev/sdb4 0 - 0 0 0 Empty /dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Set /dev/sdb1 0x ID to 0xfd ( Linux RAID Auto )
# sfdisk --change-id /dev/sdb 1 fd
# sfdisk -l /dev/sdb Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sdb1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect /dev/sdb2 121132+ 121601- 469- 3766273 5 Extended /dev/sdb3 0 - 0 0 0 Empty /dev/sdb4 0 - 0 0 0 Empty /dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Overwrite with zeros MD superblocks --remains of RAID Arrays-- if any.
# mdadm --zero-superblock --force /dev/sdb1
Create a RAID 1 Array of 2 logical disks using the v0.90 superblock format
# mdadm --create /dev/md0 --level=1 --metadata=0 --raid-disks=2 missing /dev/sdb1 mdadm: array /dev/md0 started."The version-0.90 superblock limits the number of component devices within an array to 28, and limits each component device to a maximum size of 2TB on kernel version <3.1 and 4TB on kernel version >=3.1." The version-0.90 Superblock Format
Look at proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sdb1[1]
972993472 blocks [2/1] [_U]
unused devices:
Create an ext3 file-system on the Multiple Devices MD device /dev/md0
# mkfs.ext3 -v -m .1 -b 4096 -E stride=32,stripe-width=64 /dev/md0with stripe-width=64 , it should not matter in RAID 1 Arrays.
Options explained:
The first command makes a ext3 filesystem
-v verbose
-m .1 leave .1% of disk to root
-b 4096 block size of 4kb (recommended above for large-file systems)
-E stride=32,stripe-width=64 see below calculation
RAID wiki
Backup mdadm.conf
# cp /etc/mdadm/mdadm.conf /etc/mdadm/mdadm.conf.0Delete ARRAY definitions from /etc/mdadm/mdadm.conf
# grep -v ARRAY /etc/mdadm/mdadm.conf >> /etc/mdadm/mdadm.confUpdate /etc/mdadm/mdadm.conf
# mdadm --examine --scan >> /etc/mdadm/mdadm.conf
Mount md0
# mkdir /mnt/md0 # mount /dev/md0 /mnt/md0/
Save /etc/fstab for historical reasons
# cp /etc/fstab /etc/fstab.0Remove or comment out lines starting with UUID or /dev/sd* from /etc/fstab
and make /etc/fstab look like the following.
# grep -v "#" /etc/fstab proc /proc proc defaults 0 0 /dev/md0 / ext3 errors=remount-ro 0 1 /dev/sda5 swap swap defaults,pri=1 0 0 /dev/sdb5 swap swap defaults,pri=1 0 0Swapping on RAID
Configure the bootloader --Grub2
# cp /etc/grub.d/40_custom /etc/grub.d/08_raid1 # vi /etc/grub.d/08_raid1Here is my /etc/grub.d/08_raid1 :
#!/bin/sh
exec tail -n +3 $0
# This file provides an easy way to add custom menu entries. Simply type the
# menu entries you want to add after this comment. Be careful not to change
# the 'exec tail' line above.
menuentry 'Debian GNU/Linux, with Linux 2.6.32-5-amd64 raid1' --class debian --class gnu-linux --class gnu --class os {
insmod raid
insmod mdraid
insmod part_msdos
insmod ext2
set root='(md0)'
echo 'Loading Linux 2.6.32-5-amd64 ...'
linux /boot/vmlinuz-2.6.32-5-amd64 root=/dev/md0 ro quiet
echo 'Loading initial ramdisk ...'
initrd /boot/initrd.img-2.6.32-5-amd64
}
Un-comment or add
GRUB_DISABLE_LINUX_UUID=trueat /etc/default/grub
Replace /dev/sda with /dev/md0 in /etc/mtab
and delete the line showing /dev/md0 mount on /mnt/md0
Update grub.cfg
# grub-mkconfig -o /boot/grub/grub.cfg
Update the initramfs image
# update-initramfs -u
Copy the System to /dev/md0
# cp -dpRx / /mnt/md0
Install grub on all drives.
# grub-install /dev/sdb # grub-install /dev/sda
Cross your fingers and reboot
# shutdown -r now
OK, the system came up as expected.
Check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sdb1[1]
972993472 blocks [2/1] [_U]
unused devices:
Set /dev/sda1 0x ID to 0xfd ( Linux RAID Auto )
# sfdisk --change-id /dev/sda 1 fd
List all partitions on all drives
# sfdisk -l Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sda1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect /dev/sda2 121132+ 121601- 469- 3766273 5 Extended /dev/sda3 0 - 0 0 0 Empty /dev/sda4 0 - 0 0 0 Empty /dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sdb1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect /dev/sdb2 121132+ 121601- 469- 3766273 5 Extended /dev/sdb3 0 - 0 0 0 Empty /dev/sdb4 0 - 0 0 0 Empty /dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris Disk /dev/md0: 243248368 cylinders, 2 heads, 4 sectors/track sfdisk: ERROR: sector 0 does not have an msdos signature /dev/md0: unrecognized partition table type No partitions found
Add /dev/sda1 to the RAID 1 md0
# mdadm --add /dev/md0 /dev/sda1 mdadm: added /dev/sda1
Check proc mdstat
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[2] sdb1[1]
972993472 blocks [2/1] [_U]
[>....................] recovery = 0.4% (4773696/972993472) finish=144.5min speed=111602K/sec
unused devices:
Check load
22:53:31 up 20 min, 1 user, load average: 0.66, 0.19, 0.06Not (very) bad ;)
Wait for the raid array to sync.
root@bob:~# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[2] sdb1[1]
972993472 blocks [2/1] [_U]
[===========>.........] recovery = 56.1% (546131776/972993472) finish=74.2min speed=95857K/sec
unused devices:
Done.
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0] sdb1[1]
972993472 blocks [2/2] [UU]
unused devices:
Test! What if I take off /dev/sdb ?
# shutdown -h nowtake of /dev/sdb and boot
It came up fine :) , check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0]
972993472 blocks [2/1] [U_]
unused devices:
(
Yet another not-very-cheap way for a technician
to hide 'some' tracks (I think)
in a dedicated server farm.
- shutdown the system
- take off drive a
- do crap
- shutdown the system, take off drive b , put in drive a , boot
- shutdown the system, put in drive b, and let them sync
)
Back to business ...
- shutdown the system
- put in sdb
- and boot
It came up fine, check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0]
972993472 blocks [2/1] [U_]
unused devices:
Add /dev/sdb to the RAID array
# mdadm --add /dev/md0 /dev/sdb1Check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sdb1[2] sda1[0]
972993472 blocks [2/1] [U_]
[>....................] recovery = 3.6% (35851584/972993472) finish=145.8min speed=107124K/sec
unused devices:
OK, Let 's create the "DATA" arrays.
- shutdown the system
- put in the 3TB hard drives
- Set on BIOS to boot only from /dev/sda and /dev/sdb
- boot
OK the system came up.
Check out proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0] sdb1[2]
972993472 blocks [2/1] [U_]
[>....................] recovery = 0.0% (449280/972993472) finish=15694.6min speed=1032K/sec
unused devices:
List all logical and physical disks
# sfdisk -l Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sda1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect /dev/sda2 121132+ 121601- 469- 3766273 5 Extended /dev/sda3 0 - 0 0 0 Empty /dev/sda4 0 - 0 0 0 Empty /dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track Warning: extended partition does not start at a cylinder boundary. DOS and Linux will interpret the contents differently. Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sdb1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect /dev/sdb2 121132+ 121601- 469- 3766273 5 Extended /dev/sdb3 0 - 0 0 0 Empty /dev/sdb4 0 - 0 0 0 Empty /dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris WARNING: GPT (GUID Partition Table) detected on '/dev/sdd'! The util sfdisk doesn't support GPT. Use GNU Parted. Disk /dev/sdd: 364801 cylinders, 255 heads, 63 sectors/track Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sdd1 0+ 267349- 267350- 2147483647+ ee GPT start: (c,h,s) expected (0,0,2) found (0,0,1) /dev/sdd2 0 - 0 0 0 Empty /dev/sdd3 0 - 0 0 0 Empty /dev/sdd4 0 - 0 0 0 Empty WARNING: GPT (GUID Partition Table) detected on '/dev/sdc'! The util sfdisk doesn't support GPT. Use GNU Parted. Disk /dev/sdc: 364801 cylinders, 255 heads, 63 sectors/track Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0 Device Boot Start End #cyls #blocks Id System /dev/sdc1 0+ 267349- 267350- 2147483647+ ee GPT start: (c,h,s) expected (0,0,2) found (0,0,1) /dev/sdc2 0 - 0 0 0 Empty /dev/sdc3 0 - 0 0 0 Empty /dev/sdc4 0 - 0 0 0 Empty Disk /dev/md0: 243248368 cylinders, 2 heads, 4 sectors/track sfdisk: ERROR: sector 0 does not have an msdos signature /dev/md0: unrecognized partition table type No partitions found
Partition the /dev/sdc hard drive
# parted /dev/sdc GNU Parted 2.3 Using /dev/sdc Welcome to GNU Parted! Type 'help' to view a list of commands. (parted) mklabel gpt Warning: The existing disk label on /dev/sdc will be destroyed and all data on this disk will be lost. Do you want to continue? Yes/No? yes (parted) mkpart data00 0GB 1501GB (parted) mkpart data01 1502BG 3001GB Warning: You requested a partition from 1024B to 3001GB. The closest location we can manage is 17.4kB to 1048kB. Is this still acceptable to you? Yes/No? no (parted) quit Information: You may need to update /etc/fstab.
Now, I could use parted again to partition /dev/sdd.
However, I really want to try to clone GPT disks with gdisk --GPT fdisk.
For debian GPT fdisk, currently is found in sid.
Install gdisk
# echo "deb http://ftp.it.debian.org/debian sid main" >> /etc/apt/sources.list # apt-get update # apt-get install gdiskgdisk GPT fdisk (gdisk) version 0.8.5 Type device filename, or pressNot bad, at this moment the latest GPT fdisk version is 0.8.6.to exit: #
Comment out the ftp.it.debian.org repository, add a squeeze repository and
# apt-get update
sgdisk is part of GPT fdisk --gdisk.
Copy the partition scheme of sdc to sdd and change the GUID on /dev/sdd.
# sgdisk -R=/dev/sdd /dev/sdc The operation has completed successfully. # sgdisk -G /dev/sdd The operation has completed successfully.
excerpt from the sgdisk man page.
-R, --replicate=second_device_filename
Replicate the main device's partition table on the specified second device. Note that the replicated partition table is
an exact copy, including all GUIDs; if the device should have its own unique GUIDs, you should use the -G option on the
new disk.
List all logical and physical disks.
# parted -l Model: ATA WDC WD1002FAEX-0 (scsi) Disk /dev/sda: 1000GB Sector size (logical/physical): 512B/512B Partition Table: msdos Number Start End Size Type File system Flags 1 1049kB 996GB 996GB primary ext3 boot, raid 2 996GB 1000GB 3857MB extended 5 996GB 1000GB 3857MB logical linux-swap(v1) Model: ATA WDC WD1002FAEX-0 (scsi) Disk /dev/sdb: 1000GB Sector size (logical/physical): 512B/512B Partition Table: msdos Number Start End Size Type File system Flags 1 1049kB 996GB 996GB primary ext3 boot, raid 2 996GB 1000GB 3857MB extended 5 996GB 1000GB 3857MB logical Model: ATA WDC WD30EFRX-68A (scsi) Disk /dev/sdc: 3001GB Sector size (logical/physical): 512B/4096B Partition Table: gpt Number Start End Size File system Name Flags 1 1049kB 1501GB 1501GB data00 2 1502GB 3001GB 1499GB data01 Model: ATA WDC WD30EFRX-68A (scsi) Disk /dev/sdd: 3001GB Sector size (logical/physical): 512B/4096B Partition Table: gpt Number Start End Size File system Name Flags 1 1049kB 1501GB 1501GB data00 2 1502GB 3001GB 1499GB data01 Model: Linux Software RAID Array (md) Disk /dev/md0: 996GB Sector size (logical/physical): 512B/512B Partition Table: loop Number Start End Size File system Flags 1 0.00B 996GB 996GB ext3
Set the raid flag to true on all logical partitions of /dev/sdc.
# parted /dev/sdc GNU Parted 2.3 Using /dev/sdc Welcome to GNU Parted! Type 'help' to view a list of commands. (parted) print Model: ATA WDC WD30EFRX-68A (scsi) Disk /dev/sdc: 3001GB Sector size (logical/physical): 512B/4096B Partition Table: gpt Number Start End Size File system Name Flags 1 1049kB 1501GB 1501GB data00 2 1502GB 3001GB 1499GB data01 (parted) set 1 raid on (parted) set 2 raid on (parted) print Model: ATA WDC WD30EFRX-68A (scsi) Disk /dev/sdc: 3001GB Sector size (logical/physical): 512B/4096B Partition Table: gpt Number Start End Size File system Name Flags 1 1049kB 1501GB 1501GB data00 raid 2 1502GB 3001GB 1499GB data01 raid (parted) quit Information: You may need to update /etc/fstab.
Set the raid flag to true on all logical partition of /dev/sdd.
# parted /dev/sdd GNU Parted 2.3 Using /dev/sdd Welcome to GNU Parted! Type 'help' to view a list of commands. (parted) print Model: ATA WDC WD30EFRX-68A (scsi) Disk /dev/sdd: 3001GB Sector size (logical/physical): 512B/4096B Partition Table: gpt Number Start End Size File system Name Flags 1 1049kB 1501GB 1501GB data00 2 1502GB 3001GB 1499GB data01 (parted) set 1 raid on (parted) set 2 raid on (parted) print Model: ATA WDC WD30EFRX-68A (scsi) Disk /dev/sdd: 3001GB Sector size (logical/physical): 512B/4096B Partition Table: gpt Number Start End Size File system Name Flags 1 1049kB 1501GB 1501GB data00 raid 2 1502GB 3001GB 1499GB data01 raid (parted) quit Information: You may need to update /etc/fstab.
Look the partitions in proc
# cat /proc/partitions major minor #blocks name 8 0 976762584 sda 8 1 972993536 sda1 8 2 1 sda2 8 5 3766272 sda5 8 16 976762584 sdb 8 17 972993536 sdb1 8 18 1 sdb2 8 21 3766272 sdb5 8 48 2930266584 sdd 8 49 1465819136 sdd1 8 50 1463469056 sdd2 8 32 2930266584 sdc 8 33 1465819136 sdc1 8 34 1463469056 sdc2 9 0 972993472 md0
Create the md1 data00 raid 1 Array.
# mdadm --create /dev/md1 --level=1 --metadata=0 --raid-disks=2 /dev/sdc1 /dev/sdd1 mdadm: array /dev/md1 started.
Format /dev/md1
# mkfs.ext3 -v -m .1 -b 4096 -E stride=32,stripe-width=64 /dev/md1
Create the md2 data01 raid 1 Array
# mdadm --create /dev/md2 --level=1 --metadata=0 --raid-disks=2 /dev/sdc2 /dev/sdd2 mdadm: array /dev/md2 started.
Format /dev/md2
# mkfs.ext3 -v -m .1 -b 4096 -E stride=32,stripe-width=64 /dev/md2
Update mdadmn.conf
# mdadm --examine --scan >> /etc/mdadm/mdadm.conf # vi /etc/mdadm/mdadm.confdelete multiple ARRAY definitions
Create mount points and update /etc/fstab
# mkdir /data00 # mkdir /data01 # vi /etc/fstabThis is how my /etc/fstab looks like
# grep -v "#" /etc/fstab proc /proc proc defaults 0 0 /dev/md0 / ext3 errors=remount-ro 0 1 /dev/sda5 swap swap defaults,pri=1 0 0 /dev/sdb5 swap swap defaults,pri=1 0 0 /dev/md1 /data00 ext3 errors=remount-ro 0 1 /dev/md2 /data01 ext3 errors=remount-ro 0 1
Look at /proc/mdstat
# cat /proc/mdstat
Personalities : [raid1]
md2 : active raid1 sdd2[1] sdc2[0]
1463468992 blocks [2/2] [UU]
resync=DELAYED
md1 : active raid1 sdd1[1] sdc1[0]
1465819072 blocks [2/2] [UU]
[=>...................] resync = 6.8% (100567936/1465819072) finish=172.6min speed=131767K/sec
md0 : active raid1 sda1[0] sdb1[1]
972993472 blocks [2/2] [UU]
unused devices:
# reboot
The system came up fine
Loot at what is mounted and filesystem usage.
# df -h Filesystem Size Used Avail Use% Mounted on /dev/md0 914G 1.6G 911G 1% / tmpfs 943M 0 943M 0% /lib/init/rw udev 936M 168K 936M 1% /dev tmpfs 943M 0 943M 0% /dev/shm /dev/md1 1.4T 198M 1.4T 1% /data00 /dev/md2 1.4T 198M 1.4T 1% /data01
Look at mdstat
# cat /proc/mdstat
Personalities : [raid1]
md2 : active raid1 sdc2[0] sdd2[1]
1463468992 blocks [2/2] [UU]
resync=DELAYED
md1 : active raid1 sdc1[0] sdd1[1]
1465819072 blocks [2/2] [UU]
[=>...................] resync = 8.6% (127094592/1465819072) finish=168.6min speed=132290K/sec
md0 : active raid1 sda1[0] sdb1[1]
972993472 blocks [2/2] [UU]
unused devices:
Look the load
# w 21:24:43 up 1 min, 1 user, load average: 2.17, 0.75, 0.27ouch! I 'll keep you posted on that.
Done!
Yet another Software RAID on Debian GNU Linux
