Notes taken while I was converting an 1 hard drive system to a 4 hard drives system with software RAID Arrays.
-
2 x 1TB hard drives contributing one partition each to a software RAID 1 array
and contributing one swap partition each to the system.
-
2 x 3TB hard drives contributing two partitions each to software RAID 1 Arrays.
The system:
# cat /etc/issue /etc/debian_version
Debian GNU/Linux 6.0 \n \l
6.0.7
# uname -r
2.6.32-5-amd64
Install the Multiple Devices MD --Linux Software RAID Array-- Manager
# apt-get install mdadm
Check out the disk used by the System.
# fdisk -l
Disk /dev/sda: 1000.2 GB, 1000204886016 bytes
255 heads, 63 sectors/track, 121601 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disk identifier: 0x000a8440
Device Boot Start End Blocks Id System
/dev/sda1 * 1 121133 972993536 83 Linux
/dev/sda2 121133 121602 3766273 5 Extended
/dev/sda5 121133 121602 3766272 82 Linux swap / Solaris
OK, shut it down and add the other ~ 1TB disk
Look at the disks
# sfdisk -l
Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sda1 * 0+ 121132- 121133- 972993536 83 Linux
/dev/sda2 121132+ 121601- 469- 3766273 5 Extended
/dev/sda3 0 - 0 0 0 Empty
/dev/sda4 0 - 0 0 0 Empty
/dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track
sfdisk: ERROR: sector 0 does not have an msdos signature
/dev/sdb: unrecognized partition table type
No partitions found
Copy the partition table of /dev/sda to /dev/sdb
# sfdisk -d /dev/sda | sfdisk --force /dev/sdb
List all logical drives on all hard drives
# sfdisk -l
Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sda1 * 0+ 121132- 121133- 972993536 83 Linux
/dev/sda2 121132+ 121601- 469- 3766273 5 Extended
/dev/sda3 0 - 0 0 0 Empty
/dev/sda4 0 - 0 0 0 Empty
/dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sdb1 * 0+ 121132- 121133- 972993536 83 Linux
/dev/sdb2 121132+ 121601- 469- 3766273 5 Extended
/dev/sdb3 0 - 0 0 0 Empty
/dev/sdb4 0 - 0 0 0 Empty
/dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Set /dev/sdb1 0x ID to 0xfd ( Linux RAID Auto )
# sfdisk --change-id /dev/sdb 1 fd
# sfdisk -l /dev/sdb
Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sdb1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect
/dev/sdb2 121132+ 121601- 469- 3766273 5 Extended
/dev/sdb3 0 - 0 0 0 Empty
/dev/sdb4 0 - 0 0 0 Empty
/dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Overwrite with zeros MD superblocks --remains of RAID Arrays-- if any.
# mdadm --zero-superblock --force /dev/sdb1
Create a RAID 1 Array of 2 logical disks using the v0.90 superblock format
# mdadm --create /dev/md0 --level=1 --metadata=0 --raid-disks=2 missing /dev/sdb1
mdadm: array /dev/md0 started.
"The version-0.90 superblock limits the number of component devices within an array to 28, and limits each component device to a maximum size of 2TB on kernel version <3.1 and 4TB on kernel version >=3.1."
The version-0.90 Superblock Format
Look at proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sdb1[1]
972993472 blocks [2/1] [_U]
unused devices:
Create an ext3 file-system on the Multiple Devices MD device /dev/md0
# mkfs.ext3 -v -m .1 -b 4096 -E stride=32,stripe-width=64 /dev/md0
with stripe-width=64 , it should not matter in RAID 1 Arrays.
Options explained:
The first command makes a ext3 filesystem
-v verbose
-m .1 leave .1% of disk to root
-b 4096 block size of 4kb (recommended above for large-file systems)
-E stride=32,stripe-width=64 see below calculation
RAID wiki
Backup mdadm.conf
# cp /etc/mdadm/mdadm.conf /etc/mdadm/mdadm.conf.0
Delete ARRAY definitions from /etc/mdadm/mdadm.conf
# grep -v ARRAY /etc/mdadm/mdadm.conf >> /etc/mdadm/mdadm.conf
Update /etc/mdadm/mdadm.conf
# mdadm --examine --scan >> /etc/mdadm/mdadm.conf
Mount md0
# mkdir /mnt/md0
# mount /dev/md0 /mnt/md0/
Save /etc/fstab for historical reasons
# cp /etc/fstab /etc/fstab.0
Remove or comment out lines starting with UUID or /dev/sd* from /etc/fstab
and make /etc/fstab look like the following.
# grep -v "#" /etc/fstab
proc /proc proc defaults 0 0
/dev/md0 / ext3 errors=remount-ro 0 1
/dev/sda5 swap swap defaults,pri=1 0 0
/dev/sdb5 swap swap defaults,pri=1 0 0
Swapping on RAID
Configure the bootloader --Grub2
# cp /etc/grub.d/40_custom /etc/grub.d/08_raid1
# vi /etc/grub.d/08_raid1
Here is my /etc/grub.d/08_raid1 :
#!/bin/sh
exec tail -n +3 $0
# This file provides an easy way to add custom menu entries. Simply type the
# menu entries you want to add after this comment. Be careful not to change
# the 'exec tail' line above.
menuentry 'Debian GNU/Linux, with Linux 2.6.32-5-amd64 raid1' --class debian --class gnu-linux --class gnu --class os {
insmod raid
insmod mdraid
insmod part_msdos
insmod ext2
set root='(md0)'
echo 'Loading Linux 2.6.32-5-amd64 ...'
linux /boot/vmlinuz-2.6.32-5-amd64 root=/dev/md0 ro quiet
echo 'Loading initial ramdisk ...'
initrd /boot/initrd.img-2.6.32-5-amd64
}
Un-comment or add
GRUB_DISABLE_LINUX_UUID=true
at /etc/default/grub
Replace /dev/sda with /dev/md0 in /etc/mtab
and delete the line showing /dev/md0 mount on /mnt/md0
Update grub.cfg
# grub-mkconfig -o /boot/grub/grub.cfg
Update the initramfs image
# update-initramfs -u
Copy the System to /dev/md0
# cp -dpRx / /mnt/md0
Install grub on all drives.
# grub-install /dev/sdb
# grub-install /dev/sda
Cross your fingers and reboot
# shutdown -r now
OK, the system came up as expected.
Check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sdb1[1]
972993472 blocks [2/1] [_U]
unused devices:
Set /dev/sda1 0x ID to 0xfd ( Linux RAID Auto )
# sfdisk --change-id /dev/sda 1 fd
List all partitions on all drives
# sfdisk -l
Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sda1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect
/dev/sda2 121132+ 121601- 469- 3766273 5 Extended
/dev/sda3 0 - 0 0 0 Empty
/dev/sda4 0 - 0 0 0 Empty
/dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sdb1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect
/dev/sdb2 121132+ 121601- 469- 3766273 5 Extended
/dev/sdb3 0 - 0 0 0 Empty
/dev/sdb4 0 - 0 0 0 Empty
/dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Disk /dev/md0: 243248368 cylinders, 2 heads, 4 sectors/track
sfdisk: ERROR: sector 0 does not have an msdos signature
/dev/md0: unrecognized partition table type
No partitions found
Add /dev/sda1 to the RAID 1 md0
# mdadm --add /dev/md0 /dev/sda1
mdadm: added /dev/sda1
Check proc mdstat
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[2] sdb1[1]
972993472 blocks [2/1] [_U]
[>....................] recovery = 0.4% (4773696/972993472) finish=144.5min speed=111602K/sec
unused devices:
Check load
22:53:31 up 20 min, 1 user, load average: 0.66, 0.19, 0.06
Not (very) bad ;)
Wait for the raid array to sync.
root@bob:~# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[2] sdb1[1]
972993472 blocks [2/1] [_U]
[===========>.........] recovery = 56.1% (546131776/972993472) finish=74.2min speed=95857K/sec
unused devices:
Done.
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0] sdb1[1]
972993472 blocks [2/2] [UU]
unused devices:
Test! What if I take off /dev/sdb ?
# shutdown -h now
take of /dev/sdb and boot
It came up fine :) , check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0]
972993472 blocks [2/1] [U_]
unused devices:
(
Yet another not-very-cheap way for a technician
to hide 'some' tracks (I think)
in a dedicated server farm.
- shutdown the system
- take off drive a
- do crap
- shutdown the system, take off drive b , put in drive a , boot
- shutdown the system, put in drive b, and let them sync
)
Back to business ...
- shutdown the system
- put in sdb
- and boot
It came up fine, check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0]
972993472 blocks [2/1] [U_]
unused devices:
Add /dev/sdb to the RAID array
# mdadm --add /dev/md0 /dev/sdb1
Check proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sdb1[2] sda1[0]
972993472 blocks [2/1] [U_]
[>....................] recovery = 3.6% (35851584/972993472) finish=145.8min speed=107124K/sec
unused devices:
OK, Let 's create the "DATA" arrays.
- shutdown the system
- put in the 3TB hard drives
- Set on BIOS to boot only from /dev/sda and /dev/sdb
- boot
OK the system came up.
Check out proc
# cat /proc/mdstat
Personalities : [raid1]
md0 : active raid1 sda1[0] sdb1[2]
972993472 blocks [2/1] [U_]
[>....................] recovery = 0.0% (449280/972993472) finish=15694.6min speed=1032K/sec
unused devices:
List all logical and physical disks
# sfdisk -l
Disk /dev/sda: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sda1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect
/dev/sda2 121132+ 121601- 469- 3766273 5 Extended
/dev/sda3 0 - 0 0 0 Empty
/dev/sda4 0 - 0 0 0 Empty
/dev/sda5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
Disk /dev/sdb: 121601 cylinders, 255 heads, 63 sectors/track
Warning: extended partition does not start at a cylinder boundary.
DOS and Linux will interpret the contents differently.
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sdb1 * 0+ 121132- 121133- 972993536 fd Linux raid autodetect
/dev/sdb2 121132+ 121601- 469- 3766273 5 Extended
/dev/sdb3 0 - 0 0 0 Empty
/dev/sdb4 0 - 0 0 0 Empty
/dev/sdb5 121132+ 121601- 469- 3766272 82 Linux swap / Solaris
WARNING: GPT (GUID Partition Table) detected on '/dev/sdd'! The util sfdisk doesn't support GPT. Use GNU Parted.
Disk /dev/sdd: 364801 cylinders, 255 heads, 63 sectors/track
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sdd1 0+ 267349- 267350- 2147483647+ ee GPT
start: (c,h,s) expected (0,0,2) found (0,0,1)
/dev/sdd2 0 - 0 0 0 Empty
/dev/sdd3 0 - 0 0 0 Empty
/dev/sdd4 0 - 0 0 0 Empty
WARNING: GPT (GUID Partition Table) detected on '/dev/sdc'! The util sfdisk doesn't support GPT. Use GNU Parted.
Disk /dev/sdc: 364801 cylinders, 255 heads, 63 sectors/track
Units = cylinders of 8225280 bytes, blocks of 1024 bytes, counting from 0
Device Boot Start End #cyls #blocks Id System
/dev/sdc1 0+ 267349- 267350- 2147483647+ ee GPT
start: (c,h,s) expected (0,0,2) found (0,0,1)
/dev/sdc2 0 - 0 0 0 Empty
/dev/sdc3 0 - 0 0 0 Empty
/dev/sdc4 0 - 0 0 0 Empty
Disk /dev/md0: 243248368 cylinders, 2 heads, 4 sectors/track
sfdisk: ERROR: sector 0 does not have an msdos signature
/dev/md0: unrecognized partition table type
No partitions found
Partition the /dev/sdc hard drive
# parted /dev/sdc
GNU Parted 2.3
Using /dev/sdc
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) mklabel gpt
Warning: The existing disk label on /dev/sdc will be destroyed and all data on this disk will be lost. Do you want to continue?
Yes/No? yes
(parted) mkpart data00 0GB 1501GB
(parted) mkpart data01 1502BG 3001GB
Warning: You requested a partition from 1024B to 3001GB.
The closest location we can manage is 17.4kB to 1048kB.
Is this still acceptable to you?
Yes/No? no
(parted) quit
Information: You may need to update /etc/fstab.
Now, I could use parted again to partition /dev/sdd.
However, I really want to try to clone GPT disks with gdisk --GPT fdisk.
For debian
GPT fdisk,
currently is found in sid.
Install gdisk
# echo "deb http://ftp.it.debian.org/debian sid main" >> /etc/apt/sources.list
# apt-get update
# apt-get install gdiskgdisk
GPT fdisk (gdisk) version 0.8.5
Type device filename, or press to exit:
#
Not bad,
at this moment the latest GPT fdisk version is 0.8.6.
Comment out the ftp.it.debian.org repository, add a squeeze repository and
# apt-get update
sgdisk is part of GPT fdisk --gdisk.
Copy the partition scheme of sdc to sdd and change the GUID on /dev/sdd.
# sgdisk -R=/dev/sdd /dev/sdc
The operation has completed successfully.
# sgdisk -G /dev/sdd
The operation has completed successfully.
excerpt from the sgdisk man page.
-R, --replicate=second_device_filename
Replicate the main device's partition table on the specified second device. Note that the replicated partition table is
an exact copy, including all GUIDs; if the device should have its own unique GUIDs, you should use the -G option on the
new disk.
List all logical and physical disks.
# parted -l
Model: ATA WDC WD1002FAEX-0 (scsi)
Disk /dev/sda: 1000GB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Number Start End Size Type File system Flags
1 1049kB 996GB 996GB primary ext3 boot, raid
2 996GB 1000GB 3857MB extended
5 996GB 1000GB 3857MB logical linux-swap(v1)
Model: ATA WDC WD1002FAEX-0 (scsi)
Disk /dev/sdb: 1000GB
Sector size (logical/physical): 512B/512B
Partition Table: msdos
Number Start End Size Type File system Flags
1 1049kB 996GB 996GB primary ext3 boot, raid
2 996GB 1000GB 3857MB extended
5 996GB 1000GB 3857MB logical
Model: ATA WDC WD30EFRX-68A (scsi)
Disk /dev/sdc: 3001GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1501GB 1501GB data00
2 1502GB 3001GB 1499GB data01
Model: ATA WDC WD30EFRX-68A (scsi)
Disk /dev/sdd: 3001GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1501GB 1501GB data00
2 1502GB 3001GB 1499GB data01
Model: Linux Software RAID Array (md)
Disk /dev/md0: 996GB
Sector size (logical/physical): 512B/512B
Partition Table: loop
Number Start End Size File system Flags
1 0.00B 996GB 996GB ext3
Set the raid flag to true on all logical partitions of /dev/sdc.
# parted /dev/sdc
GNU Parted 2.3
Using /dev/sdc
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) print
Model: ATA WDC WD30EFRX-68A (scsi)
Disk /dev/sdc: 3001GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1501GB 1501GB data00
2 1502GB 3001GB 1499GB data01
(parted) set 1 raid on
(parted) set 2 raid on
(parted) print
Model: ATA WDC WD30EFRX-68A (scsi)
Disk /dev/sdc: 3001GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1501GB 1501GB data00 raid
2 1502GB 3001GB 1499GB data01 raid
(parted) quit
Information: You may need to update /etc/fstab.
Set the raid flag to true on all logical partition of /dev/sdd.
# parted /dev/sdd
GNU Parted 2.3
Using /dev/sdd
Welcome to GNU Parted! Type 'help' to view a list of commands.
(parted) print
Model: ATA WDC WD30EFRX-68A (scsi)
Disk /dev/sdd: 3001GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1501GB 1501GB data00
2 1502GB 3001GB 1499GB data01
(parted) set 1 raid on
(parted) set 2 raid on
(parted) print
Model: ATA WDC WD30EFRX-68A (scsi)
Disk /dev/sdd: 3001GB
Sector size (logical/physical): 512B/4096B
Partition Table: gpt
Number Start End Size File system Name Flags
1 1049kB 1501GB 1501GB data00 raid
2 1502GB 3001GB 1499GB data01 raid
(parted) quit
Information: You may need to update /etc/fstab.
Look the partitions in proc
# cat /proc/partitions
major minor #blocks name
8 0 976762584 sda
8 1 972993536 sda1
8 2 1 sda2
8 5 3766272 sda5
8 16 976762584 sdb
8 17 972993536 sdb1
8 18 1 sdb2
8 21 3766272 sdb5
8 48 2930266584 sdd
8 49 1465819136 sdd1
8 50 1463469056 sdd2
8 32 2930266584 sdc
8 33 1465819136 sdc1
8 34 1463469056 sdc2
9 0 972993472 md0
Create the md1 data00 raid 1 Array.
# mdadm --create /dev/md1 --level=1 --metadata=0 --raid-disks=2 /dev/sdc1 /dev/sdd1
mdadm: array /dev/md1 started.
Format /dev/md1
# mkfs.ext3 -v -m .1 -b 4096 -E stride=32,stripe-width=64 /dev/md1
Create the md2 data01 raid 1 Array
# mdadm --create /dev/md2 --level=1 --metadata=0 --raid-disks=2 /dev/sdc2 /dev/sdd2
mdadm: array /dev/md2 started.
Format /dev/md2
# mkfs.ext3 -v -m .1 -b 4096 -E stride=32,stripe-width=64 /dev/md2
Update mdadmn.conf
# mdadm --examine --scan >> /etc/mdadm/mdadm.conf
# vi /etc/mdadm/mdadm.conf
delete multiple ARRAY definitions
Create mount points and update /etc/fstab
# mkdir /data00
# mkdir /data01
# vi /etc/fstab
This is how my /etc/fstab looks like
# grep -v "#" /etc/fstab
proc /proc proc defaults 0 0
/dev/md0 / ext3 errors=remount-ro 0 1
/dev/sda5 swap swap defaults,pri=1 0 0
/dev/sdb5 swap swap defaults,pri=1 0 0
/dev/md1 /data00 ext3 errors=remount-ro 0 1
/dev/md2 /data01 ext3 errors=remount-ro 0 1
Look at /proc/mdstat
# cat /proc/mdstat
Personalities : [raid1]
md2 : active raid1 sdd2[1] sdc2[0]
1463468992 blocks [2/2] [UU]
resync=DELAYED
md1 : active raid1 sdd1[1] sdc1[0]
1465819072 blocks [2/2] [UU]
[=>...................] resync = 6.8% (100567936/1465819072) finish=172.6min speed=131767K/sec
md0 : active raid1 sda1[0] sdb1[1]
972993472 blocks [2/2] [UU]
unused devices:
# reboot
The system came up fine
Loot at what is mounted and filesystem usage.
# df -h
Filesystem Size Used Avail Use% Mounted on
/dev/md0 914G 1.6G 911G 1% /
tmpfs 943M 0 943M 0% /lib/init/rw
udev 936M 168K 936M 1% /dev
tmpfs 943M 0 943M 0% /dev/shm
/dev/md1 1.4T 198M 1.4T 1% /data00
/dev/md2 1.4T 198M 1.4T 1% /data01
Look at mdstat
# cat /proc/mdstat
Personalities : [raid1]
md2 : active raid1 sdc2[0] sdd2[1]
1463468992 blocks [2/2] [UU]
resync=DELAYED
md1 : active raid1 sdc1[0] sdd1[1]
1465819072 blocks [2/2] [UU]
[=>...................] resync = 8.6% (127094592/1465819072) finish=168.6min speed=132290K/sec
md0 : active raid1 sda1[0] sdb1[1]
972993472 blocks [2/2] [UU]
unused devices:
Look the load
# w
21:24:43 up 1 min, 1 user, load average: 2.17, 0.75, 0.27
ouch! I 'll keep you posted on that.
Done!
