Thanks for the how-to. I made a cheat sheet while following it to make a root on LVM2 on RAID setup which had some gotchas. I figured I'd cut and paste my cheat sheet here in case anyone else is groping for how to do this.
- Boot a live cd
- Load the md and dm-mod modules:
- Create partitions:
Code: Select all
Device Boot Start End Blocks Id System
/dev/hda1 * 1 25 100768+ fd Linux raid autodetect
/dev/hda2 26 969 3806208 fd Linux raid autodetect
/dev/hdh1 * 1 200 100768+ fd Linux raid autodetect
/dev/hdh2 201 7752 3806208 fd Linux raid autodetect
/dev/hdh3 7753 59554 26108208 8e Linux LVM
/dev/hde1 1 14593 117218241 fd Linux raid autodetect
/dev/hdg1 1 14593 117218241 fd Linux raid autodetect
- Create /etc/raidtab:
Code: Select all
# Mirror /dev/hda with /dev/hdh
# /boot (RAID 1)
raiddev /dev/md0
raid-level 1
nr-raid-disks 2
chunk-size 32
persistent-superblock 1
device /dev/hda1
raid-disk 0
device /dev/hdh1
raid-disk 1
# / (RAID 1)
raiddev /dev/md1
raid-level 1
nr-raid-disks 2
chunk-size 32
persistent-superblock 1
device /dev/hda2
raid-disk 0
device /dev/hdh2
raid-disk 1
# Mirror /dev/hde with /dev/hdg
raiddev /dev/md2
raid-level 1
nr-raid-disks 2
chunk-size 32
persistent-superblock 1
device /dev/hde1
raid-disk 0
device /dev/hdg1
raid-disk 1
- Make the RAID
Code: Select all
mkraid /dev/md0
mkraid /dev/md1
mkraid /dev/md2
- /boot will be ext3 (no LVM)
Code: Select all
mke2fs -j -L BOOT -m 1 -v /dev/md0
- Create LVM PVs
Code: Select all
pvcreate /dev/md1 /dev/md2 /dev/hdh3
- Create /etc/lvm/lvm.conf
Code: Select all
echo 'devices { filter=["r/cdrom|hdh[12]|hd[a-gi-z]/"] }' >/etc/lvm/lvm.conf
- Create LVM VGs
Code: Select all
vgcreate m1 /dev/md1 # mirrored volume group 1
# will hold a compact fully-functioning base system
vgcreate m2 /dev/md2 # mirrored volume group 2
# stuff not particularly needed to run linux
# such as /usr/portage, mp3 directory, ... etc
vgcreate um /dev/hdh3 # unmirrored volume group
# for things like /tmp, /var/tmp, ... etc
- Create LVM LVs
Code: Select all
lvcreate -L 200M -n root m1 # root on m1
lvcreate -L 100M -n swap m1 # swap on m1
lvcreate -L 1G -n usr m1 # /usr on m1
lvcreate -L 200M -n var m1 # /var on m1
lvcreate -L 500M -n X m2 # /usr/X11R6 on m2
lvcreate -L 2G -n portage m2 # /usr/portage on m2
lvcreate -L 400M -n swap m2 # swap on m2
lvcreate -L 2G -n ushare m2 # /usr/share on m2
lvcreate -L 1G -n usrc m2 # /usr/src on m2
lvcreate -L 2G -n ccache um # /um/ccache on um (CCACHE_DIR)
lvcreate -L 5G -n tmp um # unmirrored tmp filesystem
# for /tmp, /var/tmp, ... etc
# create symbolic links to this for these
- Make and activate the swap files
Code: Select all
mkswap /dev/m1/swap
mkswap /dev/m2/swap
swapon /dev/m1/swap
swapon /dev/m2/swap
- Put filesystems on LVs
Code: Select all
mke2fs -j -L ROOT -m 1 -v /dev/m1/root
mke2fs -j -L USR -m 1 -v /dev/m1/usr
mke2fs -j -L VAR -m 1 -v /dev/m1/var
mke2fs -j -L X -m 1 -v /dev/m2/X
mke2fs -j -L PORTAGE -m 1 -v /dev/m2/portage
mke2fs -j -L USHARE -m 1 -v /dev/m2/ushare
mke2fs -j -L USRC -m 1 -v /dev/m2/usrc
mke2fs -j -L CCACHE -m 1 -v /dev/um/ccache
mke2fs -j -L TMP -m 1 -v /dev/um/tmp
- Mount the filesystems
Code: Select all
mount /dev/m1/root /mnt/gentoo
mount /dev/md0 /mnt/gentoo/boot
cd /mnt/gentoo; mkdir um usr var
mount /dev/m1/usr usr
mount /dev/m1/var var
cd usr; mkdir X11R6 portage share src
mount /dev/m2/X X11R6
mount /dev/m2/portage portage
mount /dev/m2/ushare share
mount /dev/m2/usrc src
cd ../um; mkdir ccache tmp
mount /dev/um/ccache ccache
mount /dev/um/tmp tmp
cd ..
find . -exec chmod 777 {} \;
- Create and mount /proc
Code: Select all
mkdir proc
mount -t proc proc /mnt/gentoo/proc
- Untar the stage file and remove it
- Copy over configuration files
Code: Select all
cp /etc/raidtab etc
mkdir etc/lvm
cp /etc/lvm/lvm.conf etc/lvm
(also copy from backups: hosts, resolv.conf, make.conf, /root/*, ...)
- chroot
- Make links to /tmp
Code: Select all
rm -r /tmp; ln -sf um/tmp /tmp
rm -r /var/tmp; ln -sf ../um/tmp /var/tmp
rm -r /usr/tmp; ln -sf ../um/tmp /usr/tmp
- Create an /etc/fstab
Code: Select all
# <fs> <mountpoint> <type> <opts> <dump/pass>
/dev/m1/swap none swap sw,pri=1 0 0
/dev/m2/swap none swap sw,pri=1 0 0
/dev/md0 /boot ext3 noatime 1 2
/dev/m1/root / ext3 noatime 0 1
/dev/m1/usr /usr ext3 noatime 0 0
/dev/m1/var /var ext3 noatime 0 0
/dev/um/ccache /um/ccache ext3 noatime 0 0
/dev/um/tmp /um/tmp ext3 noatime 0 0
/dev/m2/X /usr/X11R6 ext3 noatime 0 0
/dev/m2/portage /usr/portage ext3 noatime 0 0
/dev/m2/ushare /usr/share ext3 noatime 0 0
/dev/m2/usrc /usr/src ext3 noatime 0 0
/dev/cdroms/cdrom0 /mnt/cdrom auto noauto,ro,user 0 0
/dev/fd0 /mnt/floppy auto noauto,user 0 0
none /proc proc defaults 0 0
none /dev/shm tmpfs defaults 0 0
- Run through the install as usual until the step to make the kernel
- Create an empty /initrd to pivot_root the initrd into
- Create a 16MB LVM initrd
Code: Select all
mkdir /tmp/initrd
cd /tmp/initrd
dd if=/dev/zero of=devram count=16384 bs=1024
mke2fs -F -m0 -L INITRD devram 16384
mkdir tmpmnt
mount -o loop devram tmpmnt
cd tmpmnt
mkdir bin dev etc lib proc root sbin tmp usr var
- Throw files into this tmpmnt directory that are needed to bootstrap
the root filesystem as well as anything that you'd need to troubleshoot
the system if something goes wrong (essentially create a minimal
gentoo livecd). I put all executables except for "init" in bin.
- Make sure the library files are included
Code: Select all
ldd bin/* | awk '{if (/=>/) { print $3 }}' | sort -u \
| awk 'system("cp "$1" lib")'
- Create the sbin/init file
Code: Select all
#!/bin/bash
# include in the path some dirs from the real root filesystem
# for chroot, blockdev
export PATH="/sbin:/bin:/usr/sbin:/usr/bin:/initrd/bin:/initrd/sbin"
PRE="initrd:"
do_shell() {
/bin/echo
/bin/echo "*** Entering LVM2 rescue shell. Exit shell to continue booting. ***"
/bin/echo
/bin/bash
}
echo "$PRE Remounting / read/write"
mount -t ext2 -o remount,rw /dev/ram0 /
# We need /proc for device mapper
echo "$PRE Mounting /proc"
mount -t proc none /proc
# Create the /dev/mapper/control device for the ioctl
# interface using the major and minor numbers that have been allocated
# dynamically.
echo -n "$PRE Finding device mapper major and minor numbers "
MAJOR=$(sed -n 's/^ *\([0-9]\+\) \+misc$/\1/p' /proc/devices)
MINOR=$(sed -n 's/^ *\([0-9]\+\) \+device-mapper$/\1/p' /proc/misc)
if test -n "$MAJOR" -a -n "$MINOR"
then
mkdir -p -m 755 /dev/mapper
mknod -m 600 /dev/mapper/control c $MAJOR $MINOR
fi
echo "($MAJOR,$MINOR)"
# Device-Mapper dynamically allocates all device numbers. This means it is
# possible that the root volume specified to LILO or Grub may have a different
# number when the initrd runs than when the system was last running. In order
# to make sure the correct volume is mounted as root, the init script must
# determine what the desired root volume name is by getting the LVM2 root
# volume name from the kernel command line. In order for this to work
# correctly, "lvm_root=/dev/Volume_Group_Name/Root_Volume_Name" needs to be
# passed to the kernel command line (where Root_Volume_Name is replaced by
# your actual root volume's name.
for arg in `cat /proc/cmdline`
do
echo $arg | grep '^lvm_root=' > /dev/null
if [ $? -eq 0 ]
then
rootvol=${arg#lvm_root=}
break
fi
done
echo "$PRE Activating LVM2 volumes"
# run a shell if we're passed lvmrescue on commandline
grep lvmrescue /proc/cmdline 1>/dev/null 2>&1
if [ $? -eq 0 ]
then
lvm vgscan
lvm vgchange --ignorelockingfailure -P -a y
do_shell
else
lvm vgscan
lvm vgchange --ignorelockingfailure -a y
fi
echo "$PRE Mounting root filesystem $rootvol ro"
mkdir /rootvol
if ! mount -t auto -o ro $rootvol /rootvol
then
echo "\t*FAILED*";
do_shell
fi
echo "$PRE Umounting /proc"
umount /proc
echo "$PRE Changing roots"
cd /rootvol
if ! pivot_root . initrd
then
echo "\t*FAILED*"
do_shell
fi
echo "$PRE Proceeding with boot..."
exec chroot . /bin/sh -c "/bin/umount /initrd; \
/sbin/blockdev --flushbufs /dev/ram0; \
exec /sbin/init $*" < dev/console > dev/console 2>&1
- Remove the "lost+found" directory
- Create the compressed initrd in the /boot directory
Code: Select all
cd ..
umount tmpmnt
dd if=devram bs=1k count=16384 | gzip -9 >/boot/initrd-2.6.6-rc1.gz
- Compile the kernel as usual, make sure the following are compiled:
Code: Select all
General Setup->Support for hot-pluggable devices
Block devices->Loopback device suppport
Block devices->RAM disk support
Block devices->Default RAM disk size = (16384)
Block devices->Initial RAM disk (initrd) support
Multi-device support (RAID and LVM)->Multiple ...
Multi-device support (RAID and LVM)->RAID support
Multi-device support (RAID and LVM)->RAID-1 (mirroring) mode
Multi-device support (RAID and LVM)->Multipath I/O support
Multi-device support (RAID and LVM)->Device mapper support
File systems->Pseudo filesystems->/proc ...
File systems->Pseudo filesystems->Virtual memory ...
[i]Don't compile in /dev filesystem support[/i]
- Compile and install
Code: Select all
make && make modules_install
cp arch/i386/boot/bzImage /boot/kernel-2.6.6-rc1
cp System.map /boot/System.map-2.6.6-rc1
cp .config /boot/config-2.6.6-rc1
- Setup grub as usual on the MBR of the boot disk
(not the MD - it will have to resync)
Example lines in grub.conf:
Code: Select all
root (hd0,0)
kernel /kernel-2.6.6-rc1 root=/dev/ram0 lvm_root=/dev/m1/root vga=788
initrd /initrd-2.6.6-rc1.gz
- Modify /etc/runlevels/boot/checkroot so that it creates LVM nodes
before trying to check the root filesystem. Add this line in an
appropriate place:
Code: Select all
/sbin/vgscan -v --mknodes --ignorelockingfailure
- Wrap up. Exit out of the chroot environment, back up everything
to an rsync server, unmount everything, and reboot.
