gmirror gm0 destroyed on shutdown; GPT corrupt

[Anton Shterenlikht]

FreeBSD 8.0-CURRENT-200906 ia64, fresh installation

Following the handbook, section 19.1 RAID1 - mirroring, I'm trying to
use gmirror with 2 identical scsi disks:

da0 at mpt0 bus 0 target 0 lun 0
da0: <SEAGATE ST318452LC 2213> Fixed Direct Access SCSI-3 device
da0: 160.000MB/s transfers (80.000MHz, offset 63, 16bit)
da0: Command Queueing Enabled
da0: 17366MB (35566478 512 byte sectors: 255H 63S/T 2213C)
da1 at mpt0 bus 0 target 1 lun 0
da1: <SEAGATE ST318452LC 2213> Fixed Direct Access SCSI-3 device
da1: 160.000MB/s transfers (80.000MHz, offset 63, 16bit)
da1: Command Queueing Enabled
da1: 17366MB (35566478 512 byte sectors: 255H 63S/T 2213C)


# sysctl kern.geom.debugflags=17
kern.geom.debugflags: 0 -> 17

# gmirror label -vb round-robin gm0 /dev/da0
Metadata value stored on /dev/da0.
Done.

# gmirror load
#g_modevent(MIRROR, LOAD)
g_post_event_x(0xe000000004b8eb10, 0xe000000010686e40, 2, 262144)
g_load_class(MIRROR)
g_mirror_taste(MIRROR, acd0t01)
acd0: FAILURE - READ_BIG MEDIUM ERROR asc=0x02 ascq=0x00
g_detach(0xe0000000106eb580)
g_destroy_consumer(0xe0000000106eb580)
g_destroy_geom(0xe000000010724800(mirror:taste))
g_mirror_taste(MIRROR, acd0)
acd0: FAILURE - READ_BIG MEDIUM ERROR asc=0x02 ascq=0x00
g_detach(0xe0000000106eb700)
g_destroy_consumer(0xe0000000106eb700)
g_destroy_geom(0xe000000010724a00(mirror:taste))
g_mirror_taste(MIRROR, da2p6)
g_detach(0xe0000000106eb880)
g_destroy_consumer(0xe0000000106eb880)
g_destroy_geom(0xe000000010724c00(mirror:taste))
g_mirror_taste(MIRROR, da2p5)
g_detach(0xe0000000106eba80)
g_destroy_consumer(0xe0000000106eba80)
g_destroy_geom(0xe000000010724c00(mirror:taste))
g_mirror_taste(MIRROR, da2p4)
g_detach(0xe0000000106ebc00)
g_destroy_consumer(0xe0000000106ebc00)
g_destroy_geom(0xe000000010630700(mirror:taste))
g_mirror_taste(MIRROR, da2p3)
g_detach(0xe000000010738000)
g_destroy_consumer(0xe000000010738000)
g_destroy_geom(0xe000000010724e00(mirror:taste))
g_mirror_taste(MIRROR, da2p2)
g_detach(0xe000000010738180)
g_destroy_consumer(0xe000000010738180)
g_destroy_geom(0xe00000001075f400(mirror:taste))
g_mirror_taste(MIRROR, da2p1)
g_detach(0xe000000010620e80)
g_destroy_consumer(0xe000000010620e80)
g_destroy_geom(0xe000000010724a00(mirror:taste))
g_mirror_taste(MIRROR, da0p6)
g_detach(0xe0000000106e9580)
g_destroy_consumer(0xe0000000106e9580)
g_destroy_geom(0xe000000010724800(mirror:taste))
g_mirror_taste(MIRROR, da0p5)
g_detach(0xe000000010739700)
g_destroy_consumer(0xe000000010739700)
g_destroy_geom(0xe0000000108f8f00(mirror:taste))
g_mirror_taste(MIRROR, da0p4)
g_detach(0xe000000010739680)
g_destroy_consumer(0xe000000010739680)
g_destroy_geom(0xe0000000108f8d00(mirror:taste))
g_mirror_taste(MIRROR, da0p3)
g_detach(0xe000000010739800)
g_destroy_consumer(0xe000000010739800)
g_destroy_geom(0xe00000001072ce00(mirror:taste))
g_mirror_taste(MIRROR, da0p2)
g_detach(0xe000000010739780)
g_destroy_consumer(0xe000000010739780)
g_destroy_geom(0xe0000000108f8700(mirror:taste))
g_mirror_taste(MIRROR, da0p1)
g_detach(0xe0000000106e9680)
g_destroy_consumer(0xe0000000106e9680)
g_destroy_geom(0xe00000001072cb00(mirror:taste))
g_mirror_taste(MIRROR, da2)
g_detach(0xe0000000106a4900)
g_destroy_consumer(0xe0000000106a4900)
g_destroy_geom(0xe00000001075fa00(mirror:taste))
g_mirror_taste(MIRROR, da1)
g_detach(0xe0000000106ebc80)
g_destroy_consumer(0xe0000000106ebc80)
g_destroy_geom(0xe000000010630b00(mirror:taste))
g_mirror_taste(MIRROR, da0)
g_detach(0xe0000000106e8400)
g_destroy_consumer(0xe0000000106e8400)
g_destroy_geom(0xe00000001072c800(mirror:taste))
g_post_event_x(0xe000000004b861c0, 0xe0000000108f9000, 2, 0)
  ref 0xe0000000108f9000
  ref 0xe000000010760800
GEOM_MIRROR: Device mirror/gm0 launched (1/1).
g_mirror_taste(MIRROR, ufsid/4a3fa1b76cb317b5)
g_detach(0xe0000000106a4780)
g_destroy_consumer(0xe0000000106a4780)
g_destroy_geom(0xe0000000108fa300(mirror:taste))
g_mirror_taste(MIRROR, ufsid/4a3fa1b69c522d30)
g_detach(0xe0000000106e9900)
g_destroy_consumer(0xe0000000106e9900)
g_destroy_geom(0xe0000000108fac00(mirror:taste))
g_mirror_taste(MIRROR, ufsid/4a3fa1b751514347)
g_detach(0xe0000000106ebb00)
g_destroy_consumer(0xe0000000106ebb00)
g_destroy_geom(0xe00000001075f300(mirror:taste))
g_mirror_taste(MIRROR, ufsid/4a3fa1b5e5003da2)
g_detach(0xe0000000106e9880)
g_destroy_consumer(0xe0000000106e9880)
g_destroy_geom(0xe000000010763600(mirror:taste))
g_mirror_taste(MIRROR, iso9660/FreeBSD_Install)
acd0: FAILURE - READ_BIG MEDIUM ERROR asc=0x02 ascq=0x00
g_detach(0xe000000010738080)
g_destroy_consumer(0xe000000010738080)
g_destroy_geom(0xe00000001081dd00(mirror:taste))
g_mirror_taste(MIRROR, mirror/gm0)
g_detach(0xe0000000106e9980)
g_destroy_consumer(0xe0000000106e9980)
g_destroy_geom(0xe000000010763600(mirror:taste))
dev_taste(DEV,mirror/gm0)
g_part_taste(PART,mirror/gm0)

GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
GEOM: mirror/gm0: using the primary only -- recovery suggested.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

g_post_event_x(0xe000000004b861c0, 0xe0000000108fa400, 2, 0)
  ref 0xe0000000108fa400
  ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe00000001075fa00, 2, 0)
  ref 0xe00000001075fa00
  ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe0000000108f8700, 2, 0)
  ref 0xe0000000108f8700
  ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe0000000108f8500, 2, 0)
  ref 0xe0000000108f8500
  ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe000000010725a00, 2, 0)
  ref 0xe0000000108f8500
  ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe000000010725a00, 2, 0)
  ref 0xe000000010725a00
  ref 0xe0000000108f8300
g_post_event_x(0xe000000004b861c0, 0xe000000010631500, 2, 0)
  ref 0xe000000010631500
  ref 0xe0000000108f8300
g_label_taste(LABEL, mirror/gm0)
g_detach(0xe000000010738080)
g_destroy_consumer(0xe000000010738080)
g_destroy_geom(0xe00000001075ef00(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p1)
g_detach(0xe000000010739b00)
g_destroy_consumer(0xe000000010739b00)
g_destroy_geom(0xe000000010762f00(mirror:taste))
dev_taste(DEV,mirror/gm0p1)
g_part_taste(PART,mirror/gm0p1)
g_wither_geom(0xe000000010763a00(mirror/gm0p1))
g_label_taste(LABEL, mirror/gm0p1)
g_detach(0xe0000000108fc000)
g_destroy_consumer(0xe0000000108fc000)
g_destroy_geom(0xe00000001081d500(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p2)
g_detach(0xe0000000108fc100)
g_destroy_consumer(0xe0000000108fc100)
g_destroy_geom(0xe0000000108f9d00(mirror:taste))
dev_taste(DEV,mirror/gm0p2)
g_part_taste(PART,mirror/gm0p2)
g_wither_geom(0xe00000001072c800(mirror/gm0p2))
g_label_taste(LABEL, mirror/gm0p2)
g_slice_config(mirror/gm0p2, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe0000000108fae00, 2, 0)
  ref 0xe0000000108fae00
  ref 0xe0000000108faa00
g_detach(0xe0000000108fc300)
g_destroy_consumer(0xe0000000108fc300)
g_destroy_geom(0xe00000001081d100(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p3)
g_detach(0xe0000000108fc380)
g_destroy_consumer(0xe0000000108fc380)
g_destroy_geom(0xe000000010724600(mirror:taste))
dev_taste(DEV,mirror/gm0p3)
g_part_taste(PART,mirror/gm0p3)
g_wither_geom(0xe00000001075ed00(mirror/gm0p3))
g_label_taste(LABEL, mirror/gm0p3)
g_detach(0xe0000000106e9900)
g_destroy_consumer(0xe0000000106e9900)
g_destroy_geom(0xe000000010630700(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p4)
g_detach(0xe0000000108fc600)
g_destroy_consumer(0xe0000000108fc600)
g_destroy_geom(0xe0000000108fad00(mirror:taste))
dev_taste(DEV,mirror/gm0p4)
g_part_taste(PART,mirror/gm0p4)
g_wither_geom(0xe000000010762a00(mirror/gm0p4))
g_label_taste(LABEL, mirror/gm0p4)
g_slice_config(mirror/gm0p4, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe000000010630c00, 2, 0)
  ref 0xe000000010630c00
  ref 0xe0000000108f8c00
g_detach(0xe0000000108fc800)
g_destroy_consumer(0xe0000000108fc800)
g_destroy_geom(0xe000000010763900(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p5)
g_detach(0xe0000000108fcb00)
g_destroy_consumer(0xe0000000108fcb00)
g_destroy_geom(0xe000000010724800(mirror:taste))
dev_taste(DEV,mirror/gm0p5)
g_part_taste(PART,mirror/gm0p5)
g_wither_geom(0xe000000010724e00(mirror/gm0p5))
g_label_taste(LABEL, mirror/gm0p5)
g_slice_config(mirror/gm0p5, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe0000000108f8e00, 2, 0)
  ref 0xe0000000108f8e00
  ref 0xe00000001075f400
g_detach(0xe000000010739780)
g_destroy_consumer(0xe000000010739780)
g_destroy_geom(0xe00000001072c400(label:taste))
g_mirror_taste(MIRROR, mirror/gm0p6)
g_detach(0xe0000000108fcb80)
g_destroy_consumer(0xe0000000108fcb80)
g_destroy_geom(0xe00000001081d500(mirror:taste))
dev_taste(DEV,mirror/gm0p6)
g_part_taste(PART,mirror/gm0p6)
g_wither_geom(0xe0000000108f8200(mirror/gm0p6))
g_label_taste(LABEL, mirror/gm0p6)
g_slice_config(mirror/gm0p6, 0, 1)
g_post_event_x(0xe000000004b861c0, 0xe00000001081d100, 2, 0)
  ref 0xe00000001081d100
  ref 0xe0000000108f9d00
g_detach(0xe0000000108fcc80)
g_destroy_consumer(0xe0000000108fcc80)
g_destroy_geom(0xe0000000108fb300(label:taste))
g_mirror_taste(MIRROR, ufsid/4a43468a267a63a6)
g_detach(0xe0000000108fcd80)
g_destroy_consumer(0xe0000000108fcd80)
g_destroy_geom(0xe000000010724600(mirror:taste))
dev_taste(DEV,ufsid/4a43468a267a63a6)
g_part_taste(PART,ufsid/4a43468a267a63a6)
g_wither_geom(0xe000000010724600(ufsid/4a43468a267a63a6))
g_label_taste(LABEL, ufsid/4a43468a267a63a6)
g_mirror_taste(MIRROR, ufsid/4a43468cf6208bf0)
g_detach(0xe000000010739480)
g_destroy_consumer(0xe000000010739480)
g_destroy_geom(0xe00000001081d500(mirror:taste))
dev_taste(DEV,ufsid/4a43468cf6208bf0)
g_part_taste(PART,ufsid/4a43468cf6208bf0)
g_wither_geom(0xe000000010724800(ufsid/4a43468cf6208bf0))
g_label_taste(LABEL, ufsid/4a43468cf6208bf0)
g_mirror_taste(MIRROR, ufsid/4a43468b228d030c)
g_detach(0xe0000000106eaf80)
g_destroy_consumer(0xe0000000106eaf80)
g_destroy_geom(0xe00000001081c200(mirror:taste))
dev_taste(DEV,ufsid/4a43468b228d030c)
g_part_taste(PART,ufsid/4a43468b228d030c)
g_wither_geom(0xe00000001081dc00(ufsid/4a43468b228d030c))
g_label_taste(LABEL, ufsid/4a43468b228d030c)
g_mirror_taste(MIRROR, ufsid/4a43468c8715f453)
g_detach(0xe0000000108fd100)
g_destroy_consumer(0xe0000000108fd100)
g_destroy_geom(0xe0000000108f8100(mirror:taste))
dev_taste(DEV,ufsid/4a43468c8715f453)
g_part_taste(PART,ufsid/4a43468c8715f453)
g_wither_geom(0xe000000010724a00(ufsid/4a43468c8715f453))
g_label_taste(LABEL, ufsid/4a43468c8715f453)
g_detach(0xe0000000106ea680)
g_destroy_consumer(0xe0000000106ea680)
g_destroy_geom(0xe000000010724a00(ufsid/4a43468c8715f453))
g_detach(0xe0000000108fd000)
g_destroy_consumer(0xe0000000108fd000)
g_destroy_geom(0xe00000001081dc00(ufsid/4a43468b228d030c))
g_detach(0xe000000010739280)
g_destroy_consumer(0xe000000010739280)
g_destroy_geom(0xe000000010724800(ufsid/4a43468cf6208bf0))
g_detach(0xe0000000106eb880)
g_destroy_consumer(0xe0000000106eb880)
g_destroy_geom(0xe000000010724600(ufsid/4a43468a267a63a6))
g_detach(0xe000000010738000)
g_destroy_consumer(0xe000000010738000)
g_destroy_geom(0xe0000000108f8200(mirror/gm0p6))
g_detach(0xe0000000106a4900)
g_destroy_consumer(0xe0000000106a4900)
g_destroy_geom(0xe000000010724e00(mirror/gm0p5))
g_detach(0xe0000000108fc700)
g_destroy_consumer(0xe0000000108fc700)
g_destroy_geom(0xe000000010762a00(mirror/gm0p4))
g_detach(0xe0000000108fc480)
g_destroy_consumer(0xe0000000108fc480)
g_destroy_geom(0xe00000001075ed00(mirror/gm0p3))
g_detach(0xe0000000108fc200)
g_destroy_consumer(0xe0000000108fc200)
g_destroy_geom(0xe00000001072c800(mirror/gm0p2))
g_detach(0xe000000010739c80)
g_destroy_consumer(0xe000000010739c80)
g_destroy_geom(0xe000000010763a00(mirror/gm0p1))

# ls -al /dev/mirror/
total 1
dr-xr-xr-x  2 root  wheel          512 Jun 25 11:16 .
dr-xr-xr-x  9 root  wheel          512 Jun 25 11:22 ..
crw-r-----  1 root  operator    0, 105 Jun 25 11:16 gm0
crw-r-----  1 root  operator    0, 116 Jun 25 11:16 gm0p1
crw-r-----  1 root  operator    0, 117 Jun 25 11:16 gm0p2
crw-r-----  1 root  operator    0, 118 Jun 25 11:16 gm0p3
crw-r-----  1 root  operator    0, 119 Jun 25 11:16 gm0p4
crw-r-----  1 root  operator    0, 120 Jun 25 11:16 gm0p5
crw-r-----  1 root  operator    0, 121 Jun 25 11:16 gm0p6
#

#echo 'geom_mirror_load="YES"' >> /boot/loader.conf

# cat /boot/loader.conf
vfs.root.mountfrom="ufs:/dev/da0p2"
geom_mirror_load="YES"
#


On shutdown I see on console:

[skip]
g_detach(0xe0000000106c7000)
g_destroy_consumer(0xe0000000106c7000)
g_destroy_geom(0xe0000000106c3200(mirror/gm0p2))
g_destroy_geom(0xe000000010828200(mirror/gm0))
g_post_event_x(0xe000000004c33e70, 0xe0000000106c6d80, 2, 0)
g_wither_geom(0xe0000000106efa00(gm0.sync))

GEOM_MIRROR: Device gm0 destroyed.
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
g_wither_geom(0xe000000010826900(gm0))
g_detach(0xe0000000106c6d80)
g_destroy_consumer(0xe0000000106c6d80)
g_destroy_geom(0xe0000000106efa00(gm0.sync))
g_destroy_geom(0xe000000010826900(gm0))

And when the system is rebooted, there is no /dev/mirror anymore.

Please advise

many thanks

--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: [phone]
Fax: [phone]
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Marcel Moolenaar-4]

On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
> dev_taste(DEV,mirror/gm0)
> g_part_taste(PART,mirror/gm0)
>
> GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
> GEOM: mirror/gm0: using the primary only -- recovery suggested.
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

You created the mirror after the GPT, which means you destroyed
the GPT backup header. gmirror uses the last sector on the disk
for metadata and that by itself is a cause for various problems.

It's better to use gmirror per partition.

>
> #echo 'geom_mirror_load="YES"' >> /boot/loader.conf

Is /boot a symlink for /efi/boot?

> GEOM_MIRROR: Device gm0 destroyed.
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

This is normal.

>
> And when the system is rebooted, there is no /dev/mirror anymore.

You could run into a race condition between GPT and gmirror and
GPT winning (again the result of gmirror using the last sector
on a disk for metadata).

Alternatively, make sure gmirror got loaded at boot.

FYI,

--
Marcel Moolenaar
xcllnt@...



_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Marcel Moolenaar-4]

On Jun 27, 2009, at 4:15 AM, Ivan Voras wrote:

> Marcel Moolenaar wrote:
>> On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
>>> dev_taste(DEV,mirror/gm0)
>>> g_part_taste(PART,mirror/gm0)
>>>
>>> GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
>>> GEOM: mirror/gm0: using the primary only -- recovery suggested.
>>> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>> You created the mirror after the GPT, which means you destroyed
>> the GPT backup header. gmirror uses the last sector on the disk
>> for metadata and that by itself is a cause for various problems.
>> It's better to use gmirror per partition.
>
> Or create the GPT partition inside the gmirror device - then the GPT  
> backup table will be at last_sector-1, but...
>
>> You could run into a race condition between GPT and gmirror and
>> GPT winning (again the result of gmirror using the last sector
>> on a disk for metadata).
>
> unfortunately this could still happen, and will lead to the same  
> error if GPT is tasted first, since it is embedded in the first  
> sector and will assume the whole drive is available to GPT, and will  
> then proceed to not find its backup data in the last sector.
>
> It looks to me like GEOM classes should have a "priority" field for  
> tasting. Any objections to that idea?

Using the last sector is not only flawed because it creates a race
condition, it's flawed in the assumption that you can always make
a geom part of a mirror by storing meta-data on the geom without
causing corruption. This whole idea of using the last sector was
so that a fully partitioned disk with data could be turned into a
mirrored disk. A neat idea, but hardly the basis for a generic
mirroring implementation when it silently corrupts a disk.

I think it's better to change gmirror to use the first sector on the
provider. This never creates a race condition and as such, you don't
need to invent a priority scheme, that has it's own set of flaws on
top of it. The only downside is that it's not easy to make a fully
partitioned and populated disk part of a mirror: one would need to
move the data forward one sector to free the first sector. This we
can actually do by inserting a GEOM that does it while I/O is still
ongoing. The good thing is: we need a class that does exactly this
for implementing the "move" verb in gpart.

In other words: Solving the problem that putting the metadata in the
first sector creates, can and will be re-used in implementing the
gpart "move partition" feature. I doubt anyone will complain that
the creation of a mirror brings with it a few hours of disk activity
that does not inhibit normal operation...

--
Marcel Moolenaar
xcllnt@...



_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Pawel Jakub Dawidek]

On Sat, Jun 27, 2009 at 06:20:49PM -0700, Marcel Moolenaar wrote:
> Using the last sector is not only flawed because it creates a race
> condition, it's flawed in the assumption that you can always make
> a geom part of a mirror by storing meta-data on the geom without
> causing corruption. This whole idea of using the last sector was
> so that a fully partitioned disk with data could be turned into a
> mirrored disk. A neat idea, but hardly the basis for a generic
> mirroring implementation when it silently corrupts a disk.

This wasn't the idea:) People started putting gmirror on top of
partitioned disk, because it was easier/simpler/faster than creating
mirror, partitioning and copying the data. I for one never put mirror on
already partitioned disk. Although it is sometimes safe to use the last
sector.  Gjournal already looks for UFS and if UFS is in place, it
figures out if the last sector is in use - it isn't if partition size is
not multiple of UFS block size.

> I think it's better to change gmirror to use the first sector on the
> provider. This never creates a race condition and as such, you don't
> need to invent a priority scheme, that has it's own set of flaws on
> top of it. The only downside is that it's not easy to make a fully
> partitioned and populated disk part of a mirror: one would need to
> move the data forward one sector to free the first sector. This we
> can actually do by inserting a GEOM that does it while I/O is still
> ongoing. The good thing is: we need a class that does exactly this
> for implementing the "move" verb in gpart.

There were two reasons to use the last sector instead of first:

1. You want to be able to boot from gmirror. If all your data will be
   moved forward your boot sectors and kernel will be harder to find.

2. For recovery reasons you may want to turn off gmirror and still be
   able to access your data.

Note that gmirror can handle the case where disk, slice and partition
share the same last sector - it simply stores provider size in its
metadata, so once it gets disk for tasting it detects its too big and
ignores it, then slice will be given for tasting, but it also has larger
size than expected and will be ignored as well. Finally partition will
be tasted and gmirror configured.

--
Pawel Jakub Dawidek                       http://www.wheel.pl
pjd@...                           http://www.FreeBSD.org
FreeBSD committer                         Am I Evil? Yes, I Am!

[Ivan Voras-7]

2009/6/28 Marcel Moolenaar <xcllnt@...>:

> Using the last sector is not only flawed because it creates a race
> condition, it's flawed in the assumption that you can always make
> a geom part of a mirror by storing meta-data on the geom without
> causing corruption. This whole idea of using the last sector was
> so that a fully partitioned disk with data could be turned into a
> mirrored disk. A neat idea, but hardly the basis for a generic
> mirroring implementation when it silently corrupts a disk.
>
> I think it's better to change gmirror to use the first sector on the
> provider.

Yes, it would be cleaner to implement but it would also make the
mirrored devices unbootable.

But maybe the class of users needing the functionality is smaller now.

> This never creates a race condition and as such, you don't
> need to invent a priority scheme, that has it's own set of flaws on
> top of it. The only downside is that it's not easy to make a fully
> partitioned and populated disk part of a mirror: one would need to
> move the data forward one sector to free the first sector. This we
> can actually do by inserting a GEOM that does it while I/O is still
> ongoing. The good thing is: we need a class that does exactly this
> for implementing the "move" verb in gpart.

Looks too complicated and fragile. Maybe there's a need for
metadata-less automatic mirrors in some way, by storing the
configuration somewhere else, possibly in /etc.
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Kamigishi Rei]

Ivan Voras wrote:
> Yes, it would be cleaner to implement but it would also make the
> mirrored devices unbootable.
> But maybe the class of users needing the functionality is smaller now.
>  
Most dedicated server providers can't afford to use hardware RAID
systems because that would drastically increase the price of a single
system; yet many customers want mirroring.
> Looks too complicated and fragile. Maybe there's a need for
> metadata-less automatic mirrors in some way, by storing the
> configuration somewhere else, possibly in /etc.
This might be dangerous in some cases. Imagine booting with two drives
swapped; such a configuration might lead to data corruption on a volume
which was enumerated incorrectly or swapped.

--
Kamigishi Rei
KREI-RIPE
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Anton Shterenlikht]

On Thu, Jun 25, 2009 at 09:41:13AM -0700, Marcel Moolenaar wrote:

>
> On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
> > dev_taste(DEV,mirror/gm0)
> > g_part_taste(PART,mirror/gm0)
> >
> > GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
> > GEOM: mirror/gm0: using the primary only -- recovery suggested.
> > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>
> You created the mirror after the GPT, which means you destroyed
> the GPT backup header. gmirror uses the last sector on the disk
> for metadata and that by itself is a cause for various problems.

So, gmirror cannot be used on ia64 to mirror the boot disk?

Because on ia64 the last sector always contains secondary GPT.
I take it the RAID1 section, 19.4, in FBSD user manual,
was written with i386 or alpha architecture in mind.

> It's better to use gmirror per partition.

how? Is it in the manual? any link?


> > #echo 'geom_mirror_load="YES"' >> /boot/loader.conf
>
> Is /boot a symlink for /efi/boot?

yes,

lrwxr-xr-x   1 root  wheel         8 Jun 25 10:44 boot -> efi/boot


> > And when the system is rebooted, there is no /dev/mirror anymore.
>
> You could run into a race condition between GPT and gmirror and
> GPT winning (again the result of gmirror using the last sector
> on a disk for metadata).
>
> Alternatively, make sure gmirror got loaded at boot.

# kldstat
Id Refs Address            Size     Name
 1    3 0xe000000004000000 ff9c08   kernel
 2    1 0xe000000004ffa000 3c830    geom_mirror.ko
#

It's not that I desperately need to mirror a boot disk, it just that
gmirror looked so easy in the manual, I wanted to give it a go.

Perhaps I can just do a block copy to the second disk, say once a day,
and have it as a backup.

Could you also possibly comment on gvinum on ia64?

many thanks
anton

--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: [phone]
Fax: [phone]
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Lorenzo Perone]

Hi,

> On 28.06.2009, at 10:49, Pawel Jakub Dawidek wrote:
>
>> .... I for one never put mirror on
>> already partitioned disk. Although it is sometimes safe to use the  
>> last
>> sector.  Gjournal already looks for UFS and if UFS is in place, it
>> figures out if the last sector is in use - it isn't if partition  
>> size is
>> not multiple of UFS block size.
>>

does this actually also mean that gmirror used on a partition
(eg mirroring two partitions of two different disks) is
not recommended and is going to write its metadata always
on the last sector of the disk, instead of the last sector of
the partition?

regards,

Lorenzo

_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Anton Shterenlikht]

On Thu, Jun 25, 2009 at 09:41:13AM -0700, Marcel Moolenaar wrote:

>
> On Jun 25, 2009, at 4:02 AM, Anton Shterenlikht wrote:
> > dev_taste(DEV,mirror/gm0)
> > g_part_taste(PART,mirror/gm0)
> >
> > GEOM: mirror/gm0: the secondary GPT table is corrupt or invalid.
> > GEOM: mirror/gm0: using the primary only -- recovery suggested.
> > ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
>
> You created the mirror after the GPT, which means you destroyed
> the GPT backup header. gmirror uses the last sector on the disk
> for metadata and that by itself is a cause for various problems.
>
> It's better to use gmirror per partition.

Like this?

# gmirror label -vb round-robin root /dev/da0p2
gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
#

I've read some boot disk gmirror examples, e.g.

http://people.freebsd.org/~rse/mirror

however, all examples I've seen are for i386, talking about MBR, fdisk
and bsdlabel, so these are not directly applicable to ia64.

Application of gvinum for boot disk on ia64 is not clear either.
It seems gvinum section of the handbook, 21.9, is also based on i386.

Please advise

many thanks
anton

--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: [phone]
Fax: [phone]
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Wojciech Puchar-5]

>> It's better to use gmirror per partition.
>
> Like this?
>
> # gmirror label -vb round-robin root /dev/da0p2
> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
isn't that partition accessed by other process or mounted?
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Anton Shterenlikht]

On Wed, Jul 01, 2009 at 10:00:54PM +0200, Wojciech Puchar wrote:
> >> It's better to use gmirror per partition.
> >
> > Like this?
> >
> > # gmirror label -vb round-robin root /dev/da0p2
> > gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
> isn't that partition accessed by other process or mounted?

should it not be mounted?

Sorry, I was just following the handbook, but I now understand it is
incorrect when it comes to ia64.

many thanks
anton

> _______________________________________________
> freebsd-questions@... mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-questions
> To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: [phone]
Fax: [phone]
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Wojciech Puchar-5]

>>>
>>> # gmirror label -vb round-robin root /dev/da0p2
>>> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
>> isn't that partition accessed by other process or mounted?
>
> should it not be mounted?
yes it should not, no matter what architecture.

_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Anton Shterenlikht]

On Thu, Jul 02, 2009 at 03:48:41PM +0200, Wojciech Puchar wrote:
> >>>
> >>> # gmirror label -vb round-robin root /dev/da0p2
> >>> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
> >> isn't that partition accessed by other process or mounted?
> >
> > should it not be mounted?
> yes it should not, no matter what architecture.

ok, thank you

So how can I gmirror root partition? I can't unmount it, I think.
Perhaps I need to use a single-user mode?


Following is a gpart/gmirror report - some success and problems.

I did a fresh FBSD current install on ia64 on directly attached scsi, da0.

# gpart show
=>      34  35566411  da0  GPT  (17G)
        34    819200    1  efi  (400M)
    819234   1048576    2  freebsd-ufs  (512M)
   1867810   4194304    3  freebsd-swap  (2.0G)
   6062114   2097152    4  freebsd-ufs  (1.0G)
   8159266   2097152    5  freebsd-ufs  (1.0G)
  10256418  25310027    6  freebsd-ufs  (12G)

#

What I want is to mirror the whole of the boot disk to da1, which
is identical to da0, but following Marcel's advice, will apply
gmirror per partition.

So starting with efi partition:

First I create GPT scheme on da1

# gpart create -s gpt da1
da1 created
# gpart show da1
=>      34  35566411  da1  GPT  (17G)
        34  35566411       - free -  (17G)

#

then I create EFI partition of the same size as on the boot disk, da0.

# gpart add -b 34 -s 819200 -t efi da1
da1p1 added
# gpart show da1
=>      34  35566411  da1  GPT  (17G)
        34    819200    1  efi  (400M)
    819234  34747211       - free -  (17G)

#

then I umount /efi so that I can create gmirror label on da0p1.

# umount /efi
# gmirror label -vb round-robin efi /dev/da0p1
Metadata value stored on /dev/da0p1.
Done.
#

Checking gmirror

# gmirror status
      Name    Status  Components
mirror/efi  COMPLETE  da0p1
#

and another check

# gmirror list
Geom name: efi
State: COMPLETE
Components: 1
Balance: round-robin
Slice: 4096
Flags: NONE
GenID: 0
SyncID: 1
ID: 3904698645
Providers:
1. Name: mirror/efi
   Mediasize: 419429888 (400M)
   Sectorsize: 512
   Mode: r0w0e0
Consumers:
1. Name: da0p1
   Mediasize: 419430400 (400M)
   Sectorsize: 512
   Mode: r1w1e1
   State: ACTIVE
   Priority: 0
   Flags: NONE
   GenID: 0
   SyncID: 1
   ID: 1288665799

#

now insert a spare partition, da1p1, into the mirror

# gmirror insert efi /dev/da1p1

status looks fine

# gmirror status
      Name    Status  Components
mirror/efi  DEGRADED  da0p1
                      da1p1 (44%)
# gmirror status
      Name    Status  Components
mirror/efi  DEGRADED  da0p1
                      da1p1 (87%)
# gmirror status
      Name    Status  Components
mirror/efi  COMPLETE  da0p1
                      da1p1
#

and another check

# gmirror list
Geom name: efi
State: COMPLETE
Components: 2
Balance: round-robin
Slice: 4096
Flags: NONE
GenID: 0
SyncID: 1
ID: 3904698645
Providers:
1. Name: mirror/efi
   Mediasize: 419429888 (400M)
   Sectorsize: 512
   Mode: r0w0e0
Consumers:
1. Name: da0p1
   Mediasize: 419430400 (400M)
   Sectorsize: 512
   Mode: r1w1e1
   State: ACTIVE
   Priority: 0
   Flags: NONE
   GenID: 0
   SyncID: 1
   ID: 1288665799
2. Name: da1p1
   Mediasize: 419430400 (400M)
   Sectorsize: 512
   Mode: r1w1e1
   State: ACTIVE
   Priority: 0
   Flags: NONE
   GenID: 0
   SyncID: 1
   ID: 1724596009

#

So far, so good.

Now, I don't need to create the filesystem on the mirror, because EFI
was copied from da0p1 to da1p1.

So, I try to mount /dev/mirror/efi

# mount -t msdosfs /dev/mirror/efi /mnt
# df
Filesystem      1K-blocks   Used    Avail Capacity  Mounted on
/dev/da0p2         507630  35904   431116     8%    /
devfs                   1      1        0   100%    /dev
/dev/da0p5        1012974     12   931926     0%    /tmp
/dev/da0p6       12252370 252608 11019574     2%    /usr
/dev/da0p4        1012974    242   931696     0%    /var
/dev/mirror/efi    409504 163264   246240    40%    /mnt
#

again seems ok

so I proceed to modify /etc/fstab and change da0p1 into mirror/efi

# cat /etc/fstab
# Device        Mountpoint      FStype  Options         Dump    Pass#
/dev/da0p3      none            swap    sw              0       0
/dev/da0p2      /               ufs     rw              1       1
/dev/mirror/efi /efi            msdosfs rw              0       0
^^^^^^^^^^^^^^^
/dev/da0p5      /tmp            ufs     rw              2       2
/dev/da0p6      /usr            ufs     rw              2       2
/dev/da0p4      /var            ufs     rw              2       2
/dev/acd0       /cdrom          cd9660  ro,noauto       0       0
#

now I can try to just mount /efi

# umount /mnt
# mount /efi
# df
Filesystem      1K-blocks   Used    Avail Capacity  Mounted on
/dev/da0p2         507630  35904   431116     8%    /
devfs                   1      1        0   100%    /dev
/dev/da0p5        1012974     12   931926     0%    /tmp
/dev/da0p6       12252370 252608 11019574     2%    /usr
/dev/da0p4        1012974    242   931696     0%    /var
/dev/mirror/efi    409504 163264   246240    40%    /efi
#

good, working!

now to mirror root partition.

My problem is that root is mounted and cannot (?) be unmounted, unlike /efi,
on the live system.

# gpart add -b 819234 -s 1048576 -t freebsd-ufs da1
da1p2 added
#

# gmirror label -vb round-robin root /dev/da0p2
gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
#

If I create gmirror on da1, the spare disk:

# gmirror label -vb round-robin root /dev/da1p2
Metadata value stored on /dev/da0p1.
Done.
#

so that

# gmirror status
       Name    Status  Components
 mirror/efi  COMPLETE  da0p1
                       da1p1
mirror/root  COMPLETE  da1p2

#


then I still cannot insert da0p2


# gmirror insert root da0p2
gmirror: Cannot access provider da0p2.
#

So how can I gmirror root partion on a live system?

Also, if gmirror overwrites the last sector in the partition,
then gmirror per partition cannot be used for /usr, because
secondary GPT is stored in the last sector of /usr. Isn't it?

many thanks
 
--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: [phone]
Fax: [phone]
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Alban Hertroys-3]

On Jul 3, 2009, at 12:12 PM, Anton Shterenlikht wrote:

> now to mirror root partition.
>
> My problem is that root is mounted and cannot (?) be unmounted,  
> unlike /efi,
> on the live system.
>
> # gpart add -b 819234 -s 1048576 -t freebsd-ufs da1
> da1p2 added
> #
>
> # gmirror label -vb round-robin root /dev/da0p2
> gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
> #
>
> If I create gmirror on da1, the spare disk:
>
> # gmirror label -vb round-robin root /dev/da1p2
> Metadata value stored on /dev/da0p1.
> Done.
> #
>
> so that
>
> # gmirror status
>       Name    Status  Components
> mirror/efi  COMPLETE  da0p1
>                       da1p1
> mirror/root  COMPLETE  da1p2
>
> #
>
>
> then I still cannot insert da0p2
>
>
> # gmirror insert root da0p2
> gmirror: Cannot access provider da0p2.
> #
>
> So how can I gmirror root partion on a live system?

You're almost there... I did this a while ago, can't remember when,  
but I just upgraded the system that had this from FreeBSD 6.3 of  
sometime in 2006 to 7.2.

What I believe I did from this point on was:

Copy everything from the root partition to mirror/root.
Modify /etc/fstab to mount root on mirror/root.
Reboot.

Now the original root partition isn't mounted anymore, so we can do  
operate on it's geom stuff.

gmirror insert root da0p2

That should be it.
If that doesn't work you can always boot off a live file-system CD/DVD  
and perform these actions from there. You won't have man pages in that  
case though, or at least I couldn't find a way to read them off the  
DVD last I tried.

One thing I'd like to warn about at this point:
If you ever upgrade to a kernel with a newer geom metadata version and  
that new kernel crashes, you're left with a system where the new  
kernel can't boot at all while the old kernel can't mount the root  
mirror as it's now of a version it can't handle.
You can however mount a single geom provider of that root file system  
(/dev/da1p2 for example) to try to fix things.
That file-system WILL be dirty, but DON'T run fsck on it or you will  
destroy it's contents. That's what happened to my upgrade above...

Thankfully it was only my root partition with hardly any data on it  
and I did make level 0 dumps before the upgrade, but I needed to  
restore that FS from a fixit shell without man pages. Augh!

> many thanks
>
> --
> Anton Shterenlikht
> Room 2.6, Queen's Building
> Mech Eng Dept
> Bristol University
> University Walk, Bristol BS8 1TR, UK
> Tel: [phone]
> Fax: [phone]
> _______________________________________________
> freebsd-current@... mailing list
> http://lists.freebsd.org/mailman/listinfo/freebsd-current
> To unsubscribe, send any mail to "freebsd-current-unsubscribe@...
> "
>
>
>
>

Alban Hertroys

--
If you can't see the forest for the trees,
cut the trees and you'll see there is no forest.


!DSPAM:760,4a4de90f759155226611503!


_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."

[Anton Shterenlikht]

On Fri, Jul 03, 2009 at 01:18:28PM +0200, Alban Hertroys wrote:

> On Jul 3, 2009, at 12:12 PM, Anton Shterenlikht wrote:
>
> > now to mirror root partition.
> >
> > My problem is that root is mounted and cannot (?) be unmounted,  
> > unlike /efi,
> > on the live system.
> >
> > # gpart add -b 819234 -s 1048576 -t freebsd-ufs da1
> > da1p2 added
> > #
> >
> > # gmirror label -vb round-robin root /dev/da0p2
> > gmirror: Can't store metadata on /dev/da0p2: Operation not permitted.
> > #
> >
> > If I create gmirror on da1, the spare disk:
> >
> > # gmirror label -vb round-robin root /dev/da1p2
> > Metadata value stored on /dev/da0p1.
> > Done.
> > #
> >
> > so that
> >
> > # gmirror status
> >       Name    Status  Components
> > mirror/efi  COMPLETE  da0p1
> >                       da1p1
> > mirror/root  COMPLETE  da1p2
> >
> > #
> >
> >
> > then I still cannot insert da0p2
> >
> >
> > # gmirror insert root da0p2
> > gmirror: Cannot access provider da0p2.
> > #
> >
> > So how can I gmirror root partion on a live system?
>
> You're almost there... I did this a while ago, can't remember when,  
> but I just upgraded the system that had this from FreeBSD 6.3 of  
> sometime in 2006 to 7.2.
>
> What I believe I did from this point on was:
>
> Copy everything from the root partition to mirror/root.
> Modify /etc/fstab to mount root on mirror/root.
> Reboot.
>
> Now the original root partition isn't mounted anymore, so we can do  
> operate on it's geom stuff.
>
> gmirror insert root da0p2
>
> That should be it.
> If that doesn't work you can always boot off a live file-system CD/DVD  
> and perform these actions from there. You won't have man pages in that  
> case though, or at least I couldn't find a way to read them off the  
> DVD last I tried.
>
> One thing I'd like to warn about at this point:
> If you ever upgrade to a kernel with a newer geom metadata version and  
> that new kernel crashes, you're left with a system where the new  
> kernel can't boot at all while the old kernel can't mount the root  
> mirror as it's now of a version it can't handle.
> You can however mount a single geom provider of that root file system  
> (/dev/da1p2 for example) to try to fix things.
> That file-system WILL be dirty, but DON'T run fsck on it or you will  
> destroy it's contents. That's what happened to my upgrade above...
>
> Thankfully it was only my root partition with hardly any data on it  
> and I did make level 0 dumps before the upgrade, but I needed to  
> restore that FS from a fixit shell without man pages. Augh!

thank you, that was helpful.

I think I've got it, but it's a bit more complex on ia64
because /boot is a symlink to /efi/boot, which is a separate
partition.

Anyway, I've got:

# gmirror status
       Name    Status  Components
 mirror/efi  COMPLETE  da0p1
                       da1p1
mirror/root  COMPLETE  da0p2
                       da1p2
mirror/swap  COMPLETE  da0p3
                       da1p3
 mirror/var  COMPLETE  da1p4
                       da0p4
 mirror/tmp  COMPLETE  da1p5
                       da0p5
 mirror/usr  DEGRADED  da1p6
                       da0p6 (24%)
#

I'll try to write up my experience and post later.

thanks again

--
Anton Shterenlikht
Room 2.6, Queen's Building
Mech Eng Dept
Bristol University
University Walk, Bristol BS8 1TR, UK
Tel: [phone]
Fax: [phone]
_______________________________________________
freebsd-questions@... mailing list
http://lists.freebsd.org/mailman/listinfo/freebsd-questions
To unsubscribe, send any mail to "freebsd-questions-unsubscribe@..."