Coverage for drivers/SR.py : 55%

Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1#!/usr/bin/python3
2#
3# Copyright (C) Citrix Systems Inc.
4#
5# This program is free software; you can redistribute it and/or modify
6# it under the terms of the GNU Lesser General Public License as published
7# by the Free Software Foundation; version 2.1 only.
8#
9# This program is distributed in the hope that it will be useful,
10# but WITHOUT ANY WARRANTY; without even the implied warranty of
11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12# GNU Lesser General Public License for more details.
13#
14# You should have received a copy of the GNU Lesser General Public License
15# along with this program; if not, write to the Free Software Foundation, Inc.,
16# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17#
18# SR: Base class for storage repositories
19#
21import VDI
22import xml.dom.minidom
23import errno
24import xs_errors
25import XenAPI # pylint: disable=import-error
26import xmlrpc.client
27import util
28import copy
29import os
30import traceback
32MOUNT_BASE = '/var/run/sr-mount'
33DEFAULT_TAP = 'vhd'
34TAPDISK_UTIL = '/usr/sbin/td-util'
35MASTER_LVM_CONF = '/etc/lvm/master'
37# LUN per VDI key for XenCenter
38LUNPERVDI = "LUNperVDI"
41class SRException(Exception):
42 """Exception raised by storage repository operations"""
43 errno = errno.EINVAL
45 def __init__(self, reason):
46 Exception.__init__(self, reason)
48 def toxml(self):
49 return xmlrpc.client.dumps(xmlrpc.client.Fault(int(self.errno), str(self)), "", True)
52class SROSError(SRException):
53 """Wrapper for OSError"""
55 def __init__(self, errno, reason):
56 self.errno = errno
57 Exception.__init__(self, reason)
60def deviceCheck(op):
61 def wrapper(self, *args):
62 if 'device' not in self.dconf:
63 raise xs_errors.XenError('ConfigDeviceMissing')
64 return op(self, *args)
65 return wrapper
68backends = []
71def registerSR(SRClass):
72 """Register SR with handler. All SR subclasses should call this in
73 the module file
74 """
75 backends.append(SRClass)
78def driver(type):
79 """Find the SR for the given dconf string"""
80 for d in backends: 80 ↛ 83line 80 didn't jump to line 83, because the loop on line 80 didn't complete
81 if d.handles(type):
82 return d
83 raise xs_errors.XenError('SRUnknownType')
86class SR(object):
87 """Semi-abstract storage repository object.
89 Attributes:
90 uuid: string, UUID
91 label: string
92 description: string
93 vdis: dictionary, VDI objects indexed by UUID
94 physical_utilisation: int, bytes consumed by VDIs
95 virtual_allocation: int, bytes allocated to this repository (virtual)
96 physical_size: int, bytes consumed by this repository
97 sr_vditype: string, repository type
98 """
100 def handles(type):
101 """Returns True if this SR class understands the given dconf string"""
102 return False
103 handles = staticmethod(handles)
105 def __init__(self, srcmd, sr_uuid):
106 """Base class initializer. All subclasses should call SR.__init__
107 in their own
108 initializers.
110 Arguments:
111 srcmd: SRCommand instance, contains parsed arguments
112 """
113 try:
114 self.other_config = {}
115 self.srcmd = srcmd
116 self.dconf = srcmd.dconf
117 if 'session_ref' in srcmd.params:
118 self.session_ref = srcmd.params['session_ref']
119 self.session = XenAPI.xapi_local()
120 self.session._session = self.session_ref
121 if 'subtask_of' in self.srcmd.params: 121 ↛ 122line 121 didn't jump to line 122, because the condition on line 121 was never true
122 self.session.transport.add_extra_header('Subtask-of', self.srcmd.params['subtask_of'])
123 else:
124 self.session = None
126 if 'host_ref' not in self.srcmd.params:
127 self.host_ref = ""
128 else:
129 self.host_ref = self.srcmd.params['host_ref']
131 self.sr_ref = self.srcmd.params.get('sr_ref')
133 if 'device_config' in self.srcmd.params:
134 if self.dconf.get("SRmaster") == "true":
135 os.environ['LVM_SYSTEM_DIR'] = MASTER_LVM_CONF
137 if 'device_config' in self.srcmd.params:
138 if 'SCSIid' in self.srcmd.params['device_config']:
139 dev_path = '/dev/disk/by-scsid/' + self.srcmd.params['device_config']['SCSIid']
140 os.environ['LVM_DEVICE'] = dev_path
141 util.SMlog('Setting LVM_DEVICE to %s' % dev_path)
143 except TypeError:
144 raise Exception(traceback.format_exc())
145 except Exception as e:
146 raise e
147 raise xs_errors.XenError('SRBadXML')
149 self.uuid = sr_uuid
151 self.label = ''
152 self.description = ''
153 self.cmd = srcmd.params['command']
154 self.vdis = {}
155 self.physical_utilisation = 0
156 self.virtual_allocation = 0
157 self.physical_size = 0
158 self.sr_vditype = ''
159 self.passthrough = False
160 # XXX: if this is really needed then we must make a deep copy
161 self.original_srcmd = copy.deepcopy(self.srcmd)
162 self.default_vdi_visibility = True
163 self.scheds = ['none', 'noop']
164 self._mpathinit()
165 self.direct = False
166 self.ops_exclusive = []
167 self.driver_config = {}
169 self.load(sr_uuid)
171 @staticmethod
172 def from_uuid(session, sr_uuid):
173 import imp
175 _SR = session.xenapi.SR
176 sr_ref = _SR.get_by_uuid(sr_uuid)
177 sm_type = _SR.get_type(sr_ref)
178 # NB. load the SM driver module
180 _SM = session.xenapi.SM
181 sms = _SM.get_all_records_where('field "type" = "%s"' % sm_type)
182 sm_ref, sm = sms.popitem()
183 assert not sms
185 driver_path = _SM.get_driver_filename(sm_ref)
186 driver_real = os.path.realpath(driver_path)
187 module_name = os.path.basename(driver_path)
189 module = imp.load_source(module_name, driver_real)
190 target = driver(sm_type)
191 # NB. get the host pbd's device_config
193 host_ref = util.get_localhost_ref(session)
195 _PBD = session.xenapi.PBD
196 pbds = _PBD.get_all_records_where('field "SR" = "%s" and' % sr_ref +
197 'field "host" = "%s"' % host_ref)
198 pbd_ref, pbd = pbds.popitem()
199 assert not pbds
201 device_config = _PBD.get_device_config(pbd_ref)
202 # NB. make srcmd, to please our supersized SR constructor.
203 # FIXME
205 from SRCommand import SRCommand
206 cmd = SRCommand(module.DRIVER_INFO)
207 cmd.dconf = device_config
208 cmd.params = {'session_ref': session._session,
209 'host_ref': host_ref,
210 'device_config': device_config,
211 'sr_ref': sr_ref,
212 'sr_uuid': sr_uuid,
213 'command': 'nop'}
215 return target(cmd, sr_uuid)
217 def block_setscheduler(self, dev):
218 try:
219 realdev = os.path.realpath(dev)
220 disk = util.diskFromPartition(realdev)
222 # the normal case: the sr default scheduler (typically none/noop),
223 # potentially overridden by SR.other_config:scheduler
224 other_config = self.session.xenapi.SR.get_other_config(self.sr_ref)
225 sched = other_config.get('scheduler')
226 if not sched or sched in self.scheds: 226 ↛ 227line 226 didn't jump to line 227, because the condition on line 226 was never true
227 scheds = self.scheds
228 else:
229 scheds = [sched]
231 # special case: BFQ/CFQ if the underlying disk holds dom0's file systems.
232 if disk in util.dom0_disks(): 232 ↛ 233, 232 ↛ 2352 missed branches: 1) line 232 didn't jump to line 233, because the condition on line 232 was never true, 2) line 232 didn't jump to line 235, because the condition on line 232 was never false
233 scheds = ['bfq', 'cfq']
235 util.SMlog("Block scheduler: %s (%s) wants %s" % (dev, disk, scheds))
236 util.set_scheduler(realdev[5:], scheds)
237 except Exception as e:
238 util.SMlog("Failed to set block scheduler on %s: %s" % (dev, e))
240 def _addLUNperVDIkey(self):
241 try:
242 self.session.xenapi.SR.add_to_sm_config(self.sr_ref, LUNPERVDI, "true")
243 except:
244 pass
246 def create(self, uuid, size):
247 """Create this repository.
248 This operation may delete existing data.
250 The operation is NOT idempotent. The operation will fail
251 if an SR of the same UUID and driver type already exits.
253 Returns:
254 None
255 Raises:
256 SRUnimplementedMethod
257 """
258 raise xs_errors.XenError('Unimplemented')
260 def delete(self, uuid):
261 """Delete this repository and its contents.
263 This operation IS idempotent -- it will succeed if the repository
264 exists and can be deleted or if the repository does not exist.
265 The caller must ensure that all VDIs are deactivated and detached
266 and that the SR itself has been detached before delete().
267 The call will FAIL if any VDIs in the SR are in use.
269 Returns:
270 None
271 Raises:
272 SRUnimplementedMethod
273 """
274 raise xs_errors.XenError('Unimplemented')
276 def update(self, uuid):
277 """Refresh the fields in the SR object
279 Returns:
280 None
281 Raises:
282 SRUnimplementedMethod
283 """
284 # no-op unless individual backends implement it
285 return
287 def attach(self, uuid):
288 """Initiate local access to the SR. Initialises any
289 device state required to access the substrate.
291 Idempotent.
293 Returns:
294 None
295 Raises:
296 SRUnimplementedMethod
297 """
298 raise xs_errors.XenError('Unimplemented')
300 def after_master_attach(self, uuid):
301 """Perform actions required after attaching on the pool master
302 Return:
303 None
304 """
305 try:
306 self.scan(uuid)
307 except Exception as e:
308 util.SMlog("Error in SR.after_master_attach %s" % e)
309 msg_name = "POST_ATTACH_SCAN_FAILED"
310 msg_body = "Failed to scan SR %s after attaching, " \
311 "error %s" % (uuid, e)
312 self.session.xenapi.message.create(
313 msg_name, 2, "SR", uuid, msg_body)
315 def detach(self, uuid):
316 """Remove local access to the SR. Destroys any device
317 state initiated by the sr_attach() operation.
319 Idempotent. All VDIs must be detached in order for the operation
320 to succeed.
322 Returns:
323 None
324 Raises:
325 SRUnimplementedMethod
326 """
327 raise xs_errors.XenError('Unimplemented')
329 def probe(self):
330 """Perform a backend-specific scan, using the current dconf. If the
331 dconf is complete, then this will return a list of the SRs present of
332 this type on the device, if any. If the dconf is partial, then a
333 backend-specific scan will be performed, returning results that will
334 guide the user in improving the dconf.
336 Idempotent.
338 xapi will ensure that this is serialised wrt any other probes, or
339 attach or detach operations on this host.
341 Returns:
342 An XML fragment containing the scan results. These are specific
343 to the scan being performed, and the current backend.
344 Raises:
345 SRUnimplementedMethod
346 """
347 raise xs_errors.XenError('Unimplemented')
349 def scan(self, uuid):
350 """
351 Returns:
352 """
353 # Update SR parameters
354 self._db_update()
355 # Synchronise VDI list
356 scanrecord = ScanRecord(self)
357 scanrecord.synchronise()
359 def replay(self, uuid):
360 """Replay a multi-stage log entry
362 Returns:
363 None
364 Raises:
365 SRUnimplementedMethod
366 """
367 raise xs_errors.XenError('Unimplemented')
369 def content_type(self, uuid):
370 """Returns the 'content_type' of an SR as a string"""
371 return xmlrpc.client.dumps((str(self.sr_vditype), ), "", True)
373 def load(self, sr_uuid):
374 """Post-init hook"""
375 pass
377 def vdi(self, uuid):
378 """Return VDI object owned by this repository"""
379 if uuid not in self.vdis:
380 self.vdis[uuid] = VDI.VDI(self, uuid)
381 raise xs_errors.XenError('Unimplemented')
382 return self.vdis[uuid]
384 def forget_vdi(self, uuid):
385 vdi = self.session.xenapi.VDI.get_by_uuid(uuid)
386 self.session.xenapi.VDI.db_forget(vdi)
388 def cleanup(self):
389 # callback after the op is done
390 pass
392 def _db_update(self):
393 sr = self.session.xenapi.SR.get_by_uuid(self.uuid)
394 self.session.xenapi.SR.set_virtual_allocation(sr, str(self.virtual_allocation))
395 self.session.xenapi.SR.set_physical_size(sr, str(self.physical_size))
396 self.session.xenapi.SR.set_physical_utilisation(sr, str(self.physical_utilisation))
398 def _toxml(self):
399 dom = xml.dom.minidom.Document()
400 element = dom.createElement("sr")
401 dom.appendChild(element)
403 # Add default uuid, physical_utilisation, physical_size and
404 # virtual_allocation entries
405 for attr in ('uuid', 'physical_utilisation', 'virtual_allocation',
406 'physical_size'):
407 try:
408 aval = getattr(self, attr)
409 except AttributeError:
410 raise xs_errors.XenError(
411 'InvalidArg', opterr='Missing required field [%s]' % attr)
413 entry = dom.createElement(attr)
414 element.appendChild(entry)
415 textnode = dom.createTextNode(str(aval))
416 entry.appendChild(textnode)
418 # Add the default_vdi_visibility entry
419 entry = dom.createElement('default_vdi_visibility')
420 element.appendChild(entry)
421 if not self.default_vdi_visibility:
422 textnode = dom.createTextNode('False')
423 else:
424 textnode = dom.createTextNode('True')
425 entry.appendChild(textnode)
427 # Add optional label and description entries
428 for attr in ('label', 'description'):
429 try:
430 aval = getattr(self, attr)
431 except AttributeError:
432 continue
433 if aval:
434 entry = dom.createElement(attr)
435 element.appendChild(entry)
436 textnode = dom.createTextNode(str(aval))
437 entry.appendChild(textnode)
439 # Create VDI sub-list
440 if self.vdis:
441 for uuid in self.vdis:
442 if not self.vdis[uuid].deleted:
443 vdinode = dom.createElement("vdi")
444 element.appendChild(vdinode)
445 self.vdis[uuid]._toxml(dom, vdinode)
447 return dom
449 def _fromxml(self, str, tag):
450 dom = xml.dom.minidom.parseString(str)
451 objectlist = dom.getElementsByTagName(tag)[0]
452 taglist = {}
453 for node in objectlist.childNodes:
454 taglist[node.nodeName] = ""
455 for n in node.childNodes:
456 if n.nodeType == n.TEXT_NODE:
457 taglist[node.nodeName] += n.data
458 return taglist
460 def _splitstring(self, str):
461 elementlist = []
462 for i in range(0, len(str)):
463 elementlist.append(str[i])
464 return elementlist
466 def _mpathinit(self):
467 self.mpath = "false"
468 try:
469 if 'multipathing' in self.dconf and \ 469 ↛ 471line 469 didn't jump to line 471, because the condition on line 469 was never true
470 'multipathhandle' in self.dconf:
471 self.mpath = self.dconf['multipathing']
472 self.mpathhandle = self.dconf['multipathhandle']
473 else:
474 hconf = self.session.xenapi.host.get_other_config(self.host_ref)
475 self.mpath = hconf['multipathing']
476 self.mpathhandle = hconf.get('multipathhandle', 'dmp')
478 if self.mpath != "true": 478 ↛ 482line 478 didn't jump to line 482, because the condition on line 478 was never false
479 self.mpath = "false"
480 self.mpathhandle = "null"
482 if not os.path.exists("/opt/xensource/sm/mpath_%s.py" % self.mpathhandle): 482 ↛ 487line 482 didn't jump to line 487, because the condition on line 482 was never false
483 raise IOError("File does not exist = %s" % self.mpathhandle)
484 except:
485 self.mpath = "false"
486 self.mpathhandle = "null"
487 module_name = "mpath_%s" % self.mpathhandle
488 self.mpathmodule = __import__(module_name)
490 def _mpathHandle(self):
491 if self.mpath == "true": 491 ↛ 492line 491 didn't jump to line 492, because the condition on line 491 was never true
492 self.mpathmodule.activate()
493 else:
494 self.mpathmodule.deactivate()
496 def _pathrefresh(self, obj):
497 SCSIid = getattr(self, 'SCSIid')
498 self.dconf['device'] = self.mpathmodule.path(SCSIid)
499 super(obj, self).load(self.uuid)
501 def _setMultipathableFlag(self, SCSIid=''):
502 try:
503 sm_config = self.session.xenapi.SR.get_sm_config(self.sr_ref)
504 sm_config['multipathable'] = 'true'
505 self.session.xenapi.SR.set_sm_config(self.sr_ref, sm_config)
507 if self.mpath == "true" and len(SCSIid):
508 cmd = ['/opt/xensource/sm/mpathcount.py', SCSIid]
509 util.pread2(cmd)
510 except:
511 pass
513 def check_dconf(self, key_list, raise_flag=True):
514 """ Checks if all keys in 'key_list' exist in 'self.dconf'.
516 Input:
517 key_list: a list of keys to check if they exist in self.dconf
518 raise_flag: if true, raise an exception if there are 1 or more
519 keys missing
521 Return: set() containing the missing keys (empty set() if all exist)
522 Raise: xs_errors.XenError('ConfigParamsMissing')
523 """
525 missing_keys = {key for key in key_list if key not in self.dconf}
527 if missing_keys and raise_flag:
528 errstr = 'device-config is missing the following parameters: ' + \
529 ', '.join([key for key in missing_keys])
530 raise xs_errors.XenError('ConfigParamsMissing', opterr=errstr)
532 return missing_keys
535class ScanRecord:
536 def __init__(self, sr):
537 self.sr = sr
538 self.__xenapi_locations = {}
539 self.__xenapi_records = util.list_VDI_records_in_sr(sr)
540 for vdi in list(self.__xenapi_records.keys()): 540 ↛ 541line 540 didn't jump to line 541, because the loop on line 540 never started
541 self.__xenapi_locations[util.to_plain_string(self.__xenapi_records[vdi]['location'])] = vdi
542 self.__sm_records = {}
543 for vdi in list(sr.vdis.values()):
544 # We initialise the sm_config field with the values from the database
545 # The sm_config_overrides contains any new fields we want to add to
546 # sm_config, and also any field to delete (by virtue of having
547 # sm_config_overrides[key]=None)
548 try:
549 if not hasattr(vdi, "sm_config"): 549 ↛ 555line 549 didn't jump to line 555, because the condition on line 549 was never false
550 vdi.sm_config = self.__xenapi_records[self.__xenapi_locations[vdi.location]]['sm_config'].copy()
551 except:
552 util.SMlog("missing config for vdi: %s" % vdi.location)
553 vdi.sm_config = {}
555 vdi._override_sm_config(vdi.sm_config)
557 self.__sm_records[vdi.location] = vdi
559 xenapi_locations = set(self.__xenapi_locations.keys())
560 sm_locations = set(self.__sm_records.keys())
562 # These ones are new on disk
563 self.new = sm_locations.difference(xenapi_locations)
564 # These have disappeared from the disk
565 self.gone = xenapi_locations.difference(sm_locations)
566 # These are the ones which are still present but might have changed...
567 existing = sm_locations.intersection(xenapi_locations)
568 # Synchronise the uuid fields using the location as the primary key
569 # This ensures we know what the UUIDs are even though they aren't stored
570 # in the storage backend.
571 for location in existing: 571 ↛ 572line 571 didn't jump to line 572, because the loop on line 571 never started
572 sm_vdi = self.get_sm_vdi(location)
573 xenapi_vdi = self.get_xenapi_vdi(location)
574 sm_vdi.uuid = util.default(sm_vdi, "uuid", lambda: xenapi_vdi['uuid'])
576 # Only consider those whose configuration looks different
577 self.existing = [x for x in existing if not(self.get_sm_vdi(x).in_sync_with_xenapi_record(self.get_xenapi_vdi(x)))]
579 if len(self.new) != 0:
580 util.SMlog("new VDIs on disk: " + repr(self.new))
581 if len(self.gone) != 0: 581 ↛ 582line 581 didn't jump to line 582, because the condition on line 581 was never true
582 util.SMlog("VDIs missing from disk: " + repr(self.gone))
583 if len(self.existing) != 0: 583 ↛ 584line 583 didn't jump to line 584, because the condition on line 583 was never true
584 util.SMlog("VDIs changed on disk: " + repr(self.existing))
586 def get_sm_vdi(self, location):
587 return self.__sm_records[location]
589 def get_xenapi_vdi(self, location):
590 return self.__xenapi_records[self.__xenapi_locations[location]]
592 def all_xenapi_locations(self):
593 return set(self.__xenapi_locations.keys())
595 def synchronise_new(self):
596 """Add XenAPI records for new disks"""
597 for location in self.new:
598 vdi = self.get_sm_vdi(location)
599 util.SMlog("Introducing VDI with location=%s" % (vdi.location))
600 vdi._db_introduce()
602 def synchronise_gone(self):
603 """Delete XenAPI record for old disks"""
604 for location in self.gone: 604 ↛ 605line 604 didn't jump to line 605, because the loop on line 604 never started
605 vdi = self.get_xenapi_vdi(location)
606 util.SMlog("Forgetting VDI with location=%s uuid=%s" % (util.to_plain_string(vdi['location']), vdi['uuid']))
607 try:
608 self.sr.forget_vdi(vdi['uuid'])
609 except XenAPI.Failure as e:
610 if util.isInvalidVDI(e):
611 util.SMlog("VDI %s not found, ignoring exception" %
612 vdi['uuid'])
613 else:
614 raise
616 def synchronise_existing(self):
617 """Update existing XenAPI records"""
618 for location in self.existing: 618 ↛ 619line 618 didn't jump to line 619, because the loop on line 618 never started
619 vdi = self.get_sm_vdi(location)
621 util.SMlog("Updating VDI with location=%s uuid=%s" % (vdi.location, vdi.uuid))
622 vdi._db_update()
624 def synchronise(self):
625 """Perform the default SM -> xenapi synchronisation; ought to be good enough
626 for most plugins."""
627 self.synchronise_new()
628 self.synchronise_gone()
629 self.synchronise_existing()