1#!/usr/bin/env python3 

2# 

3# Copyright (C) 2020 Vates SAS - ronan.abhamon@vates.fr 

4# 

5# This program is free software: you can redistribute it and/or modify 

6# it under the terms of the GNU General Public License as published by 

7# the Free Software Foundation, either version 3 of the License, or 

8# (at your option) any later version. 

9# This program is distributed in the hope that it will be useful, 

10# but WITHOUT ANY WARRANTY; without even the implied warranty of 

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

12# GNU General Public License for more details. 

13# 

14# You should have received a copy of the GNU General Public License 

15# along with this program. If not, see <https://www.gnu.org/licenses/>. 

16 

17from sm_typing import override 

18 

19from linstorjournaler import LinstorJournaler 

20from linstorvolumemanager import LinstorVolumeManager 

21import base64 

22import errno 

23import json 

24import socket 

25import time 

26import util 

27import vhdutil 

28import xs_errors 

29 

30MANAGER_PLUGIN = 'linstor-manager' 

31 

32 

33def call_remote_method(session, host_ref, method, args): 

34 try: 

35 response = session.xenapi.host.call_plugin( 

36 host_ref, MANAGER_PLUGIN, method, args 

37 ) 

38 except Exception as e: 

39 util.SMlog('call-plugin on {} ({} with {}) exception: {}'.format( 

40 host_ref, method, args, e 

41 )) 

42 raise util.SMException(str(e)) 

43 

44 util.SMlog('call-plugin on {} ({} with {}) returned: {}'.format( 

45 host_ref, method, args, response 

46 )) 

47 

48 return response 

49 

50 

51def check_ex(path, ignoreMissingFooter = False, fast = False): 

52 cmd = [vhdutil.VHD_UTIL, "check", vhdutil.OPT_LOG_ERR, "-n", path] 

53 if ignoreMissingFooter: 

54 cmd.append("-i") 

55 if fast: 

56 cmd.append("-B") 

57 

58 vhdutil.ioretry(cmd) 

59 

60 

61class LinstorCallException(util.SMException): 

62 def __init__(self, cmd_err): 

63 self.cmd_err = cmd_err 

64 

65 @override 

66 def __str__(self) -> str: 

67 return str(self.cmd_err) 

68 

69 

70class ErofsLinstorCallException(LinstorCallException): 

71 pass 

72 

73 

74class NoPathLinstorCallException(LinstorCallException): 

75 pass 

76 

77def log_successful_call(target_host, device_path, vdi_uuid, remote_method, response): 

78 util.SMlog('Successful access on {} for device {} ({}): `{}` => {}'.format( 

79 target_host, device_path, vdi_uuid, remote_method, str(response) 

80 ), priority=util.LOG_DEBUG) 

81 

82def log_failed_call(target_host, next_target, device_path, vdi_uuid, remote_method, e): 

83 util.SMlog('Failed to call method on {} for device {} ({}): {}. Trying accessing on {}... (cause: {})'.format( 

84 target_host, device_path, vdi_uuid, remote_method, next_target, e 

85 ), priority=util.LOG_DEBUG) 

86 

87def linstorhostcall(local_method, remote_method): 

88 def decorated(response_parser): 

89 def wrapper(*args, **kwargs): 

90 self = args[0] 

91 vdi_uuid = args[1] 

92 

93 device_path = self._linstor.build_device_path( 

94 self._linstor.get_volume_name(vdi_uuid) 

95 ) 

96 

97 if not self._session: 

98 return self._call_local_method(local_method, device_path, *args[2:], **kwargs) 

99 

100 remote_args = { 

101 'devicePath': device_path, 

102 'groupName': self._linstor.group_name 

103 } 

104 remote_args.update(**kwargs) 

105 remote_args = {str(key): str(value) for key, value in remote_args.items()} 

106 

107 this_host_ref = util.get_this_host_ref(self._session) 

108 def call_method(host_label, host_ref): 

109 if host_ref == this_host_ref: 

110 return self._call_local_method(local_method, device_path, *args[2:], **kwargs) 

111 response = call_remote_method(self._session, host_ref, remote_method, remote_args) 

112 log_successful_call(host_label, device_path, vdi_uuid, remote_method, response) 

113 return response_parser(self, vdi_uuid, response) 

114 

115 # 1. Try on attached host. 

116 try: 

117 host_ref_attached = next(iter(util.get_hosts_attached_on(self._session, [vdi_uuid])), None) 

118 if host_ref_attached: 

119 return call_method('attached host', host_ref_attached) 

120 except Exception as e: 

121 log_failed_call('attached host', 'master', device_path, vdi_uuid, remote_method, e) 

122 

123 # 2. Try on master host. 

124 try: 

125 return call_method('master', util.get_master_ref(self._session)) 

126 except Exception as e: 

127 log_failed_call('master', 'primary', device_path, vdi_uuid, remote_method, e) 

128 

129 # 3. Try on a primary. 

130 hosts = self._get_hosts(remote_method, device_path) 

131 

132 nodes, primary_hostname = self._linstor.find_up_to_date_diskful_nodes(vdi_uuid) 

133 if primary_hostname: 

134 try: 

135 return call_method('primary', self._find_host_ref_from_hostname(hosts, primary_hostname)) 

136 except Exception as remote_e: 

137 self._raise_openers_exception(device_path, remote_e) 

138 

139 log_failed_call('primary', 'another node', device_path, vdi_uuid, remote_method, 'no primary') 

140 

141 # 4. Try on any host with local data. 

142 try: 

143 return call_method('another node', next(filter(None, 

144 (self._find_host_ref_from_hostname(hosts, hostname) for hostname in nodes) 

145 ), None)) 

146 except Exception as remote_e: 

147 self._raise_openers_exception(device_path, remote_e) 

148 

149 return wrapper 

150 return decorated 

151 

152 

153def linstormodifier(): 

154 def decorated(func): 

155 def wrapper(*args, **kwargs): 

156 self = args[0] 

157 

158 ret = func(*args, **kwargs) 

159 self._linstor.invalidate_resource_cache() 

160 return ret 

161 return wrapper 

162 return decorated 

163 

164 

165class LinstorVhdUtil: 

166 MAX_SIZE = 2 * 1024 * 1024 * 1024 * 1024 # Max VHD size. 

167 

168 def __init__(self, session, linstor): 

169 self._session = session 

170 self._linstor = linstor 

171 

172 def create_chain_paths(self, vdi_uuid, readonly=False): 

173 # OPTIMIZE: Add a limit_to_first_allocated_block param to limit vhdutil calls. 

174 # Useful for the snapshot code algorithm. 

175 

176 leaf_vdi_path = self._linstor.get_device_path(vdi_uuid) 

177 path = leaf_vdi_path 

178 while True: 

179 if not util.pathexists(path): 

180 raise xs_errors.XenError( 

181 'VDIUnavailable', opterr='Could not find: {}'.format(path) 

182 ) 

183 

184 # Diskless path can be created on the fly, ensure we can open it. 

185 def check_volume_usable(): 

186 while True: 

187 try: 

188 with open(path, 'r' if readonly else 'r+'): 

189 pass 

190 except IOError as e: 

191 if e.errno == errno.ENODATA: 

192 time.sleep(2) 

193 continue 

194 if e.errno == errno.EROFS or e.errno == errno.EMEDIUMTYPE: 

195 util.SMlog('Volume not attachable because used. Openers: {}'.format( 

196 self._linstor.get_volume_openers(vdi_uuid) 

197 )) 

198 raise 

199 break 

200 util.retry(check_volume_usable, 15, 2) 

201 

202 vdi_uuid = self.get_vhd_info(vdi_uuid).parentUuid 

203 if not vdi_uuid: 

204 break 

205 path = self._linstor.get_device_path(vdi_uuid) 

206 readonly = True # Non-leaf is always readonly. 

207 

208 return leaf_vdi_path 

209 

210 # -------------------------------------------------------------------------- 

211 # Getters: read locally and try on another host in case of failure. 

212 # -------------------------------------------------------------------------- 

213 

214 def check(self, vdi_uuid, ignore_missing_footer=False, fast=False): 

215 kwargs = { 

216 'ignoreMissingFooter': ignore_missing_footer, 

217 'fast': fast 

218 } 

219 try: 

220 self._check(vdi_uuid, **kwargs) 

221 return True 

222 except Exception as e: 

223 util.SMlog('Call to `check` failed: {}'.format(e)) 

224 return False 

225 

226 @linstorhostcall(check_ex, 'check') 

227 def _check(self, vdi_uuid, response): 

228 return util.strtobool(response) 

229 

230 def get_vhd_info(self, vdi_uuid, include_parent=True): 

231 kwargs = { 

232 'includeParent': include_parent, 

233 'resolveParent': False 

234 } 

235 return self._get_vhd_info(vdi_uuid, self._extract_uuid, **kwargs) 

236 

237 @linstorhostcall(vhdutil.getVHDInfo, 'getVHDInfo') 

238 def _get_vhd_info(self, vdi_uuid, response): 

239 obj = json.loads(response) 

240 

241 vhd_info = vhdutil.VHDInfo(vdi_uuid) 

242 vhd_info.sizeVirt = obj['sizeVirt'] 

243 vhd_info.sizePhys = obj['sizePhys'] 

244 if 'parentPath' in obj: 

245 vhd_info.parentPath = obj['parentPath'] 

246 vhd_info.parentUuid = obj['parentUuid'] 

247 vhd_info.hidden = obj['hidden'] 

248 vhd_info.path = obj['path'] 

249 

250 return vhd_info 

251 

252 @linstorhostcall(vhdutil.hasParent, 'hasParent') 

253 def has_parent(self, vdi_uuid, response): 

254 return util.strtobool(response) 

255 

256 def get_parent(self, vdi_uuid): 

257 return self._get_parent(vdi_uuid, self._extract_uuid) 

258 

259 @linstorhostcall(vhdutil.getParent, 'getParent') 

260 def _get_parent(self, vdi_uuid, response): 

261 return response 

262 

263 @linstorhostcall(vhdutil.getSizeVirt, 'getSizeVirt') 

264 def get_size_virt(self, vdi_uuid, response): 

265 return int(response) 

266 

267 @linstorhostcall(vhdutil.getMaxResizeSize, 'getMaxResizeSize') 

268 def get_max_resize_size(self, vdi_uuid, response): 

269 return int(response) 

270 

271 @linstorhostcall(vhdutil.getSizePhys, 'getSizePhys') 

272 def get_size_phys(self, vdi_uuid, response): 

273 return int(response) 

274 

275 @linstorhostcall(vhdutil.getAllocatedSize, 'getAllocatedSize') 

276 def get_allocated_size(self, vdi_uuid, response): 

277 return int(response) 

278 

279 @linstorhostcall(vhdutil.getDepth, 'getDepth') 

280 def get_depth(self, vdi_uuid, response): 

281 return int(response) 

282 

283 @linstorhostcall(vhdutil.getKeyHash, 'getKeyHash') 

284 def get_key_hash(self, vdi_uuid, response): 

285 return response or None 

286 

287 @linstorhostcall(vhdutil.getBlockBitmap, 'getBlockBitmap') 

288 def get_block_bitmap(self, vdi_uuid, response): 

289 return base64.b64decode(response) 

290 

291 @linstorhostcall('_get_drbd_size', 'getDrbdSize') 

292 def get_drbd_size(self, vdi_uuid, response): 

293 return int(response) 

294 

295 def _get_drbd_size(self, path): 

296 (ret, stdout, stderr) = util.doexec(['blockdev', '--getsize64', path]) 

297 if ret == 0: 

298 return int(stdout.strip()) 

299 raise util.SMException('Failed to get DRBD size: {}'.format(stderr)) 

300 

301 # -------------------------------------------------------------------------- 

302 # Setters: only used locally. 

303 # -------------------------------------------------------------------------- 

304 

305 @linstormodifier() 

306 def create(self, path, size, static, msize=0): 

307 return self._call_local_method_or_fail(vhdutil.create, path, size, static, msize) 

308 

309 @linstormodifier() 

310 def set_size_phys(self, path, size, debug=True): 

311 return self._call_local_method_or_fail(vhdutil.setSizePhys, path, size, debug) 

312 

313 @linstormodifier() 

314 def set_parent(self, path, parentPath, parentRaw=False): 

315 return self._call_local_method_or_fail(vhdutil.setParent, path, parentPath, parentRaw) 

316 

317 @linstormodifier() 

318 def set_hidden(self, path, hidden=True): 

319 return self._call_local_method_or_fail(vhdutil.setHidden, path, hidden) 

320 

321 @linstormodifier() 

322 def set_key(self, path, key_hash): 

323 return self._call_local_method_or_fail(vhdutil.setKey, path, key_hash) 

324 

325 @linstormodifier() 

326 def kill_data(self, path): 

327 return self._call_local_method_or_fail(vhdutil.killData, path) 

328 

329 @linstormodifier() 

330 def snapshot(self, path, parent, parentRaw, msize=0, checkEmpty=True): 

331 return self._call_local_method_or_fail(vhdutil.snapshot, path, parent, parentRaw, msize, checkEmpty) 

332 

333 def inflate(self, journaler, vdi_uuid, vdi_path, new_size, old_size): 

334 # Only inflate if the LINSTOR volume capacity is not enough. 

335 new_size = LinstorVolumeManager.round_up_volume_size(new_size) 

336 if new_size <= old_size: 

337 return 

338 

339 util.SMlog( 

340 'Inflate {} (size={}, previous={})' 

341 .format(vdi_path, new_size, old_size) 

342 ) 

343 

344 journaler.create( 

345 LinstorJournaler.INFLATE, vdi_uuid, old_size 

346 ) 

347 self._linstor.resize_volume(vdi_uuid, new_size) 

348 

349 result_size = self.get_drbd_size(vdi_uuid) 

350 if result_size < new_size: 

351 util.SMlog( 

352 'WARNING: Cannot inflate volume to {}B, result size: {}B' 

353 .format(new_size, result_size) 

354 ) 

355 

356 self._zeroize(vdi_path, result_size - vhdutil.VHD_FOOTER_SIZE) 

357 self.set_size_phys(vdi_path, result_size, False) 

358 journaler.remove(LinstorJournaler.INFLATE, vdi_uuid) 

359 

360 def deflate(self, vdi_path, new_size, old_size, zeroize=False): 

361 if zeroize: 

362 assert old_size > vhdutil.VHD_FOOTER_SIZE 

363 self._zeroize(vdi_path, old_size - vhdutil.VHD_FOOTER_SIZE) 

364 

365 new_size = LinstorVolumeManager.round_up_volume_size(new_size) 

366 if new_size >= old_size: 

367 return 

368 

369 util.SMlog( 

370 'Deflate {} (new size={}, previous={})' 

371 .format(vdi_path, new_size, old_size) 

372 ) 

373 

374 self.set_size_phys(vdi_path, new_size) 

375 # TODO: Change the LINSTOR volume size using linstor.resize_volume. 

376 

377 # -------------------------------------------------------------------------- 

378 # Remote setters: write locally and try on another host in case of failure. 

379 # -------------------------------------------------------------------------- 

380 

381 @linstormodifier() 

382 def set_size_virt(self, path, size, jfile): 

383 kwargs = { 

384 'size': size, 

385 'jfile': jfile 

386 } 

387 return self._call_method(vhdutil.setSizeVirt, 'setSizeVirt', path, use_parent=False, **kwargs) 

388 

389 @linstormodifier() 

390 def set_size_virt_fast(self, path, size): 

391 kwargs = { 

392 'size': size 

393 } 

394 return self._call_method(vhdutil.setSizeVirtFast, 'setSizeVirtFast', path, use_parent=False, **kwargs) 

395 

396 @linstormodifier() 

397 def force_parent(self, path, parentPath, parentRaw=False): 

398 kwargs = { 

399 'parentPath': str(parentPath), 

400 'parentRaw': parentRaw 

401 } 

402 return self._call_method(vhdutil.setParent, 'setParent', path, use_parent=False, **kwargs) 

403 

404 @linstormodifier() 

405 def force_coalesce(self, path): 

406 return int(self._call_method(vhdutil.coalesce, 'coalesce', path, use_parent=True)) 

407 

408 @linstormodifier() 

409 def force_repair(self, path): 

410 return self._call_method(vhdutil.repair, 'repair', path, use_parent=False) 

411 

412 @linstormodifier() 

413 def force_deflate(self, path, newSize, oldSize, zeroize): 

414 kwargs = { 

415 'newSize': newSize, 

416 'oldSize': oldSize, 

417 'zeroize': zeroize 

418 } 

419 return self._call_method('_force_deflate', 'deflate', path, use_parent=False, **kwargs) 

420 

421 def _force_deflate(self, path, newSize, oldSize, zeroize): 

422 self.deflate(path, newSize, oldSize, zeroize) 

423 

424 # -------------------------------------------------------------------------- 

425 # Static helpers. 

426 # -------------------------------------------------------------------------- 

427 

428 @classmethod 

429 def compute_volume_size(cls, virtual_size, image_type): 

430 if image_type == vhdutil.VDI_TYPE_VHD: 

431 # All LINSTOR VDIs have the metadata area preallocated for 

432 # the maximum possible virtual size (for fast online VDI.resize). 

433 meta_overhead = vhdutil.calcOverheadEmpty(cls.MAX_SIZE) 

434 bitmap_overhead = vhdutil.calcOverheadBitmap(virtual_size) 

435 virtual_size += meta_overhead + bitmap_overhead 

436 elif image_type != vhdutil.VDI_TYPE_RAW: 

437 raise Exception('Invalid image type: {}'.format(image_type)) 

438 

439 return LinstorVolumeManager.round_up_volume_size(virtual_size) 

440 

441 # -------------------------------------------------------------------------- 

442 # Helpers. 

443 # -------------------------------------------------------------------------- 

444 

445 def _extract_uuid(self, device_path): 

446 # TODO: Remove new line in the vhdutil module. Not here. 

447 return self._linstor.get_volume_uuid_from_device_path( 

448 device_path.rstrip('\n') 

449 ) 

450 

451 def _get_hosts(self, remote_method, device_path): 

452 try: 

453 return self._session.xenapi.host.get_all_records() 

454 except Exception as e: 

455 raise xs_errors.XenError( 

456 'VDIUnavailable', 

457 opterr='Unable to get host list to run vhdutil command `{}` (path={}): {}' 

458 .format(remote_method, device_path, e) 

459 ) 

460 

461 # -------------------------------------------------------------------------- 

462 

463 @staticmethod 

464 def _find_host_ref_from_hostname(hosts, hostname): 

465 return next((ref for ref, rec in hosts.items() if rec['hostname'] == hostname), None) 

466 

467 def _raise_openers_exception(self, device_path, e): 

468 if isinstance(e, util.CommandException): 

469 e_str = 'cmd: `{}`, code: `{}`, reason: `{}`'.format(e.cmd, e.code, e.reason) 

470 else: 

471 e_str = str(e) 

472 

473 try: 

474 volume_uuid = self._linstor.get_volume_uuid_from_device_path( 

475 device_path 

476 ) 

477 e_wrapper = Exception( 

478 e_str + ' (openers: {})'.format( 

479 self._linstor.get_volume_openers(volume_uuid) 

480 ) 

481 ) 

482 except Exception as illformed_e: 

483 e_wrapper = Exception( 

484 e_str + ' (unable to get openers: {})'.format(illformed_e) 

485 ) 

486 util.SMlog('raise opener exception: {}'.format(e_wrapper)) 

487 raise e_wrapper # pylint: disable = E0702 

488 

489 def _call_local_method(self, local_method, device_path, *args, **kwargs): 

490 if isinstance(local_method, str): 

491 local_method = getattr(self, local_method) 

492 

493 try: 

494 def local_call(): 

495 try: 

496 return local_method(device_path, *args, **kwargs) 

497 except util.CommandException as e: 

498 if e.code == errno.EROFS or e.code == errno.EMEDIUMTYPE: 

499 raise ErofsLinstorCallException(e) # Break retry calls. 

500 if e.code == errno.ENOENT: 

501 raise NoPathLinstorCallException(e) 

502 raise e 

503 # Retry only locally if it's not an EROFS exception. 

504 return util.retry(local_call, 5, 2, exceptions=[util.CommandException]) 

505 except util.CommandException as e: 

506 util.SMlog('failed to execute locally vhd-util (sys {})'.format(e.code)) 

507 raise e 

508 

509 def _call_local_method_or_fail(self, local_method, device_path, *args, **kwargs): 

510 try: 

511 return self._call_local_method(local_method, device_path, *args, **kwargs) 

512 except ErofsLinstorCallException as e: 

513 # Volume is locked on a host, find openers. 

514 self._raise_openers_exception(device_path, e.cmd_err) 

515 

516 def _call_method(self, local_method, remote_method, device_path, use_parent, *args, **kwargs): 

517 # Note: `use_parent` exists to know if the VHD parent is used by the local/remote method. 

518 # Normally in case of failure, if the parent is unused we try to execute the method on 

519 # another host using the DRBD opener list. In the other case, if the parent is required, 

520 # we must check where this last one is open instead of the child. 

521 

522 if isinstance(local_method, str): 

523 local_method = getattr(self, local_method) 

524 

525 # A. Try to write locally... 

526 try: 

527 return self._call_local_method(local_method, device_path, *args, **kwargs) 

528 except Exception: 

529 pass 

530 

531 util.SMlog('unable to execute `{}` locally, retry using a writable host...'.format(remote_method)) 

532 

533 # B. Execute the command on another host. 

534 # B.1. Get host list. 

535 hosts = self._get_hosts(remote_method, device_path) 

536 

537 # B.2. Prepare remote args. 

538 remote_args = { 

539 'devicePath': device_path, 

540 'groupName': self._linstor.group_name 

541 } 

542 remote_args.update(**kwargs) 

543 remote_args = {str(key): str(value) for key, value in remote_args.items()} 

544 

545 volume_uuid = self._linstor.get_volume_uuid_from_device_path( 

546 device_path 

547 ) 

548 parent_volume_uuid = None 

549 if use_parent: 

550 parent_volume_uuid = self.get_parent(volume_uuid) 

551 

552 openers_uuid = parent_volume_uuid if use_parent else volume_uuid 

553 

554 # B.3. Call! 

555 def remote_call(): 

556 try: 

557 all_openers = self._linstor.get_volume_openers(openers_uuid) 

558 except Exception as e: 

559 raise xs_errors.XenError( 

560 'VDIUnavailable', 

561 opterr='Unable to get DRBD openers to run vhd-util command `{}` (path={}): {}' 

562 .format(remote_method, device_path, e) 

563 ) 

564 

565 no_host_found = True 

566 for hostname, openers in all_openers.items(): 

567 if not openers: 

568 continue 

569 

570 host_ref = self._find_host_ref_from_hostname(hosts, hostname) 

571 if not host_ref: 

572 continue 

573 

574 no_host_found = False 

575 try: 

576 return call_remote_method(self._session, host_ref, remote_method, remote_args) 

577 except Exception: 

578 pass 

579 

580 if no_host_found: 

581 try: 

582 return local_method(device_path, *args, **kwargs) 

583 except Exception as e: 

584 self._raise_openers_exception(device_path, e) 

585 

586 raise xs_errors.XenError( 

587 'VDIUnavailable', 

588 opterr='No valid host found to run vhd-util command `{}` (path=`{}`, openers=`{}`)' 

589 .format(remote_method, device_path, openers) 

590 ) 

591 return util.retry(remote_call, 5, 2) 

592 

593 @staticmethod 

594 def _zeroize(path, size): 

595 if not util.zeroOut(path, size, vhdutil.VHD_FOOTER_SIZE): 

596 raise xs_errors.XenError( 

597 'EIO', 

598 opterr='Failed to zero out VHD footer {}'.format(path) 

599 )