1# Copyright (C) Citrix Systems Inc. 

2# 

3# This program is free software; you can redistribute it and/or modify 

4# it under the terms of the GNU Lesser General Public License as published 

5# by the Free Software Foundation; version 2.1 only. 

6# 

7# This program is distributed in the hope that it will be useful, 

8# but WITHOUT ANY WARRANTY; without even the implied warranty of 

9# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

10# GNU Lesser General Public License for more details. 

11# 

12# You should have received a copy of the GNU Lesser General Public License 

13# along with this program; if not, write to the Free Software Foundation, Inc., 

14# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 

15# 

16# Miscellaneous utility functions 

17# 

18 

19from sm_typing import Any, List, Optional, override 

20 

21import contextlib 

22import atexit 

23import os 

24import re 

25import sys 

26import subprocess 

27import shutil 

28import tempfile 

29import signal 

30import time 

31import types 

32import datetime 

33import errno 

34import functools 

35import socket 

36import threading 

37import xml.dom.minidom 

38import scsiutil 

39import stat 

40import xs_errors 

41import XenAPI # pylint: disable=import-error 

42import xmlrpc.client 

43import http.client 

44import base64 

45import syslog 

46import resource 

47import traceback 

48import glob 

49import copy 

50import contextlib 

51from sm_typing import override 

52 

53from functools import reduce 

54from sm_typing import List, Optional 

55 

56NO_LOGGING_STAMPFILE = '/etc/xensource/no_sm_log' 

57 

58IORETRY_MAX = 20 # retries 

59IORETRY_PERIOD = 1.0 # seconds 

60 

61LOGGING = not (os.path.exists(NO_LOGGING_STAMPFILE)) 

62_SM_SYSLOG_FACILITY = syslog.LOG_LOCAL2 

63LOG_EMERG = syslog.LOG_EMERG 

64LOG_ALERT = syslog.LOG_ALERT 

65LOG_CRIT = syslog.LOG_CRIT 

66LOG_ERR = syslog.LOG_ERR 

67LOG_WARNING = syslog.LOG_WARNING 

68LOG_NOTICE = syslog.LOG_NOTICE 

69LOG_INFO = syslog.LOG_INFO 

70LOG_DEBUG = syslog.LOG_DEBUG 

71 

72ISCSI_REFDIR = '/var/run/sr-ref' 

73 

74CMD_DD = "/bin/dd" 

75CMD_KICKPIPE = '/opt/xensource/libexec/kickpipe' 

76 

77FIST_PAUSE_PERIOD = 30 # seconds 

78 

79class _TimeoutContextManager(contextlib.AbstractContextManager): 

80 def __init__(self, delay: int) -> None: 

81 self.delay = delay 

82 self.old_handler: Any = None 

83 

84 @override 

85 def __enter__(self) -> "_TimeoutContextManager": 

86 def handle_timeout(_signum: int, _frame: Optional[types.FrameType]) -> None: 

87 raise TimeoutError(f"Timed out after {self.delay} seconds") 

88 self.old_handler = signal.signal(signal.SIGALRM, handle_timeout) 

89 signal.alarm(self.delay) 

90 return self 

91 

92 @override 

93 def __exit__( 

94 self, 

95 exc_type: Optional[Any], 

96 exc_val: Optional[BaseException], 

97 exc_tb: Optional[types.TracebackType] 

98 ): 

99 signal.alarm(0) 

100 if self.old_handler is not None: 

101 signal.signal(signal.SIGALRM, self.old_handler) 

102 self.old_handler = None 

103 return False 

104 

105 def __call__(self, func): 

106 def wrapper(*args, **kwargs): 

107 with self: 

108 return func(*args, **kwargs) 

109 return wrapper 

110 

111def timeout(*args, **kwargs): 

112 if len(args) >= 2 and callable(args[1]): 

113 delay, func = args[0], args[1] 

114 remaining_args = args[2:] 

115 with _TimeoutContextManager(delay): 

116 return func(*remaining_args, **kwargs) 

117 

118 if len(args) == 1 and callable(args[0]): 

119 raise TypeError("timeout() missing 1 required argument: 'delay'") 

120 

121 if args: 

122 delay = args[0] 

123 elif "delay" in kwargs: 

124 delay = kwargs["delay"] 

125 else: 

126 raise TypeError("timeout() missing 1 required argument: 'delay'") 

127 return _TimeoutContextManager(delay) 

128 

129class SMException(Exception): 

130 """Base class for all SM exceptions for easier catching & wrapping in 

131 XenError""" 

132 

133 

134class CommandException(SMException): 

135 def error_message(self, code): 

136 if code > 0: 

137 return os.strerror(code) 

138 elif code < 0: 

139 return "Signalled %s" % (abs(code)) 

140 return "Success" 

141 

142 def __init__(self, code, cmd="", reason='exec failed'): 

143 self.code = code 

144 self.cmd = cmd 

145 self.reason = reason 

146 Exception.__init__(self, self.error_message(code)) 

147 

148 

149class SRBusyException(SMException): 

150 """The SR could not be locked""" 

151 pass 

152 

153 

154def logException(tag): 

155 info = sys.exc_info() 

156 if info[0] == SystemExit: 156 ↛ 158line 156 didn't jump to line 158, because the condition on line 156 was never true

157 # this should not be happening when catching "Exception", but it is 

158 sys.exit(0) 

159 tb = reduce(lambda a, b: "%s%s" % (a, b), traceback.format_tb(info[2])) 

160 str = "***** %s: EXCEPTION %s, %s\n%s" % (tag, info[0], info[1], tb) 

161 SMlog(str) 

162 

163 

164def roundup(divisor, value): 

165 """Retruns the rounded up value so it is divisible by divisor.""" 

166 

167 if value == 0: 167 ↛ 168line 167 didn't jump to line 168, because the condition on line 167 was never true

168 value = 1 

169 if value % divisor != 0: 

170 return ((int(value) // divisor) + 1) * divisor 

171 return value 

172 

173 

174def to_plain_string(obj): 

175 if obj is None: 

176 return None 

177 if isinstance(obj, dict) and len(obj) == 0: 

178 SMlog(f"util.to_plain_string() corrected empty dict to empty str") 

179 return "" 

180 return str(obj) 

181 

182 

183def shellquote(arg): 

184 return '"%s"' % arg.replace('"', '\\"') 

185 

186 

187def make_WWN(name): 

188 hex_prefix = name.find("0x") 

189 if (hex_prefix >= 0): 189 ↛ 192line 189 didn't jump to line 192, because the condition on line 189 was never false

190 name = name[name.find("0x") + 2:len(name)] 

191 # inject dashes for each nibble 

192 if (len(name) == 16): # sanity check 192 ↛ 196line 192 didn't jump to line 196, because the condition on line 192 was never false

193 name = name[0:2] + "-" + name[2:4] + "-" + name[4:6] + "-" + \ 

194 name[6:8] + "-" + name[8:10] + "-" + name[10:12] + "-" + \ 

195 name[12:14] + "-" + name[14:16] 

196 return name 

197 

198 

199def synchronized(func): 

200 lock = threading.RLock() 

201 

202 @functools.wraps(func) 

203 def wrapper(*args, **kwargs): 

204 with lock: 

205 return func(*args, **kwargs) 

206 

207 return wrapper 

208 

209 

210@synchronized 

211def _writeToSyslog(ident, facility, priority, message): 

212 syslog.openlog(ident, 0, facility) 

213 syslog.syslog(priority, message) 

214 syslog.closelog() 

215 

216 

217def _logToSyslog(ident, facility, priority, message): 

218 pid = os.getpid() 

219 thread_name = threading.current_thread().name 

220 _writeToSyslog(ident, facility, priority, f"[{pid}][{thread_name}] {message}") 

221 

222 

223def SMlog(message, ident="SM", priority=LOG_INFO): 

224 if LOGGING: 224 ↛ exitline 224 didn't return from function 'SMlog', because the condition on line 224 was never false

225 for message_line in str(message).split('\n'): 

226 _logToSyslog(ident, _SM_SYSLOG_FACILITY, priority, message_line) 

227 

228 

229class LoggerCounter: 

230 def __init__(self, max_repeats): 

231 self.previous_message = None 

232 self.max_repeats = max_repeats 

233 self.repeat_counter = 0 

234 

235 def log(self, message): 

236 self.repeat_counter += 1 

237 if self.previous_message != message or self.repeat_counter == self.max_repeats: 

238 SMlog(message) 

239 self.previous_message = message 

240 self.repeat_counter = 0 

241 

242def _getDateString(): 

243 d = datetime.datetime.now() 

244 t = d.timetuple() 

245 return "%s-%s-%s:%s:%s:%s" % \ 

246 (t[0], t[1], t[2], t[3], t[4], t[5]) 

247 

248 

249def doexec(args, inputtext=None, new_env=None, text=True): 

250 """Execute a subprocess, then return its return code, stdout and stderr""" 

251 env = None 

252 if new_env: 

253 env = dict(os.environ) 

254 env.update(new_env) 

255 proc = subprocess.Popen(args, stdin=subprocess.PIPE, 

256 stdout=subprocess.PIPE, 

257 stderr=subprocess.PIPE, 

258 close_fds=True, env=env, 

259 universal_newlines=text) 

260 

261 if not text and inputtext is not None: 261 ↛ 262line 261 didn't jump to line 262, because the condition on line 261 was never true

262 inputtext = inputtext.encode() 

263 

264 (stdout, stderr) = proc.communicate(inputtext) 

265 

266 rc = proc.returncode 

267 return rc, stdout, stderr 

268 

269 

270def is_string(value): 

271 return isinstance(value, str) 

272 

273 

274# These are partially tested functions that replicate the behaviour of 

275# the original pread,pread2 and pread3 functions. Potentially these can 

276# replace the original ones at some later date. 

277# 

278# cmdlist is a list of either single strings or pairs of strings. For 

279# each pair, the first component is passed to exec while the second is 

280# written to the logs. 

281def pread(cmdlist, close_stdin=False, scramble=None, expect_rc=0, 

282 quiet=False, new_env=None, text=True): 

283 cmdlist_for_exec = [] 

284 cmdlist_for_log = [] 

285 for item in cmdlist: 

286 if is_string(item): 286 ↛ 296line 286 didn't jump to line 296, because the condition on line 286 was never false

287 cmdlist_for_exec.append(item) 

288 if scramble: 288 ↛ 289line 288 didn't jump to line 289, because the condition on line 288 was never true

289 if item.find(scramble) != -1: 

290 cmdlist_for_log.append("<filtered out>") 

291 else: 

292 cmdlist_for_log.append(item) 

293 else: 

294 cmdlist_for_log.append(item) 

295 else: 

296 cmdlist_for_exec.append(item[0]) 

297 cmdlist_for_log.append(item[1]) 

298 

299 if not quiet: 299 ↛ 301line 299 didn't jump to line 301, because the condition on line 299 was never false

300 SMlog(cmdlist_for_log) 

301 (rc, stdout, stderr) = doexec(cmdlist_for_exec, new_env=new_env, text=text) 

302 if rc != expect_rc: 

303 SMlog("FAILED in util.pread: (rc %d) stdout: '%s', stderr: '%s'" % \ 

304 (rc, stdout, stderr)) 

305 if quiet: 305 ↛ 306line 305 didn't jump to line 306, because the condition on line 305 was never true

306 SMlog("Command was: %s" % cmdlist_for_log) 

307 if '' == stderr: 307 ↛ 308line 307 didn't jump to line 308, because the condition on line 307 was never true

308 stderr = stdout 

309 raise CommandException(rc, str(cmdlist), stderr.strip()) 

310 if not quiet: 310 ↛ 312line 310 didn't jump to line 312, because the condition on line 310 was never false

311 SMlog(" pread SUCCESS") 

312 return stdout 

313 

314 

315# POSIX guaranteed atomic within the same file system. 

316# Supply directory to ensure tempfile is created 

317# in the same directory. 

318def atomicFileWrite(targetFile, directory, text): 

319 

320 file = None 

321 try: 

322 # Create file only current pid can write/read to 

323 # our responsibility to clean it up. 

324 _, tempPath = tempfile.mkstemp(dir=directory) 

325 file = open(tempPath, 'w') 

326 file.write(text) 

327 

328 # Ensure flushed to disk. 

329 file.flush() 

330 os.fsync(file.fileno()) 

331 file.close() 

332 

333 os.rename(tempPath, targetFile) 

334 except OSError: 

335 SMlog("FAILED to atomic write to %s" % (targetFile)) 

336 

337 finally: 

338 if (file is not None) and (not file.closed): 

339 file.close() 

340 

341 if os.path.isfile(tempPath): 

342 os.remove(tempPath) 

343 

344 

345#Read STDOUT from cmdlist and discard STDERR output 

346def pread2(cmdlist, quiet=False, text=True): 

347 return pread(cmdlist, quiet=quiet, text=text) 

348 

349 

350#Read STDOUT from cmdlist, feeding 'text' to STDIN 

351def pread3(cmdlist, text): 

352 SMlog(cmdlist) 

353 (rc, stdout, stderr) = doexec(cmdlist, text) 

354 if rc: 

355 SMlog("FAILED in util.pread3: (errno %d) stdout: '%s', stderr: '%s'" % \ 

356 (rc, stdout, stderr)) 

357 if '' == stderr: 

358 stderr = stdout 

359 raise CommandException(rc, str(cmdlist), stderr.strip()) 

360 SMlog(" pread3 SUCCESS") 

361 return stdout 

362 

363 

364def listdir(path, quiet=False): 

365 cmd = ["ls", path, "-1", "--color=never"] 

366 try: 

367 text = pread2(cmd, quiet=quiet)[:-1] 

368 if len(text) == 0: 

369 return [] 

370 return text.split('\n') 

371 except CommandException as inst: 

372 if inst.code == errno.ENOENT: 

373 raise CommandException(errno.EIO, inst.cmd, inst.reason) 

374 else: 

375 raise CommandException(inst.code, inst.cmd, inst.reason) 

376 

377 

378def gen_uuid(): 

379 cmd = ["uuidgen", "-r"] 

380 return pread(cmd)[:-1] 

381 

382 

383def match_uuid(s): 

384 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}") 

385 return regex.search(s, 0) 

386 

387 

388def findall_uuid(s): 

389 regex = re.compile("[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}") 

390 return regex.findall(s, 0) 

391 

392 

393def exactmatch_uuid(s): 

394 regex = re.compile("^[0-9a-f]{8}-(([0-9a-f]{4})-){3}[0-9a-f]{12}$") 

395 return regex.search(s, 0) 

396 

397 

398def start_log_entry(srpath, path, args): 

399 logstring = str(datetime.datetime.now()) 

400 logstring += " log: " 

401 logstring += srpath 

402 logstring += " " + path 

403 for element in args: 

404 logstring += " " + element 

405 try: 

406 file = open(srpath + "/filelog.txt", "a") 

407 file.write(logstring) 

408 file.write("\n") 

409 file.close() 

410 except: 

411 pass 

412 

413 # failed to write log ... 

414 

415def end_log_entry(srpath, path, args): 

416 # for teminating, use "error" or "done" 

417 logstring = str(datetime.datetime.now()) 

418 logstring += " end: " 

419 logstring += srpath 

420 logstring += " " + path 

421 for element in args: 

422 logstring += " " + element 

423 try: 

424 file = open(srpath + "/filelog.txt", "a") 

425 file.write(logstring) 

426 file.write("\n") 

427 file.close() 

428 except: 

429 pass 

430 

431 # failed to write log ... 

432 # for now print 

433 # print "%s" % logstring 

434 

435def ioretry(f, errlist=[errno.EIO], maxretry=IORETRY_MAX, period=IORETRY_PERIOD, **ignored): 

436 retries = 0 

437 while True: 

438 try: 

439 return f() 

440 except OSError as ose: 

441 err = int(ose.errno) 

442 if not err in errlist: 

443 raise CommandException(err, str(f), "OSError") 

444 except CommandException as ce: 

445 if not int(ce.code) in errlist: 

446 raise 

447 

448 retries += 1 

449 if retries >= maxretry: 

450 break 

451 

452 time.sleep(period) 

453 

454 raise CommandException(errno.ETIMEDOUT, str(f), "Timeout") 

455 

456 

457def ioretry_stat(path, maxretry=IORETRY_MAX): 

458 # this ioretry is similar to the previous method, but 

459 # stat does not raise an error -- so check its return 

460 retries = 0 

461 while retries < maxretry: 

462 stat = os.statvfs(path) 

463 if stat.f_blocks != -1: 

464 return stat 

465 time.sleep(1) 

466 retries += 1 

467 raise CommandException(errno.EIO, "os.statvfs") 

468 

469 

470def sr_get_capability(sr_uuid, session=None): 

471 result = [] 

472 api_session = None 

473 try: 

474 if session is None: 

475 api_session = ApiSession("SM-sr-get-capability") 

476 session = api_session.session 

477 sr_ref = session.xenapi.SR.get_by_uuid(sr_uuid) 

478 sm_type = session.xenapi.SR.get_record(sr_ref)['type'] 

479 sm_rec = session.xenapi.SM.get_all_records_where( 

480 "field \"type\" = \"%s\"" % sm_type) 

481 

482 # SM expects at least one entry of any SR type 

483 if len(sm_rec) > 0: 

484 result = list(sm_rec.values())[0]['capabilities'] 

485 finally: 

486 if api_session: 

487 session.logout() 

488 return result 

489 

490def sr_get_driver_info(driver_info): 

491 results = {} 

492 # first add in the vanilla stuff 

493 for key in ['name', 'description', 'vendor', 'copyright', \ 

494 'driver_version', 'required_api_version']: 

495 results[key] = driver_info[key] 

496 # add the capabilities (xmlrpc array) 

497 # enforcing activate/deactivate for blktap2 

498 caps = driver_info['capabilities'] 

499 if "ATOMIC_PAUSE" in caps: 499 ↛ 500line 499 didn't jump to line 500, because the condition on line 499 was never true

500 for cap in ("VDI_ACTIVATE", "VDI_DEACTIVATE"): 

501 if not cap in caps: 

502 caps.append(cap) 

503 elif "VDI_ACTIVATE" in caps or "VDI_DEACTIVATE" in caps: 503 ↛ 504line 503 didn't jump to line 504, because the condition on line 503 was never true

504 SMlog("Warning: vdi_[de]activate present for %s" % driver_info["name"]) 

505 

506 results['capabilities'] = caps 

507 # add in the configuration options 

508 options = [] 

509 for option in driver_info['configuration']: 

510 options.append({'key': option[0], 'description': option[1]}) 

511 results['configuration'] = options 

512 return xmlrpc.client.dumps((results, ), "", True) 

513 

514 

515def return_nil(): 

516 return xmlrpc.client.dumps((None, ), "", True, allow_none=True) 

517 

518 

519def SRtoXML(SRlist): 

520 dom = xml.dom.minidom.Document() 

521 driver = dom.createElement("SRlist") 

522 dom.appendChild(driver) 

523 

524 for key in SRlist.keys(): 

525 dict = SRlist[key] 

526 entry = dom.createElement("SR") 

527 driver.appendChild(entry) 

528 

529 e = dom.createElement("UUID") 

530 entry.appendChild(e) 

531 textnode = dom.createTextNode(key) 

532 e.appendChild(textnode) 

533 

534 if 'size' in dict: 

535 e = dom.createElement("Size") 

536 entry.appendChild(e) 

537 textnode = dom.createTextNode(str(dict['size'])) 

538 e.appendChild(textnode) 

539 

540 if 'storagepool' in dict: 

541 e = dom.createElement("StoragePool") 

542 entry.appendChild(e) 

543 textnode = dom.createTextNode(str(dict['storagepool'])) 

544 e.appendChild(textnode) 

545 

546 if 'aggregate' in dict: 

547 e = dom.createElement("Aggregate") 

548 entry.appendChild(e) 

549 textnode = dom.createTextNode(str(dict['aggregate'])) 

550 e.appendChild(textnode) 

551 

552 return dom.toprettyxml() 

553 

554 

555def pathexists(path): 

556 try: 

557 os.lstat(path) 

558 return True 

559 except OSError as inst: 

560 if inst.errno == errno.EIO: 560 ↛ 561line 560 didn't jump to line 561, because the condition on line 560 was never true

561 time.sleep(1) 

562 try: 

563 os.lstat(path) 

564 return True 

565 except: 

566 pass 

567 raise CommandException(errno.EIO, "os.lstat(%s)" % path, "failed") 

568 return False 

569 

570 

571def force_unlink(path): 

572 try: 

573 os.unlink(path) 

574 except OSError as e: 

575 if e.errno != errno.ENOENT: 575 ↛ 576line 575 didn't jump to line 576, because the condition on line 575 was never true

576 raise 

577 

578 

579def create_secret(session, secret): 

580 ref = session.xenapi.secret.create({'value': secret}) 

581 return session.xenapi.secret.get_uuid(ref) 

582 

583 

584def get_secret(session, uuid): 

585 try: 

586 ref = session.xenapi.secret.get_by_uuid(uuid) 

587 return session.xenapi.secret.get_value(ref) 

588 except: 

589 raise xs_errors.XenError('InvalidSecret', opterr='Unable to look up secret [%s]' % uuid) 

590 

591 

592def get_real_path(path): 

593 "Follow symlinks to the actual file" 

594 absPath = path 

595 directory = '' 

596 while os.path.islink(absPath): 

597 directory = os.path.dirname(absPath) 

598 absPath = os.readlink(absPath) 

599 absPath = os.path.join(directory, absPath) 

600 return absPath 

601 

602 

603def wait_for_path(path, timeout): 

604 for i in range(0, timeout): 604 ↛ 608line 604 didn't jump to line 608, because the loop on line 604 didn't complete

605 if len(glob.glob(path)): 605 ↛ 607line 605 didn't jump to line 607, because the condition on line 605 was never false

606 return True 

607 time.sleep(1) 

608 return False 

609 

610 

611def wait_for_nopath(path, timeout): 

612 for i in range(0, timeout): 

613 if not os.path.exists(path): 

614 return True 

615 time.sleep(1) 

616 return False 

617 

618 

619def wait_for_path_multi(path, timeout): 

620 for i in range(0, timeout): 

621 paths = glob.glob(path) 

622 SMlog("_wait_for_paths_multi: paths = %s" % paths) 

623 if len(paths): 

624 SMlog("_wait_for_paths_multi: return first path: %s" % paths[0]) 

625 return paths[0] 

626 time.sleep(1) 

627 return "" 

628 

629 

630def isdir(path): 

631 try: 

632 st = os.stat(path) 

633 return stat.S_ISDIR(st.st_mode) 

634 except OSError as inst: 

635 if inst.errno == errno.EIO: 635 ↛ 636line 635 didn't jump to line 636, because the condition on line 635 was never true

636 raise CommandException(errno.EIO, "os.stat(%s)" % path, "failed") 

637 return False 

638 

639 

640def get_single_entry(path): 

641 f = open(path, 'r') 

642 line = f.readline() 

643 f.close() 

644 return line.rstrip() 

645 

646 

647def get_fs_size(path): 

648 st = ioretry_stat(path) 

649 return st.f_blocks * st.f_frsize 

650 

651 

652def get_fs_utilisation(path): 

653 st = ioretry_stat(path) 

654 return (st.f_blocks - st.f_bfree) * \ 

655 st.f_frsize 

656 

657 

658def ismount(path): 

659 """Test whether a path is a mount point""" 

660 try: 

661 s1 = os.stat(path) 

662 s2 = os.stat(os.path.join(path, '..')) 

663 except OSError as inst: 

664 raise CommandException(inst.errno, "os.stat") 

665 dev1 = s1.st_dev 

666 dev2 = s2.st_dev 

667 if dev1 != dev2: 

668 return True # path/.. on a different device as path 

669 ino1 = s1.st_ino 

670 ino2 = s2.st_ino 

671 if ino1 == ino2: 

672 return True # path/.. is the same i-node as path 

673 return False 

674 

675 

676def makedirs(name, mode=0o777): 

677 head, tail = os.path.split(name) 

678 if not tail: 678 ↛ 679line 678 didn't jump to line 679, because the condition on line 678 was never true

679 head, tail = os.path.split(head) 

680 if head and tail and not pathexists(head): 

681 makedirs(head, mode) 

682 if tail == os.curdir: 682 ↛ 683line 682 didn't jump to line 683, because the condition on line 682 was never true

683 return 

684 try: 

685 os.mkdir(name, mode) 

686 except OSError as exc: 

687 if exc.errno == errno.EEXIST and os.path.isdir(name): 687 ↛ 688line 687 didn't jump to line 688, because the condition on line 687 was never true

688 if mode: 

689 os.chmod(name, mode) 

690 pass 

691 else: 

692 raise 

693 

694 

695def zeroOut(path, fromByte, bytes): 

696 """write 'bytes' zeros to 'path' starting from fromByte (inclusive)""" 

697 blockSize = 4096 

698 

699 fromBlock = fromByte // blockSize 

700 if fromByte % blockSize: 

701 fromBlock += 1 

702 bytesBefore = fromBlock * blockSize - fromByte 

703 if bytesBefore > bytes: 

704 bytesBefore = bytes 

705 bytes -= bytesBefore 

706 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1", 

707 "seek=%s" % fromByte, "count=%s" % bytesBefore] 

708 try: 

709 pread2(cmd) 

710 except CommandException: 

711 return False 

712 

713 blocks = bytes // blockSize 

714 bytes -= blocks * blockSize 

715 fromByte = (fromBlock + blocks) * blockSize 

716 if blocks: 

717 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=%s" % blockSize, 

718 "seek=%s" % fromBlock, "count=%s" % blocks] 

719 try: 

720 pread2(cmd) 

721 except CommandException: 

722 return False 

723 

724 if bytes: 

725 cmd = [CMD_DD, "if=/dev/zero", "of=%s" % path, "bs=1", 

726 "seek=%s" % fromByte, "count=%s" % bytes] 

727 try: 

728 pread2(cmd) 

729 except CommandException: 

730 return False 

731 

732 return True 

733 

734 

735def wipefs(blockdev): 

736 "Wipe filesystem signatures from `blockdev`" 

737 pread2(["/usr/sbin/wipefs", "-a", blockdev]) 

738 

739 

740def match_rootdev(s): 

741 regex = re.compile("^PRIMARY_DISK") 

742 return regex.search(s, 0) 

743 

744 

745def getrootdev(): 

746 filename = '/etc/xensource-inventory' 

747 try: 

748 f = open(filename, 'r') 

749 except: 

750 raise xs_errors.XenError('EIO', \ 

751 opterr="Unable to open inventory file [%s]" % filename) 

752 rootdev = '' 

753 for line in filter(match_rootdev, f.readlines()): 

754 rootdev = line.split("'")[1] 

755 if not rootdev: 755 ↛ 756line 755 didn't jump to line 756, because the condition on line 755 was never true

756 raise xs_errors.XenError('NoRootDev') 

757 return rootdev 

758 

759 

760def getrootdevID(): 

761 rootdev = getrootdev() 

762 try: 

763 rootdevID = scsiutil.getSCSIid(rootdev) 

764 except: 

765 SMlog("util.getrootdevID: Unable to verify serial or SCSIid of device: %s" \ 

766 % rootdev) 

767 return '' 

768 

769 if not len(rootdevID): 

770 SMlog("util.getrootdevID: Unable to identify scsi device [%s] via scsiID" \ 

771 % rootdev) 

772 

773 return rootdevID 

774 

775 

776class ApiSession(contextlib.AbstractContextManager): 

777 session = None 

778 

779 def __init__(self, originator="SM"): 

780 self.originator = originator 

781 # First acquire a valid session 

782 self.session = self._login() 

783 atexit.register(self._atexit) 

784 

785 def _login(self): 

786 if self.session: 

787 return self.session 

788 session = XenAPI.xapi_local() 

789 try: 

790 session.xenapi.login_with_password('root', '', '', self.originator) 

791 except Exception as exc: 

792 raise xs_errors.XenError( 

793 "APISession", 

794 opterr=f"[{self.originator}] Unable to open local XAPI session" 

795 ) from exc 

796 SMlog("ApiSession [{}] login".format(self.originator), priority=LOG_DEBUG) 

797 return session 

798 

799 def _logout(self, log): 

800 """Closes an API session""" 

801 if self.session is None: 801 ↛ 802line 801 didn't jump to line 802, because the condition on line 801 was never true

802 SMlog("ApiSession [{}] session is None {}".format(self.originator, log), priority=LOG_DEBUG) 

803 return 

804 self.session.xenapi.session.logout() 

805 SMlog("ApiSession [{}] {}".format(self.originator, log), priority=LOG_DEBUG) 

806 self.session = None 

807 

808 def logout(self, log="logout"): 

809 atexit.unregister(self._atexit) 

810 self._logout(log=log) 

811 

812 def __del__(self): 

813 if self.session: 813 ↛ 814line 813 didn't jump to line 814, because the condition on line 813 was never true

814 self.logout(log="logout del") 

815 

816 def _atexit(self): 

817 self._logout(log="logout atexit") 

818 

819 @override 

820 def __enter__(self): 

821 self.session = self._login() 

822 return self.session 

823 

824 @override 

825 def __exit__(self, _type, _value, _traceback): 

826 self.logout(log=f"logout exception[{_type}] {_value}" if _type else "logout context") 

827 

828 

829def xapi_safe_call(function): 

830 """ 

831 Decorator to catch classic XAPI connection problems and retry forever. 

832 The method should be only called for XAPI calls. 

833 eg: blktap2.py#VDI:_remove_tag() 

834 """ 

835 def wrapper(*args, **kwargs): 

836 call_str = f"{function.__name__}(args={args}, kwargs={kwargs})" 

837 while True: 

838 try: 

839 return function(*args, **kwargs) 

840 except xmlrpc.client.ProtocolError as e: 

841 # If there's a connection error, keep trying forever. 

842 if e.errcode == http.HTTPStatus.INTERNAL_SERVER_ERROR.value: 

843 continue 

844 raise 

845 except Exception as e: 

846 SMlog(f"Failed XAPI call `{call_str}`: `{e}`") 

847 raise 

848 return wrapper 

849 

850 

851def get_this_host(): 

852 uuid = None 

853 f = open("/etc/xensource-inventory", 'r') 

854 for line in f.readlines(): 

855 if line.startswith("INSTALLATION_UUID"): 

856 uuid = line.split("'")[1] 

857 f.close() 

858 return uuid 

859 

860 

861def get_master_ref(session): 

862 pools = session.xenapi.pool.get_all() 

863 return session.xenapi.pool.get_master(pools[0]) 

864 

865 

866def is_master(session): 

867 return get_this_host_ref(session) == get_master_ref(session) 

868 

869 

870def get_localhost_ref(session): 

871 filename = '/etc/xensource-inventory' 

872 try: 

873 f = open(filename, 'r') 

874 except: 

875 raise xs_errors.XenError('EIO', \ 

876 opterr="Unable to open inventory file [%s]" % filename) 

877 domid = '' 

878 for line in filter(match_domain_id, f.readlines()): 

879 domid = line.split("'")[1] 

880 if not domid: 

881 raise xs_errors.XenError('APILocalhost') 

882 

883 vms = session.xenapi.VM.get_all_records_where('field "uuid" = "%s"' % domid) 

884 for vm in vms: 

885 record = vms[vm] 

886 if record["uuid"] == domid: 

887 hostid = record["resident_on"] 

888 return hostid 

889 raise xs_errors.XenError('APILocalhost') 

890 

891 

892def match_domain_id(s): 

893 regex = re.compile("^CONTROL_DOMAIN_UUID") 

894 return regex.search(s, 0) 

895 

896 

897def get_hosts_attached_on(session, vdi_uuids): 

898 host_refs = {} 

899 for vdi_uuid in vdi_uuids: 

900 try: 

901 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid) 

902 except XenAPI.Failure: 

903 SMlog("VDI %s not in db, ignoring" % vdi_uuid) 

904 continue 

905 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref) 

906 for key in [x for x in sm_config.keys() if x.startswith('host_')]: 

907 host_refs[key[len('host_'):]] = True 

908 return host_refs.keys() 

909 

910def get_hosts_attached_on_with_vdi_uuid(session, vdi_uuids): 

911 """ 

912 Return a dict of {vdi_uuid: host OpaqueRef} 

913 """ 

914 host_refs = {} 

915 for vdi_uuid in vdi_uuids: 

916 try: 

917 vdi_ref = session.xenapi.VDI.get_by_uuid(vdi_uuid) 

918 except XenAPI.Failure: 

919 SMlog("VDI %s not in db, ignoring" % vdi_uuid) 

920 continue 

921 sm_config = session.xenapi.VDI.get_sm_config(vdi_ref) 

922 for key in [x for x in sm_config.keys() if x.startswith('host_')]: 

923 host_refs[vdi_uuid] = key[len('host_'):] 

924 return host_refs 

925 

926def get_this_host_address(session): 

927 host_uuid = get_this_host() 

928 host_ref = session.xenapi.host.get_by_uuid(host_uuid) 

929 return session.xenapi.host.get_record(host_ref)['address'] 

930 

931def get_host_addresses(session): 

932 addresses = [] 

933 hosts = session.xenapi.host.get_all_records() 

934 for record in hosts.values(): 

935 addresses.append(record['address']) 

936 return addresses 

937 

938def get_this_host_ref(session): 

939 host_uuid = get_this_host() 

940 host_ref = session.xenapi.host.get_by_uuid(host_uuid) 

941 return host_ref 

942 

943 

944def get_slaves_attached_on(session, vdi_uuids): 

945 "assume this host is the SR master" 

946 host_refs = get_hosts_attached_on(session, vdi_uuids) 

947 master_ref = get_this_host_ref(session) 

948 return [x for x in host_refs if x != master_ref] 

949 

950def get_enabled_hosts(session): 

951 """ 

952 Returns a list of host refs that are enabled in the pool. 

953 """ 

954 return list(session.xenapi.host.get_all_records_where('field "enabled" = "true"').keys()) 

955 

956def get_online_hosts(session): 

957 online_hosts = [] 

958 hosts = session.xenapi.host.get_all_records() 

959 for host_ref, host_rec in hosts.items(): 

960 metricsRef = host_rec["metrics"] 

961 metrics = session.xenapi.host_metrics.get_record(metricsRef) 

962 if metrics["live"]: 

963 online_hosts.append(host_ref) 

964 return online_hosts 

965 

966 

967def get_all_slaves(session): 

968 "assume this host is the SR master" 

969 host_refs = get_online_hosts(session) 

970 master_ref = get_this_host_ref(session) 

971 return [x for x in host_refs if x != master_ref] 

972 

973 

974def is_attached_rw(sm_config): 

975 for key, val in sm_config.items(): 

976 if key.startswith("host_") and val == "RW": 

977 return True 

978 return False 

979 

980 

981def attached_as(sm_config): 

982 for key, val in sm_config.items(): 

983 if key.startswith("host_") and (val == "RW" or val == "RO"): 983 ↛ 984line 983 didn't jump to line 984, because the condition on line 983 was never true

984 return val 

985 

986 

987def find_my_pbd_record(session, host_ref, sr_ref): 

988 try: 

989 pbds = session.xenapi.PBD.get_all_records() 

990 for pbd_ref in pbds.keys(): 

991 if pbds[pbd_ref]['host'] == host_ref and pbds[pbd_ref]['SR'] == sr_ref: 

992 return [pbd_ref, pbds[pbd_ref]] 

993 return None 

994 except Exception as e: 

995 SMlog("Caught exception while looking up PBD for host %s SR %s: %s" % (str(host_ref), str(sr_ref), str(e))) 

996 return None 

997 

998 

999def find_my_pbd(session, host_ref, sr_ref): 

1000 ret = find_my_pbd_record(session, host_ref, sr_ref) 

1001 if ret is not None: 

1002 return ret[0] 

1003 else: 

1004 return None 

1005 

1006 

1007def test_hostPBD_devs(session, sr_uuid, devs): 

1008 host = get_localhost_ref(session) 

1009 sr = session.xenapi.SR.get_by_uuid(sr_uuid) 

1010 try: 

1011 pbds = session.xenapi.PBD.get_all_records() 

1012 except: 

1013 raise xs_errors.XenError('APIPBDQuery') 

1014 for dev in devs.split(','): 

1015 for pbd in pbds: 

1016 record = pbds[pbd] 

1017 # it's ok if it's *our* PBD 

1018 if record["SR"] == sr: 

1019 break 

1020 if record["host"] == host: 

1021 devconfig = record["device_config"] 

1022 if 'device' in devconfig: 

1023 for device in devconfig['device'].split(','): 

1024 if os.path.realpath(device) == os.path.realpath(dev): 

1025 return True 

1026 return False 

1027 

1028 

1029def test_hostPBD_lun(session, targetIQN, LUNid): 

1030 host = get_localhost_ref(session) 

1031 try: 

1032 pbds = session.xenapi.PBD.get_all_records() 

1033 except: 

1034 raise xs_errors.XenError('APIPBDQuery') 

1035 for pbd in pbds: 

1036 record = pbds[pbd] 

1037 if record["host"] == host: 

1038 devconfig = record["device_config"] 

1039 if 'targetIQN' in devconfig and 'LUNid' in devconfig: 

1040 if devconfig['targetIQN'] == targetIQN and \ 

1041 devconfig['LUNid'] == LUNid: 

1042 return True 

1043 return False 

1044 

1045 

1046def test_SCSIid(session, sr_uuid, SCSIid): 

1047 if sr_uuid is not None: 

1048 sr = session.xenapi.SR.get_by_uuid(sr_uuid) 

1049 try: 

1050 pbds = session.xenapi.PBD.get_all_records() 

1051 except: 

1052 raise xs_errors.XenError('APIPBDQuery') 

1053 for pbd in pbds: 

1054 record = pbds[pbd] 

1055 # it's ok if it's *our* PBD 

1056 # During FC SR creation, devscan.py passes sr_uuid as None 

1057 if sr_uuid is not None: 

1058 if record["SR"] == sr: 

1059 break 

1060 devconfig = record["device_config"] 

1061 sm_config = session.xenapi.SR.get_sm_config(record["SR"]) 

1062 if 'SCSIid' in devconfig and devconfig['SCSIid'] == SCSIid: 

1063 return True 

1064 elif 'SCSIid' in sm_config and sm_config['SCSIid'] == SCSIid: 

1065 return True 

1066 elif 'scsi-' + SCSIid in sm_config: 

1067 return True 

1068 return False 

1069 

1070 

1071def _incr_iscsiSR_refcount(targetIQN, uuid): 

1072 if not os.path.exists(ISCSI_REFDIR): 

1073 os.mkdir(ISCSI_REFDIR) 

1074 filename = os.path.join(ISCSI_REFDIR, targetIQN) 

1075 try: 

1076 f = open(filename, 'a+') 

1077 except: 

1078 raise xs_errors.XenError('LVMRefCount', \ 

1079 opterr='file %s' % filename) 

1080 

1081 f.seek(0) 

1082 found = False 

1083 refcount = 0 

1084 for line in filter(match_uuid, f.readlines()): 

1085 refcount += 1 

1086 if line.find(uuid) != -1: 

1087 found = True 

1088 if not found: 

1089 f.write("%s\n" % uuid) 

1090 refcount += 1 

1091 f.close() 

1092 return refcount 

1093 

1094 

1095def _decr_iscsiSR_refcount(targetIQN, uuid): 

1096 filename = os.path.join(ISCSI_REFDIR, targetIQN) 

1097 if not os.path.exists(filename): 

1098 return 0 

1099 try: 

1100 f = open(filename, 'a+') 

1101 except: 

1102 raise xs_errors.XenError('LVMRefCount', \ 

1103 opterr='file %s' % filename) 

1104 

1105 f.seek(0) 

1106 output = [] 

1107 refcount = 0 

1108 for line in filter(match_uuid, f.readlines()): 

1109 if line.find(uuid) == -1: 

1110 output.append(line.rstrip()) 

1111 refcount += 1 

1112 if not refcount: 

1113 os.unlink(filename) 

1114 return refcount 

1115 

1116 # Re-open file and truncate 

1117 f.close() 

1118 f = open(filename, 'w') 

1119 for i in range(0, refcount): 

1120 f.write("%s\n" % output[i]) 

1121 f.close() 

1122 return refcount 

1123 

1124 

1125# The agent enforces 1 PBD per SR per host, so we 

1126# check for active SR entries not attached to this host 

1127def test_activePoolPBDs(session, host, uuid): 

1128 try: 

1129 pbds = session.xenapi.PBD.get_all_records() 

1130 except: 

1131 raise xs_errors.XenError('APIPBDQuery') 

1132 for pbd in pbds: 

1133 record = pbds[pbd] 

1134 if record["host"] != host and record["SR"] == uuid \ 

1135 and record["currently_attached"]: 

1136 return True 

1137 return False 

1138 

1139 

1140def remove_mpathcount_field(session, host_ref, sr_ref, SCSIid): 

1141 try: 

1142 pbdref = find_my_pbd(session, host_ref, sr_ref) 

1143 if pbdref is not None: 

1144 key = "mpath-" + SCSIid 

1145 session.xenapi.PBD.remove_from_other_config(pbdref, key) 

1146 except: 

1147 pass 

1148 

1149 

1150def kickpipe_mpathcount(): 

1151 """ 

1152 Issue a kick to the mpathcount service. This will ensure that mpathcount runs 

1153 shortly to update the multipath config records, if it was not already activated 

1154 by a UDEV event. 

1155 """ 

1156 cmd = [CMD_KICKPIPE, "mpathcount"] 

1157 (rc, stdout, stderr) = doexec(cmd) 

1158 return (rc == 0) 

1159 

1160 

1161def _testHost(hostname, port, errstring): 

1162 SMlog("_testHost: Testing host/port: %s,%d" % (hostname, port)) 

1163 try: 

1164 sockinfo = socket.getaddrinfo(hostname, int(port))[0] 

1165 except: 

1166 logException('Exception occured getting IP for %s' % hostname) 

1167 raise xs_errors.XenError('DNSError') 

1168 

1169 timeout = 5 

1170 

1171 sock = socket.socket(sockinfo[0], socket.SOCK_STREAM) 

1172 # Only allow the connect to block for up to timeout seconds 

1173 sock.settimeout(timeout) 

1174 try: 

1175 sock.connect(sockinfo[4]) 

1176 # Fix for MS storage server bug 

1177 sock.send(b'\n') 

1178 sock.close() 

1179 except socket.error as reason: 

1180 SMlog("_testHost: Connect failed after %d seconds (%s) - %s" \ 

1181 % (timeout, hostname, reason)) 

1182 raise xs_errors.XenError(errstring) 

1183 

1184 

1185def match_scsiID(s, id): 

1186 regex = re.compile(id) 

1187 return regex.search(s, 0) 

1188 

1189 

1190def _isSCSIid(s): 

1191 regex = re.compile("^scsi-") 

1192 return regex.search(s, 0) 

1193 

1194 

1195def is_usb_device(device): 

1196 cmd = ["udevadm", "info", "-q", "path", "-n", device] 

1197 result = pread2(cmd).split('/') 

1198 return len(result) >= 5 and result[4].startswith('usb') 

1199 

1200 

1201def test_scsiserial(session, device): 

1202 device = os.path.realpath(device) 

1203 if not scsiutil._isSCSIdev(device): 

1204 SMlog("util.test_scsiserial: Not a serial device: %s" % device) 

1205 return False 

1206 serial = "" 

1207 try: 

1208 serial += scsiutil.getserial(device) 

1209 except: 

1210 # Error allowed, SCSIid is the important one 

1211 pass 

1212 

1213 try: 

1214 scsiID = scsiutil.getSCSIid(device) 

1215 except: 

1216 SMlog("util.test_scsiserial: Unable to verify serial or SCSIid of device: %s" \ 

1217 % device) 

1218 return False 

1219 if not len(scsiID): 

1220 SMlog("util.test_scsiserial: Unable to identify scsi device [%s] via scsiID" \ 

1221 % device) 

1222 return False 

1223 

1224 # USB devices can have identical SCSI IDs - prefer matching with serial number 

1225 try: 

1226 usb_device_with_serial = serial and is_usb_device(device) 

1227 except: 

1228 usb_device_with_serial = False 

1229 SMlog("Unable to check if device is USB:") 

1230 SMlog(traceback.format_exc()) 

1231 

1232 try: 

1233 SRs = session.xenapi.SR.get_all_records() 

1234 except: 

1235 raise xs_errors.XenError('APIFailure') 

1236 for SR in SRs: 

1237 record = SRs[SR] 

1238 conf = record["sm_config"] 

1239 if 'devserial' in conf: 

1240 for dev in conf['devserial'].split(','): 

1241 if not usb_device_with_serial and _isSCSIid(dev): 

1242 if match_scsiID(dev, scsiID): 

1243 return True 

1244 elif len(serial) and dev == serial: 

1245 return True 

1246 return False 

1247 

1248 

1249def default(self, field, thunk): 

1250 try: 

1251 return getattr(self, field) 

1252 except: 

1253 return thunk() 

1254 

1255 

1256def list_VDI_records_in_sr(sr): 

1257 """Helper function which returns a list of all VDI records for this SR 

1258 stored in the XenAPI server, useful for implementing SR.scan""" 

1259 sr_ref = sr.session.xenapi.SR.get_by_uuid(sr.uuid) 

1260 vdis = sr.session.xenapi.VDI.get_all_records_where("field \"SR\" = \"%s\"" % sr_ref) 

1261 return vdis 

1262 

1263 

1264# Given a partition (e.g. sda1), get a disk name: 

1265def diskFromPartition(partition): 

1266 # check whether this is a device mapper device (e.g. /dev/dm-0) 

1267 m = re.match('(/dev/)?(dm-[0-9]+)(p[0-9]+)?$', partition) 

1268 if m is not None: 1268 ↛ 1269line 1268 didn't jump to line 1269, because the condition on line 1268 was never true

1269 return m.group(2) 

1270 

1271 numlen = 0 # number of digit characters 

1272 m = re.match(r"\D+(\d+)", partition) 

1273 if m is not None: 1273 ↛ 1274line 1273 didn't jump to line 1274, because the condition on line 1273 was never true

1274 numlen = len(m.group(1)) 

1275 

1276 # is it a cciss? 

1277 if True in [partition.startswith(x) for x in ['cciss', 'ida', 'rd']]: 1277 ↛ 1278line 1277 didn't jump to line 1278, because the condition on line 1277 was never true

1278 numlen += 1 # need to get rid of trailing 'p' 

1279 

1280 # is it a mapper path? 

1281 if partition.startswith("mapper"): 1281 ↛ 1282line 1281 didn't jump to line 1282, because the condition on line 1281 was never true

1282 if re.search("p[0-9]*$", partition): 

1283 numlen = len(re.match(r"\d+", partition[::-1]).group(0)) + 1 

1284 SMlog("Found mapper part, len %d" % numlen) 

1285 else: 

1286 numlen = 0 

1287 

1288 # is it /dev/disk/by-id/XYZ-part<k>? 

1289 if partition.startswith("disk/by-id"): 1289 ↛ 1290line 1289 didn't jump to line 1290, because the condition on line 1289 was never true

1290 return partition[:partition.rfind("-part")] 

1291 

1292 return partition[:len(partition) - numlen] 

1293 

1294 

1295def dom0_disks(): 

1296 """Disks carrying dom0, e.g. ['/dev/sda']""" 

1297 disks = [] 

1298 with open("/etc/mtab", 'r') as f: 

1299 for line in f: 

1300 (dev, mountpoint, fstype, opts, freq, passno) = line.split(' ') 

1301 if mountpoint == '/': 

1302 disk = diskFromPartition(dev) 

1303 if not (disk in disks): 

1304 disks.append(disk) 

1305 SMlog("Dom0 disks: %s" % disks) 

1306 return disks 

1307 

1308 

1309def set_scheduler_sysfs_node(node, scheds): 

1310 """ 

1311 Set the scheduler for a sysfs node (e.g. '/sys/block/sda') 

1312 according to prioritized list schedulers 

1313 Try to set the first item, then fall back to the next on failure 

1314 """ 

1315 

1316 path = os.path.join(node, "queue", "scheduler") 

1317 if not os.path.exists(path): 1317 ↛ 1321line 1317 didn't jump to line 1321, because the condition on line 1317 was never false

1318 SMlog("no path %s" % path) 

1319 return 

1320 

1321 stored_error = None 

1322 for sched in scheds: 

1323 try: 

1324 with open(path, 'w') as file: 

1325 file.write("%s\n" % sched) 

1326 SMlog("Set scheduler to [%s] on [%s]" % (sched, node)) 

1327 return 

1328 except (OSError, IOError) as err: 

1329 stored_error = err 

1330 

1331 SMlog("Error setting schedulers to [%s] on [%s], %s" % (scheds, node, str(stored_error))) 

1332 

1333 

1334def set_scheduler(dev, schedulers=None): 

1335 if schedulers is None: 1335 ↛ 1338line 1335 didn't jump to line 1338, because the condition on line 1335 was never false

1336 schedulers = ["none", "noop"] 

1337 

1338 devices = [] 

1339 if not scsiutil.match_dm(dev): 1339 ↛ 1343line 1339 didn't jump to line 1343, because the condition on line 1339 was never false

1340 # Remove partition numbers 

1341 devices.append(diskFromPartition(dev).replace('/', '!')) 

1342 else: 

1343 rawdev = diskFromPartition(dev) 

1344 devices = [os.path.realpath(x)[5:] for x in scsiutil._genReverseSCSIidmap(rawdev.split('/')[-1])] 

1345 

1346 for d in devices: 

1347 set_scheduler_sysfs_node("/sys/block/%s" % d, schedulers) 

1348 

1349 

1350# This function queries XAPI for the existing VDI records for this SR 

1351def _getVDIs(srobj): 

1352 VDIs = [] 

1353 try: 

1354 sr_ref = getattr(srobj, 'sr_ref') 

1355 except AttributeError: 

1356 return VDIs 

1357 

1358 refs = srobj.session.xenapi.SR.get_VDIs(sr_ref) 

1359 for vdi in refs: 

1360 ref = srobj.session.xenapi.VDI.get_record(vdi) 

1361 ref['vdi_ref'] = vdi 

1362 VDIs.append(ref) 

1363 return VDIs 

1364 

1365 

1366def get_sr_uuid_from_vdi_ref(session, vdi_ref: str) -> str: 

1367 sr_ref = session.xenapi.VDI.get_SR(vdi_ref) 

1368 return session.xenapi.SR.get_uuid(sr_ref) 

1369 

1370 

1371def get_sr_uuid_from_vdi_uuid(session, vdi_uuid: str) -> str: 

1372 return get_sr_uuid_from_vdi_ref(session, session.xenapi.VDI.get_by_uuid(vdi_uuid)) 

1373 

1374 

1375def _getVDI(srobj, vdi_uuid): 

1376 vdi = srobj.session.xenapi.VDI.get_by_uuid(vdi_uuid) 

1377 ref = srobj.session.xenapi.VDI.get_record(vdi) 

1378 ref['vdi_ref'] = vdi 

1379 return ref 

1380 

1381 

1382def _convertDNS(name): 

1383 addr = socket.getaddrinfo(name, None)[0][4][0] 

1384 return addr 

1385 

1386 

1387def _containsVDIinuse(srobj): 

1388 VDIs = _getVDIs(srobj) 

1389 for vdi in VDIs: 

1390 if not vdi['managed']: 

1391 continue 

1392 sm_config = vdi['sm_config'] 

1393 if 'SRRef' in sm_config: 

1394 try: 

1395 PBDs = srobj.session.xenapi.SR.get_PBDs(sm_config['SRRef']) 

1396 for pbd in PBDs: 

1397 record = PBDs[pbd] 

1398 if record["host"] == srobj.host_ref and \ 

1399 record["currently_attached"]: 

1400 return True 

1401 except: 

1402 pass 

1403 return False 

1404 

1405 

1406def isVDICommand(cmd): 

1407 if cmd is None or cmd in ["vdi_attach", "vdi_detach", 

1408 "vdi_activate", "vdi_deactivate", 

1409 "vdi_epoch_begin", "vdi_epoch_end"]: 

1410 return True 

1411 else: 

1412 return False 

1413 

1414 

1415######################### 

1416# Daemon helper functions 

1417def p_id_fork(): 

1418 try: 

1419 p_id = os.fork() 

1420 except OSError as e: 

1421 print("Fork failed: %s (%d)" % (e.strerror, e.errno)) 

1422 sys.exit(-1) 

1423 

1424 if (p_id == 0): 

1425 os.setsid() 

1426 try: 

1427 p_id = os.fork() 

1428 except OSError as e: 

1429 print("Fork failed: %s (%d)" % (e.strerror, e.errno)) 

1430 sys.exit(-1) 

1431 if (p_id == 0): 

1432 os.chdir('/opt/xensource/sm') 

1433 os.umask(0) 

1434 else: 

1435 os._exit(0) 

1436 else: 

1437 os._exit(0) 

1438 

1439 

1440def daemon(): 

1441 p_id_fork() 

1442 # Query the max file descriptor parameter for this process 

1443 maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] 

1444 

1445 # Close any fds that are open 

1446 for fd in range(0, maxfd): 

1447 try: 

1448 os.close(fd) 

1449 except: 

1450 pass 

1451 

1452 # Redirect STDIN to STDOUT and STDERR 

1453 os.open('/dev/null', os.O_RDWR) 

1454 os.dup2(0, 1) 

1455 os.dup2(0, 2) 

1456 

1457################################################################################ 

1458# 

1459# Fist points 

1460# 

1461 

1462# * The global variable 'fistpoint' define the list of all possible fistpoints; 

1463# 

1464# * To activate a fistpoint called 'name', you need to create the file '/tmp/fist_name' 

1465# on the SR master; 

1466# 

1467# * At the moment, activating a fist point can lead to two possible behaviors: 

1468# - if '/tmp/fist_LVHDRT_exit' exists, then the function called during the fistpoint is _exit; 

1469# - otherwise, the function called is _pause. 

1470 

1471def _pause(secs, name): 

1472 SMlog("Executing fist point %s: sleeping %d seconds ..." % (name, secs)) 

1473 time.sleep(secs) 

1474 SMlog("Executing fist point %s: done" % name) 

1475 

1476 

1477def _exit(name): 

1478 SMlog("Executing fist point %s: exiting the current process ..." % name) 

1479 raise xs_errors.XenError('FistPoint', opterr='%s' % name) 

1480 

1481 

1482class FistPoint: 

1483 def __init__(self, points): 

1484 #SMlog("Fist points loaded") 

1485 self.points = points 

1486 

1487 def is_legal(self, name): 

1488 return (name in self.points) 

1489 

1490 def is_active(self, name): 

1491 return os.path.exists("/tmp/fist_%s" % name) 

1492 

1493 def mark_sr(self, name, sruuid, started): 

1494 with ApiSession("SM-sr-fist-point") as session: 

1495 sr = session.xenapi.SR.get_by_uuid(sruuid) 

1496 if started: 

1497 session.xenapi.SR.add_to_other_config(sr, name, "active") 

1498 else: 

1499 session.xenapi.SR.remove_from_other_config(sr, name) 

1500 

1501 def activate(self, name, sruuid): 

1502 if name in self.points: 

1503 if self.is_active(name): 

1504 self.mark_sr(name, sruuid, True) 

1505 if self.is_active("LVHDRT_exit"): 1505 ↛ 1506line 1505 didn't jump to line 1506, because the condition on line 1505 was never true

1506 self.mark_sr(name, sruuid, False) 

1507 _exit(name) 

1508 else: 

1509 _pause(FIST_PAUSE_PERIOD, name) 

1510 self.mark_sr(name, sruuid, False) 

1511 else: 

1512 SMlog("Unknown fist point: %s" % name) 

1513 

1514 def activate_custom_fn(self, name, fn): 

1515 if name in self.points: 1515 ↛ 1521line 1515 didn't jump to line 1521, because the condition on line 1515 was never false

1516 if self.is_active(name): 1516 ↛ 1517line 1516 didn't jump to line 1517, because the condition on line 1516 was never true

1517 SMlog("Executing fist point %s: starting ..." % name) 

1518 fn() 

1519 SMlog("Executing fist point %s: done" % name) 

1520 else: 

1521 SMlog("Unknown fist point: %s" % name) 

1522 

1523 

1524def list_find(f, seq): 

1525 for item in seq: 

1526 if f(item): 

1527 return item 

1528 

1529GCPAUSE_FISTPOINT = "GCLoop_no_pause" 

1530 

1531fistpoint = FistPoint(["LVHDRT_finding_a_suitable_pair", 

1532 "LVHDRT_inflating_the_parent", 

1533 "LVHDRT_resizing_while_vdis_are_paused", 

1534 "LVHDRT_coalescing_VHD_data", 

1535 "LVHDRT_coalescing_before_inflate_grandparent", 

1536 "LVHDRT_relinking_grandchildren", 

1537 "LVHDRT_before_create_relink_journal", 

1538 "LVHDRT_xapiSM_serialization_tests", 

1539 "LVHDRT_clone_vdi_after_create_journal", 

1540 "LVHDRT_clone_vdi_after_shrink_parent", 

1541 "LVHDRT_clone_vdi_after_first_snap", 

1542 "LVHDRT_clone_vdi_after_second_snap", 

1543 "LVHDRT_clone_vdi_after_parent_hidden", 

1544 "LVHDRT_clone_vdi_after_parent_ro", 

1545 "LVHDRT_clone_vdi_before_remove_journal", 

1546 "LVHDRT_clone_vdi_after_lvcreate", 

1547 "LVHDRT_clone_vdi_before_undo_clone", 

1548 "LVHDRT_clone_vdi_after_undo_clone", 

1549 "LVHDRT_inflate_after_create_journal", 

1550 "LVHDRT_inflate_after_setSize", 

1551 "LVHDRT_inflate_after_zeroOut", 

1552 "LVHDRT_inflate_after_setSizePhys", 

1553 "LVHDRT_inflate_after_setSizePhys", 

1554 "LVHDRT_coaleaf_before_coalesce", 

1555 "LVHDRT_coaleaf_after_coalesce", 

1556 "LVHDRT_coaleaf_one_renamed", 

1557 "LVHDRT_coaleaf_both_renamed", 

1558 "LVHDRT_coaleaf_after_vdirec", 

1559 "LVHDRT_coaleaf_before_delete", 

1560 "LVHDRT_coaleaf_after_delete", 

1561 "LVHDRT_coaleaf_before_remove_j", 

1562 "LVHDRT_coaleaf_undo_after_rename", 

1563 "LVHDRT_coaleaf_undo_after_rename2", 

1564 "LVHDRT_coaleaf_undo_after_refcount", 

1565 "LVHDRT_coaleaf_undo_after_deflate", 

1566 "LVHDRT_coaleaf_undo_end", 

1567 "LVHDRT_coaleaf_stop_after_recovery", 

1568 "LVHDRT_coaleaf_finish_after_inflate", 

1569 "LVHDRT_coaleaf_finish_end", 

1570 "LVHDRT_coaleaf_delay_1", 

1571 "LVHDRT_coaleaf_delay_2", 

1572 "LVHDRT_coaleaf_delay_3", 

1573 "testsm_clone_allow_raw", 

1574 "xenrt_default_vdi_type_legacy", 

1575 "blktap_activate_inject_failure", 

1576 "blktap_activate_error_handling", 

1577 GCPAUSE_FISTPOINT, 

1578 "cleanup_coalesceVHD_inject_failure", 

1579 "cleanup_tracker_no_progress", 

1580 "FileSR_fail_hardlink", 

1581 "FileSR_fail_snap1", 

1582 "FileSR_fail_snap2", 

1583 "LVM_journaler_exists", 

1584 "LVM_journaler_none", 

1585 "LVM_journaler_badname", 

1586 "LVM_journaler_readfail", 

1587 "LVM_journaler_writefail"]) 

1588 

1589 

1590def set_dirty(session, sr): 

1591 try: 

1592 session.xenapi.SR.add_to_other_config(sr, "dirty", "") 

1593 SMlog("set_dirty %s succeeded" % (repr(sr))) 

1594 except: 

1595 SMlog("set_dirty %s failed (flag already set?)" % (repr(sr))) 

1596 

1597 

1598def doesFileHaveOpenHandles(fileName): 

1599 SMlog("Entering doesFileHaveOpenHandles with file: %s" % fileName) 

1600 (retVal, processAndPidTuples) = \ 

1601 findRunningProcessOrOpenFile(fileName, False) 

1602 

1603 if not retVal: 

1604 SMlog("Failed to determine if file %s has open handles." % \ 

1605 fileName) 

1606 # err on the side of caution 

1607 return True 

1608 else: 

1609 if len(processAndPidTuples) > 0: 

1610 return True 

1611 else: 

1612 return False 

1613 

1614 

1615# extract SR uuid from the passed in devmapper entry and return 

1616# /dev/mapper/VG_XenStorage--c3d82e92--cb25--c99b--b83a--482eebab4a93-MGT 

1617def extractSRFromDevMapper(path): 

1618 try: 

1619 path = os.path.basename(path) 

1620 path = path[len('VG_XenStorage-') + 1:] 

1621 path = path.replace('--', '/') 

1622 path = path[0:path.rfind('-')] 

1623 return path.replace('/', '-') 

1624 except: 

1625 return '' 

1626 

1627 

1628def pid_is_alive(pid): 

1629 """ 

1630 Try to kill PID with signal 0. 

1631 If we succeed, the PID is alive, so return True. 

1632 If we get an EPERM error, the PID is alive but we are not allowed to 

1633 signal it. Still return true. 

1634 Any other error (e.g. ESRCH), return False 

1635 """ 

1636 try: 

1637 os.kill(pid, 0) 

1638 return True 

1639 except OSError as e: 

1640 if e.errno == errno.EPERM: 

1641 return True 

1642 return False 

1643 

1644 

1645def get_process_cmdline(pid: int) -> List[str]: 

1646 try: 

1647 with open(os.path.join('/proc', str(pid), 'cmdline'), 'rb') as f: 

1648 line = f.read().split(b'\0') 

1649 return [arg.decode() for arg in line] 

1650 except OSError as e: 

1651 if e.errno != errno.ENOENT: 

1652 raise 

1653 return [] 

1654 

1655 

1656# Looks at /proc and figures either 

1657# If a process is still running (default), returns open file names 

1658# If any running process has open handles to the given file (process = False) 

1659# returns process names and pids 

1660def findRunningProcessOrOpenFile(name, process=True): 

1661 retVal = True 

1662 links = [] 

1663 processandpids = [] 

1664 sockets = set() 

1665 try: 

1666 SMlog("Entering findRunningProcessOrOpenFile with params: %s" % \ 

1667 [name, process]) 

1668 

1669 # Look at all pids 

1670 pids = [pid for pid in os.listdir('/proc') if pid.isdigit()] 

1671 for pid in sorted(pids): 

1672 try: 

1673 try: 

1674 f = None 

1675 f = open(os.path.join('/proc', pid, 'cmdline'), 'r') 

1676 prog = f.read()[:-1] 

1677 if prog: 1677 ↛ 1686line 1677 didn't jump to line 1686, because the condition on line 1677 was never false

1678 # Just want the process name 

1679 argv = prog.split('\x00') 

1680 prog = argv[0] 

1681 except IOError as e: 

1682 if e.errno in (errno.ENOENT, errno.ESRCH): 

1683 SMlog("ERROR %s reading %s, ignore" % (e.errno, pid)) 

1684 continue 

1685 finally: 

1686 if f is not None: 1686 ↛ 1671,   1686 ↛ 16892 missed branches: 1) line 1686 didn't jump to line 1671, because the continue on line 1684 wasn't executed, 2) line 1686 didn't jump to line 1689, because the condition on line 1686 was never false

1687 f.close() 1687 ↛ 1671line 1687 didn't jump to line 1671, because the continue on line 1684 wasn't executed

1688 

1689 try: 

1690 fd_dir = os.path.join('/proc', pid, 'fd') 

1691 files = os.listdir(fd_dir) 

1692 except OSError as e: 

1693 if e.errno in (errno.ENOENT, errno.ESRCH): 

1694 SMlog("ERROR %s reading fds for %s, ignore" % (e.errno, pid)) 

1695 # Ignore pid that are no longer valid 

1696 continue 

1697 else: 

1698 raise 

1699 

1700 for file in files: 

1701 try: 

1702 link = os.readlink(os.path.join(fd_dir, file)) 

1703 except OSError: 

1704 continue 

1705 

1706 if process: 1706 ↛ 1711line 1706 didn't jump to line 1711, because the condition on line 1706 was never false

1707 if name == prog: 1707 ↛ 1700line 1707 didn't jump to line 1700, because the condition on line 1707 was never false

1708 links.append(link) 

1709 else: 

1710 # need to return process name and pid tuples 

1711 if link == name: 

1712 processandpids.append((prog, pid)) 

1713 

1714 # Get the connected sockets 

1715 if name == prog: 

1716 sockets.update(get_connected_sockets(pid)) 

1717 

1718 # We will only have a non-empty processandpids if some fd entries were found. 

1719 # Before returning them, verify that all the PIDs in question are properly alive. 

1720 # There is no specific guarantee of when a PID's /proc directory will disappear 

1721 # when it exits, particularly relative to filedescriptor cleanup, so we want to 

1722 # make sure we're not reporting a false positive. 

1723 processandpids = [x for x in processandpids if pid_is_alive(int(x[1]))] 

1724 for pp in processandpids: 1724 ↛ 1725line 1724 didn't jump to line 1725, because the loop on line 1724 never started

1725 SMlog(f"File {name} has an open handle with process {pp[0]} with pid {pp[1]}") 

1726 

1727 except Exception as e: 

1728 SMlog("Exception checking running process or open file handles. " \ 

1729 "Error: %s" % str(e)) 

1730 retVal = False 

1731 

1732 if process: 1732 ↛ 1735line 1732 didn't jump to line 1735, because the condition on line 1732 was never false

1733 return retVal, links, sockets 

1734 else: 

1735 return retVal, processandpids 

1736 

1737 

1738def get_connected_sockets(pid): 

1739 sockets = set() 

1740 try: 

1741 # Lines in /proc/<pid>/net/unix are formatted as follows 

1742 # (see Linux source net/unix/af_unix.c, unix_seq_show() ) 

1743 # - Pointer address to socket (hex) 

1744 # - Refcount (HEX) 

1745 # - 0 

1746 # - State (HEX, 0 or __SO_ACCEPTCON) 

1747 # - Type (HEX - but only 0001 of interest) 

1748 # - Connection state (HEX - but only 03, SS_CONNECTED of interest) 

1749 # - Inode number 

1750 # - Path (optional) 

1751 open_sock_matcher = re.compile( 

1752 r'^[0-9a-f]+: [0-9A-Fa-f]+ [0-9A-Fa-f]+ [0-9A-Fa-f]+ 0001 03 \d+ (.*)$') 

1753 with open( 

1754 os.path.join('/proc', str(pid), 'net', 'unix'), 'r') as f: 

1755 lines = f.readlines() 

1756 for line in lines: 

1757 match = open_sock_matcher.match(line) 

1758 if match: 

1759 sockets.add(match[1]) 

1760 except OSError as e: 

1761 if e.errno in (errno.ENOENT, errno.ESRCH): 

1762 # Ignore pid that are no longer valid 

1763 SMlog("ERROR %s reading sockets for %s, ignore" % 

1764 (e.errno, pid)) 

1765 else: 

1766 raise 

1767 return sockets 

1768 

1769 

1770def retry(f, maxretry=20, period=3, exceptions=[Exception]): 

1771 retries = 0 

1772 while True: 

1773 try: 

1774 return f() 

1775 except Exception as e: 

1776 for exception in exceptions: 

1777 if isinstance(e, exception): 

1778 SMlog('Got exception: {}. Retry number: {}'.format( 

1779 str(e), retries 

1780 )) 

1781 break 

1782 else: 

1783 SMlog('Got bad exception: {}. Raising...'.format(e)) 

1784 raise e 

1785 

1786 retries += 1 

1787 if retries >= maxretry: 

1788 break 

1789 

1790 time.sleep(period) 

1791 

1792 return f() 

1793 

1794 

1795def getCslDevPath(svid): 

1796 basepath = "/dev/disk/by-csldev/" 

1797 if svid.startswith("NETAPP_"): 

1798 # special attention for NETAPP SVIDs 

1799 svid_parts = svid.split("__") 

1800 globstr = basepath + "NETAPP__LUN__" + "*" + svid_parts[2] + "*" + svid_parts[-1] + "*" 

1801 else: 

1802 globstr = basepath + svid + "*" 

1803 

1804 return globstr 

1805 

1806 

1807# Use device in /dev pointed to by cslg path which consists of svid 

1808def get_scsiid_from_svid(md_svid): 

1809 cslg_path = getCslDevPath(md_svid) 

1810 abs_path = glob.glob(cslg_path) 

1811 if abs_path: 

1812 real_path = os.path.realpath(abs_path[0]) 

1813 return scsiutil.getSCSIid(real_path) 

1814 else: 

1815 return None 

1816 

1817 

1818def get_isl_scsiids(session): 

1819 # Get cslg type SRs 

1820 SRs = session.xenapi.SR.get_all_records_where('field "type" = "cslg"') 

1821 

1822 # Iterate through the SR to get the scsi ids 

1823 scsi_id_ret = [] 

1824 for SR in SRs: 

1825 sr_rec = SRs[SR] 

1826 # Use the md_svid to get the scsi id 

1827 scsi_id = get_scsiid_from_svid(sr_rec['sm_config']['md_svid']) 

1828 if scsi_id: 

1829 scsi_id_ret.append(scsi_id) 

1830 

1831 # Get the vdis in the SR and do the same procedure 

1832 vdi_recs = session.xenapi.VDI.get_all_records_where('field "SR" = "%s"' % SR) 

1833 for vdi_rec in vdi_recs: 

1834 vdi_rec = vdi_recs[vdi_rec] 

1835 scsi_id = get_scsiid_from_svid(vdi_rec['sm_config']['SVID']) 

1836 if scsi_id: 

1837 scsi_id_ret.append(scsi_id) 

1838 

1839 return scsi_id_ret 

1840 

1841 

1842class extractXVA: 

1843 # streams files as a set of file and checksum, caller should remove 

1844 # the files, if not needed. The entire directory (Where the files 

1845 # and checksum) will only be deleted as part of class cleanup. 

1846 HDR_SIZE = 512 

1847 BLOCK_SIZE = 512 

1848 SIZE_LEN = 12 - 1 # To remove \0 from tail 

1849 SIZE_OFFSET = 124 

1850 ZERO_FILLED_REC = 2 

1851 NULL_IDEN = '\x00' 

1852 DIR_IDEN = '/' 

1853 CHECKSUM_IDEN = '.checksum' 

1854 OVA_FILE = 'ova.xml' 

1855 

1856 # Init gunzips the file using a subprocess, and reads stdout later 

1857 # as and when needed 

1858 def __init__(self, filename): 

1859 self.__extract_path = '' 

1860 self.__filename = filename 

1861 cmd = 'gunzip -cd %s' % filename 

1862 try: 

1863 self.spawn_p = subprocess.Popen( 

1864 cmd, shell=True, \ 

1865 stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ 

1866 stderr=subprocess.PIPE, close_fds=True) 

1867 except Exception as e: 

1868 SMlog("Error: %s. Uncompress failed for %s" % (str(e), filename)) 

1869 raise Exception(str(e)) 

1870 

1871 # Create dir to extract the files 

1872 self.__extract_path = tempfile.mkdtemp() 

1873 

1874 def __del__(self): 

1875 shutil.rmtree(self.__extract_path) 

1876 

1877 # Class supports Generator expression. 'for f_name, checksum in getTuple()' 

1878 # returns filename, checksum content. Returns filename, '' in case 

1879 # of checksum file missing. e.g. ova.xml 

1880 def getTuple(self): 

1881 zerod_record = 0 

1882 ret_f_name = '' 

1883 ret_base_f_name = '' 

1884 

1885 try: 

1886 # Read tar file as sets of file and checksum. 

1887 while True: 

1888 # Read the output of spawned process, or output of gunzip 

1889 f_hdr = self.spawn_p.stdout.read(self.HDR_SIZE) 

1890 

1891 # Break out in case of end of file 

1892 if f_hdr == '': 

1893 if zerod_record == extractXVA.ZERO_FILLED_REC: 

1894 break 

1895 else: 

1896 SMlog('Error. Expects %d zero records', \ 

1897 extractXVA.ZERO_FILLED_REC) 

1898 raise Exception('Unrecognized end of file') 

1899 

1900 # Watch out for zero records, two zero records 

1901 # denote end of file. 

1902 if f_hdr == extractXVA.NULL_IDEN * extractXVA.HDR_SIZE: 

1903 zerod_record += 1 

1904 continue 

1905 

1906 f_name = f_hdr[:f_hdr.index(extractXVA.NULL_IDEN)] 

1907 # File header may be for a folder, if so ignore the header 

1908 if not f_name.endswith(extractXVA.DIR_IDEN): 

1909 f_size_octal = f_hdr[extractXVA.SIZE_OFFSET: \ 

1910 extractXVA.SIZE_OFFSET + extractXVA.SIZE_LEN] 

1911 f_size = int(f_size_octal, 8) 

1912 if f_name.endswith(extractXVA.CHECKSUM_IDEN): 

1913 if f_name.rstrip(extractXVA.CHECKSUM_IDEN) == \ 

1914 ret_base_f_name: 

1915 checksum = self.spawn_p.stdout.read(f_size) 

1916 yield(ret_f_name, checksum) 

1917 else: 

1918 # Expects file followed by its checksum 

1919 SMlog('Error. Sequence mismatch starting with %s', \ 

1920 ret_f_name) 

1921 raise Exception( \ 

1922 'Files out of sequence starting with %s', \ 

1923 ret_f_name) 

1924 else: 

1925 # In case of ova.xml, read the contents into a file and 

1926 # return the file name to the caller. For other files, 

1927 # read the contents into a file, it will 

1928 # be used when a .checksum file is encountered. 

1929 ret_f_name = '%s/%s' % (self.__extract_path, f_name) 

1930 ret_base_f_name = f_name 

1931 

1932 # Check if the folder exists on the target location, 

1933 # else create it. 

1934 folder_path = ret_f_name[:ret_f_name.rfind('/')] 

1935 if not os.path.exists(folder_path): 

1936 os.mkdir(folder_path) 

1937 

1938 # Store the file to the tmp folder, strip the tail \0 

1939 f = open(ret_f_name, 'w') 

1940 f.write(self.spawn_p.stdout.read(f_size)) 

1941 f.close() 

1942 if f_name == extractXVA.OVA_FILE: 

1943 yield(ret_f_name, '') 

1944 

1945 # Skip zero'd portion of data block 

1946 round_off = f_size % extractXVA.BLOCK_SIZE 

1947 if round_off != 0: 

1948 zeros = self.spawn_p.stdout.read( 

1949 extractXVA.BLOCK_SIZE - round_off) 

1950 except Exception as e: 

1951 SMlog("Error: %s. File set extraction failed %s" % (str(e), \ 

1952 self.__filename)) 

1953 

1954 # Kill and Drain stdout of the gunzip process, 

1955 # else gunzip might block on stdout 

1956 os.kill(self.spawn_p.pid, signal.SIGTERM) 

1957 self.spawn_p.communicate() 

1958 raise Exception(str(e)) 

1959 

1960illegal_xml_chars = [(0x00, 0x08), (0x0B, 0x1F), (0x7F, 0x84), (0x86, 0x9F), 

1961 (0xD800, 0xDFFF), (0xFDD0, 0xFDDF), (0xFFFE, 0xFFFF), 

1962 (0x1FFFE, 0x1FFFF), (0x2FFFE, 0x2FFFF), (0x3FFFE, 0x3FFFF), 

1963 (0x4FFFE, 0x4FFFF), (0x5FFFE, 0x5FFFF), (0x6FFFE, 0x6FFFF), 

1964 (0x7FFFE, 0x7FFFF), (0x8FFFE, 0x8FFFF), (0x9FFFE, 0x9FFFF), 

1965 (0xAFFFE, 0xAFFFF), (0xBFFFE, 0xBFFFF), (0xCFFFE, 0xCFFFF), 

1966 (0xDFFFE, 0xDFFFF), (0xEFFFE, 0xEFFFF), (0xFFFFE, 0xFFFFF), 

1967 (0x10FFFE, 0x10FFFF)] 

1968 

1969illegal_ranges = ["%s-%s" % (chr(low), chr(high)) 

1970 for (low, high) in illegal_xml_chars 

1971 if low < sys.maxunicode] 

1972 

1973illegal_xml_re = re.compile(u'[%s]' % u''.join(illegal_ranges)) 

1974 

1975 

1976def isLegalXMLString(s): 

1977 """Tells whether this is a valid XML string (i.e. it does not contain 

1978 illegal XML characters specified in 

1979 http://www.w3.org/TR/2004/REC-xml-20040204/#charsets). 

1980 """ 

1981 

1982 if len(s) > 0: 

1983 return re.search(illegal_xml_re, s) is None 

1984 else: 

1985 return True 

1986 

1987 

1988def unictrunc(string, max_bytes): 

1989 """ 

1990 Given a string, returns the largest number of elements for a prefix 

1991 substring of it, such that the UTF-8 encoding of this substring takes no 

1992 more than the given number of bytes. 

1993 

1994 The string may be given as a unicode string or a UTF-8 encoded byte 

1995 string, and the number returned will be in characters or bytes 

1996 accordingly. Note that in the latter case, the substring will still be a 

1997 valid UTF-8 encoded string (which is to say, it won't have been truncated 

1998 part way through a multibyte sequence for a unicode character). 

1999 

2000 string: the string to truncate 

2001 max_bytes: the maximum number of bytes the truncated string can be 

2002 """ 

2003 if isinstance(string, str): 

2004 return_chars = True 

2005 else: 

2006 return_chars = False 

2007 string = string.decode('UTF-8') 

2008 

2009 cur_chars = 0 

2010 cur_bytes = 0 

2011 for char in string: 

2012 charsize = len(char.encode('UTF-8')) 

2013 if cur_bytes + charsize > max_bytes: 

2014 break 

2015 else: 

2016 cur_chars += 1 

2017 cur_bytes += charsize 

2018 return cur_chars if return_chars else cur_bytes 

2019 

2020 

2021def hideValuesInPropMap(propmap, propnames): 

2022 """ 

2023 Worker function: input simple map of prop name/value pairs, and 

2024 a list of specific propnames whose values we want to hide. 

2025 Loop through the "hide" list, and if any are found, hide the 

2026 value and return the altered map. 

2027 If none found, return the original map 

2028 """ 

2029 matches = [] 

2030 for propname in propnames: 

2031 if propname in propmap: 2031 ↛ 2032line 2031 didn't jump to line 2032, because the condition on line 2031 was never true

2032 matches.append(propname) 

2033 

2034 if matches: 2034 ↛ 2035line 2034 didn't jump to line 2035, because the condition on line 2034 was never true

2035 deepCopyRec = copy.deepcopy(propmap) 

2036 for match in matches: 

2037 deepCopyRec[match] = '******' 

2038 return deepCopyRec 

2039 

2040 return propmap 

2041# define the list of propnames whose value we want to hide 

2042 

2043PASSWD_PROP_KEYS = ['password', 'cifspassword', 'chappassword', 'incoming_chappassword'] 

2044DEFAULT_SEGMENT_LEN = 950 

2045 

2046 

2047def hidePasswdInConfig(config): 

2048 """ 

2049 Function to hide passwd values in a simple prop map, 

2050 for example "device_config" 

2051 """ 

2052 return hideValuesInPropMap(config, PASSWD_PROP_KEYS) 

2053 

2054 

2055def hidePasswdInParams(params, configProp): 

2056 """ 

2057 Function to hide password values in a specified property which 

2058 is a simple map of prop name/values, and is itself an prop entry 

2059 in a larger property map. 

2060 For example, param maps containing "device_config", or 

2061 "sm_config", etc 

2062 """ 

2063 params[configProp] = hideValuesInPropMap(params[configProp], PASSWD_PROP_KEYS) 

2064 return params 

2065 

2066 

2067def hideMemberValuesInXmlParams(xmlParams, propnames=PASSWD_PROP_KEYS): 

2068 """ 

2069 Function to hide password values in XML params, specifically 

2070 for the XML format of incoming params to SR modules. 

2071 Uses text parsing: loop through the list of specific propnames 

2072 whose values we want to hide, and: 

2073 - Assemble a full "prefix" containing each property name, e.g., 

2074 "<member><name>password</name><value>" 

2075 - Test the XML if it contains that string, save the index. 

2076 - If found, get the index of the ending tag 

2077 - Truncate the return string starting with the password value. 

2078 - Append the substitute "*******" value string. 

2079 - Restore the rest of the original string starting with the end tag. 

2080 """ 

2081 findStrPrefixHead = "<member><name>" 

2082 findStrPrefixTail = "</name><value>" 

2083 findStrSuffix = "</value>" 

2084 strlen = len(xmlParams) 

2085 

2086 for propname in propnames: 

2087 findStrPrefix = findStrPrefixHead + propname + findStrPrefixTail 

2088 idx = xmlParams.find(findStrPrefix) 

2089 if idx != -1: # if found any of them 

2090 idx += len(findStrPrefix) 

2091 idx2 = xmlParams.find(findStrSuffix, idx) 

2092 if idx2 != -1: 

2093 retStr = xmlParams[0:idx] 

2094 retStr += "******" 

2095 retStr += xmlParams[idx2:strlen] 

2096 return retStr 

2097 else: 

2098 return xmlParams 

2099 return xmlParams 

2100 

2101 

2102def splitXmlText(xmlData, segmentLen=DEFAULT_SEGMENT_LEN, showContd=False): 

2103 """ 

2104 Split xml string data into substrings small enough for the 

2105 syslog line length limit. Split at tag end markers ( ">" ). 

2106 Usage: 

2107 strList = [] 

2108 strList = splitXmlText( longXmlText, maxLineLen ) # maxLineLen is optional 

2109 """ 

2110 remainingData = str(xmlData) 

2111 

2112 # "Un-pretty-print" 

2113 remainingData = remainingData.replace('\n', '') 

2114 remainingData = remainingData.replace('\t', '') 

2115 

2116 remainingChars = len(remainingData) 

2117 returnData = '' 

2118 

2119 thisLineNum = 0 

2120 while remainingChars > segmentLen: 

2121 thisLineNum = thisLineNum + 1 

2122 index = segmentLen 

2123 tmpStr = remainingData[:segmentLen] 

2124 tmpIndex = tmpStr.rfind('>') 

2125 if tmpIndex != -1: 

2126 index = tmpIndex + 1 

2127 

2128 tmpStr = tmpStr[:index] 

2129 remainingData = remainingData[index:] 

2130 remainingChars = len(remainingData) 

2131 

2132 if showContd: 

2133 if thisLineNum != 1: 

2134 tmpStr = '(Cont\'d): ' + tmpStr 

2135 tmpStr = tmpStr + ' (Cont\'d):' 

2136 

2137 returnData += tmpStr + '\n' 

2138 

2139 if showContd and thisLineNum > 0: 

2140 remainingData = '(Cont\'d): ' + remainingData 

2141 returnData += remainingData 

2142 

2143 return returnData 

2144 

2145 

2146def inject_failure(): 

2147 raise Exception('injected failure') 

2148 

2149 

2150def open_atomic(path, mode=None): 

2151 """Atomically creates a file if, and only if it does not already exist. 

2152 Leaves the file open and returns the file object. 

2153 

2154 path: the path to atomically open 

2155 mode: "r" (read), "w" (write), or "rw" (read/write) 

2156 returns: an open file object""" 

2157 

2158 assert path 

2159 

2160 flags = os.O_CREAT | os.O_EXCL 

2161 modes = {'r': os.O_RDONLY, 'w': os.O_WRONLY, 'rw': os.O_RDWR} 

2162 if mode: 

2163 if mode not in modes: 

2164 raise Exception('invalid access mode ' + mode) 

2165 flags |= modes[mode] 

2166 fd = os.open(path, flags) 

2167 try: 

2168 if mode: 

2169 return os.fdopen(fd, mode) 

2170 else: 

2171 return os.fdopen(fd) 

2172 except: 

2173 os.close(fd) 

2174 raise 

2175 

2176 

2177def isInvalidVDI(exception): 

2178 return exception.details[0] == "HANDLE_INVALID" or \ 

2179 exception.details[0] == "UUID_INVALID" 

2180 

2181 

2182def get_pool_restrictions(session): 

2183 """Returns pool restrictions as a map, @session must be already 

2184 established.""" 

2185 return list(session.xenapi.pool.get_all_records().values())[0]['restrictions'] 

2186 

2187 

2188def read_caching_is_restricted(session): 

2189 """Tells whether read caching is restricted.""" 

2190 if session is None: 2190 ↛ 2191line 2190 didn't jump to line 2191, because the condition on line 2190 was never true

2191 return True 

2192 restrictions = get_pool_restrictions(session) 

2193 if 'restrict_read_caching' in restrictions and \ 2193 ↛ 2195line 2193 didn't jump to line 2195, because the condition on line 2193 was never true

2194 restrictions['restrict_read_caching'] == "true": 

2195 return True 

2196 return False 

2197 

2198 

2199def sessions_less_than_targets(other_config, device_config): 

2200 if 'multihomelist' in device_config and 'iscsi_sessions' in other_config: 

2201 sessions = int(other_config['iscsi_sessions']) 

2202 targets = len(device_config['multihomelist'].split(',')) 

2203 SMlog("Targets %d and iscsi_sessions %d" % (targets, sessions)) 

2204 return (sessions < targets) 

2205 else: 

2206 return False 

2207 

2208 

2209def enable_and_start_service(name, start): 

2210 attempt = 0 

2211 while True: 

2212 attempt += 1 

2213 fn = 'enable' if start else 'disable' 

2214 args = ('systemctl', fn, '--now', name) 

2215 (ret, out, err) = doexec(args) 

2216 if ret == 0: 

2217 return 

2218 elif attempt >= 3: 

2219 raise Exception( 

2220 'Failed to {} {}: {} {}'.format(fn, name, out, err) 

2221 ) 

2222 time.sleep(1) 

2223 

2224 

2225def stop_service(name): 

2226 args = ('systemctl', 'stop', name) 

2227 (ret, out, err) = doexec(args) 

2228 if ret == 0: 

2229 return 

2230 raise Exception('Failed to stop {}: {} {}'.format(name, out, err)) 

2231 

2232 

2233def restart_service(name): 

2234 attempt = 0 

2235 while True: 

2236 attempt += 1 

2237 SMlog('Restarting service {} {}...'.format(name, attempt)) 

2238 args = ('systemctl', 'restart', name) 

2239 (ret, out, err) = doexec(args) 

2240 if ret == 0: 

2241 return 

2242 elif attempt >= 3: 

2243 SMlog('Restart service FAILED {} {}'.format(name, attempt)) 

2244 raise Exception( 

2245 'Failed to restart {}: {} {}'.format(name, out, err) 

2246 ) 

2247 time.sleep(1) 

2248 

2249 

2250def check_pid_exists(pid): 

2251 try: 

2252 os.kill(pid, 0) 

2253 except OSError: 

2254 return False 

2255 else: 

2256 return True 

2257 

2258 

2259def get_openers_pid(path: str) -> Optional[List[int]]: 

2260 cmd = ["lsof", "-t", path] 

2261 

2262 try: 

2263 list = [] 

2264 ret = pread2(cmd) 

2265 for line in ret.splitlines(): 

2266 list.append(int(line)) 

2267 return list 

2268 except CommandException as e: 

2269 if e.code == 1: # `lsof` return 1 if there is no openers 

2270 return None 

2271 else: 

2272 raise e 

2273 

2274 

2275def make_profile(name, function): 

2276 """ 

2277 Helper to execute cProfile using unique log file. 

2278 """ 

2279 

2280 import cProfile 

2281 import itertools 

2282 import os.path 

2283 import time 

2284 

2285 assert name 

2286 assert function 

2287 

2288 FOLDER = '/tmp/sm-perfs/' 

2289 makedirs(FOLDER) 

2290 

2291 filename = time.strftime('{}_%Y%m%d_%H%M%S.prof'.format(name)) 

2292 

2293 def gen_path(path): 

2294 yield path 

2295 root, ext = os.path.splitext(path) 

2296 for i in itertools.count(start=1, step=1): 

2297 yield root + '.{}.'.format(i) + ext 

2298 

2299 for profile_path in gen_path(FOLDER + filename): 

2300 try: 

2301 file = open_atomic(profile_path, 'w') 

2302 file.close() 

2303 break 

2304 except OSError as e: 

2305 if e.errno == errno.EEXIST: 

2306 pass 

2307 else: 

2308 raise 

2309 

2310 try: 

2311 SMlog('* Start profiling of {} ({}) *'.format(name, filename)) 

2312 cProfile.runctx('function()', None, locals(), profile_path) 

2313 finally: 

2314 SMlog('* End profiling of {} ({}) *'.format(name, filename)) 

2315 

2316 

2317def strtobool(str: str) -> bool: 

2318 # Note: `distutils` package is deprecated and slated for removal in Python 3.12. 

2319 # There is not alternative for strtobool. 

2320 # See: https://peps.python.org/pep-0632/#migration-advice 

2321 # So this is a custom implementation with differences: 

2322 # - A boolean is returned instead of integer 

2323 # - Empty string and None are supported (False is returned in this case) 

2324 if not str: 2324 ↛ 2326line 2324 didn't jump to line 2326, because the condition on line 2324 was never false

2325 return False 

2326 str = str.lower() 

2327 if str in ('y', 'yes', 't', 'true', 'on', '1'): 

2328 return True 

2329 if str in ('n', 'no', 'f', 'false', 'off', '0'): 

2330 return False 

2331 raise ValueError("invalid truth value '{}'".format(str)) 

2332 

2333 

2334def find_executable(name): 

2335 return shutil.which(name) 

2336 

2337 

2338def conditional_decorator(decorator, condition): 

2339 def wrapper(func): 

2340 if not condition: 2340 ↛ 2342line 2340 didn't jump to line 2342, because the condition on line 2340 was never false

2341 return func 

2342 return decorator(func) 

2343 return wrapper 

2344 

2345 

2346def get_srs_uuid_from_type(session, type): 

2347 srs = session.xenapi.SR.get_all_records_where(f"field \"type\" = \"{type}\"") 

2348 return {sr["uuid"]: sr for sr in srs.values()} 

2349 

2350 

2351def get_linstor_srs_uuid(session): 

2352 import LinstorSR # pylint: disable=C0415 

2353 return get_srs_uuid_from_type(session, LinstorSR.LinstorSR.DRIVER_TYPE) 

2354 

2355 

2356def find_pbd_ref_from_dconf_value(session, srs, key, value, value_modifier = None): 

2357 for sr in srs.values(): 

2358 for pbd_ref in sr["PBDs"]: 

2359 device_config = session.xenapi.PBD.get_device_config(pbd_ref) 

2360 cur_value = device_config.get(key) 

2361 if value_modifier: 

2362 cur_value = value_modifier(cur_value) 

2363 if cur_value and cur_value == value: 

2364 return pbd_ref 

2365 return None