Coverage for drivers/qcow2util.py: 26%

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1#!/usr/bin/env python3

5# This program is free software: you can redistribute it and/or modify

6# it under the terms of the GNU General Public License as published by

7# the Free Software Foundation, either version 3 of the License, or

8# (at your option) any later version.

9# This program is distributed in the hope that it will be useful,

10# but WITHOUT ANY WARRANTY; without even the implied warranty of

11# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the

12# GNU General Public License for more details.

13#

14# You should have received a copy of the GNU General Public License

15# along with this program. If not, see <https://www.gnu.org/licenses/>.

17from sm_typing import Any, Callable, Dict, Final, List, Optional, Tuple, cast, override

18from typing import BinaryIO

20import errno

21import os

22import re

23import time

24import struct

25import zlib

26import json

27from pathlib import Path

29import util

30import xs_errors

31from blktap2 import TapCtl

32from cowutil import CowUtil, CowImageInfo

33from lvmcache import LVMCache

34from constants import NS_PREFIX_LVM, VG_PREFIX

36# ------------------------------------------------------------------------------

38MAX_QCOW_CHAIN_LENGTH: Final = 30

40QCOW2_DEFAULT_CLUSTER_SIZE: Final = 64 * 1024 # 64 KiB

42MIN_QCOW_SIZE: Final = QCOW2_DEFAULT_CLUSTER_SIZE

44MAX_QCOW_SIZE: Final = 17589500641280 # Max size so that the fully allocated size with metadata is under the max size of EXT4 (17592185061376 bytes fully allocated)

46QEMU_IMG: Final = "/usr/bin/qemu-img"

47QCOW2_HELPER = "/opt/xensource/libexec/qcow2_helper"

49QCOW2_TYPE: Final = "qcow2"

50RAW_TYPE: Final = "raw"

52# ------------------------------------------------------------------------------

54class QCowUtil(CowUtil):

56 # We followed specifications found here:

57 # https://github.com/qemu/qemu/blob/master/docs/interop/qcow2.txt

59 QCOW2_MAGIC = 0x514649FB # b"QFI\xfb": Magic number for QCOW2 files

60 QCOW2_HEADER_SIZE = 104 # In fact the last information we need is at offset 40-47

61 QCOW2_L2_SIZE = QCOW2_DEFAULT_CLUSTER_SIZE

62 QCOW2_BACKING_FILE_OFFSET = 8

64 ALLOCATED_ENTRY_BIT = (

65 0x8000_0000_0000_0000 # Bit 63 is the allocated bit for standard cluster

66 )

67 CLUSTER_TYPE_BIT = 0x4000_0000_0000_0000 # 0 for standard, 1 for compressed cluster

68 L2_OFFSET_MASK = 0x00FF_FFFF_FFFF_FF00 # Bits 9-55 are offset of L2 table.

69 CLUSTER_DESCRIPTION_MASK = 0x3FFF_FFFF_FFFF_FFFF # Bit 0-61 is cluster description

70 STANDARD_CLUSTER_OFFSET_MASK = (

71 0x00FF_FFFF_FFFF_FF00 # Bits 9-55 are offset of standard cluster

72 )

74 def __init__(self):

75 self.qcow_read = False

77 def _read_qcow2(self, path: str, read_clusters: bool = False):

78 phys_disk_size = self.getSizePhys(path)

79 with open(path, "rb") as qcow2_file:

80 self.filename = path # Keep the filename if clean is called

81 self.header = self._read_qcow2_header(qcow2_file)

82 if read_clusters:

83 self.l1 = self._get_l1_entries(qcow2_file)

84 # The l1_to_l2 allows to get L2 entries for a given L1. If L1 entry

85 # is not allocated we store an empty list.

86 self.l1_to_l2: Dict[int, List[int]] = {}

88 for l1_entry in self.l1:

89 l2_offset = l1_entry & self.L2_OFFSET_MASK

90 if l2_offset == 0:

91 self.l1_to_l2[l1_entry] = []

92 elif l2_offset > phys_disk_size: #TODO: This sometime happen for a correct VDI (while coalescing online?)

93 raise xs_errors.XenError("VDISize", "L2 Offset is bigger than physical disk {}".format(path))

94 else:

95 self.l1_to_l2[l1_entry] = self._get_l2_entries(

96 qcow2_file, l2_offset

97 )

98 self.qcow_read = True

100 def _get_l1_entries(self, file: BinaryIO) -> List[int]:

101 """Returns the list of all L1 entries.

102

103 Args:

104 file: The qcow2 file object.

105

106 Returns:

107 list: List of all L1 entries

108 """

109 l1_table_offset = self.header["l1_table_offset"]

110 file.seek(l1_table_offset)

111

112 l1_table_size = self.header["l1_size"] * 8 # Each L1 entry is 8 bytes

113 l1_table = file.read(l1_table_size)

114

115 return [

116 struct.unpack(">Q", l1_table[i : i + 8])[0]

117 for i in range(0, len(l1_table), 8)

118 ]

119

120 @staticmethod

121 def _get_l2_entries(file: BinaryIO, l2_offset: int) -> List[int]:

122 """Returns the list of all L2 entries at a given L2 offset.

123

124 Args:

125 file: The qcow2 file.

126 l2_offset: the L2 offset where to look for entries

127

128 Returns:

129 list: List of all L2 entries

130 """

131 # The size of L2 is 65536 bytes and each entry is 8 bytes.

132 file.seek(l2_offset)

133 l2_table = file.read(QCowUtil.QCOW2_L2_SIZE)

134

135 return [

136 struct.unpack(">Q", l2_table[i : i + 8])[0]

137 for i in range(0, len(l2_table), 8)

138 ]

139

140 @staticmethod

141 def _read_qcow2_backingfile(file: BinaryIO, backing_file_offset: int , backing_file_size: int) -> str:

142 if backing_file_offset == 0:

143 return ""

144

145 file.seek(backing_file_offset)

146 parent_name = file.read(backing_file_size)

147 return parent_name.decode("UTF-8")

148

149 @staticmethod

150 def _read_qcow2_header(file: BinaryIO) -> Dict[str, Any]:

151 """Returns a dict containing some information from QCow2 header.

152

153 Args:

154 file: The qcow2 file object.

155

156 Returns:

157 dict: magic, version, cluster_bits, l1_size and l1_table_offset.

158

159 Raises:

160 ValueError: if qcow2 magic is not recognized or cluster size not supported.

161 """

162 # The header is as follow:

163 #

164 # magic: u32, // Magic string "QFI\xfb"

165 # version: u32, // Version (2 or 3)

166 # backing_file_offset: u64, // Offset to the backing file name

167 # backing_file_size: u32, // Size of the backing file name

168 # cluster_bits: u32, // Bits used for addressing within a cluster

169 # size: u64, // Virtual disk size

170 # crypt_method: u32, // 0 = no encryption, 1 = AES encryption

171 # l1_size: u32, // Number of entries in the L1 table

172 # l1_table_offset: u64, // Offset to the active L1 table

173 # refcount_table_offset: u64, // Offset to the refcount table

174 # refcount_table_clusters: u32, // Number of clusters for the refcount table

175 # nb_snapshots: u32, // Number of snapshots in the image

176 # snapshots_offset: u64, // Offset to the snapshot table

177

178 file.seek(0)

179 header = file.read(QCowUtil.QCOW2_HEADER_SIZE)

180 (

181 magic,

182 version,

183 backing_file_offset,

184 backing_file_size,

185 cluster_bits,

186 size,

187 _,

188 l1_size,

189 l1_table_offset,

190 refcount_table_offset,

191 _,

192 _,

193 snapshots_offset,

194 ) = struct.unpack(">IIQIIQIIQQIIQ", header[:72])

195

196 if magic != QCowUtil.QCOW2_MAGIC:

197 raise ValueError("Not a valid QCOW2 file")

198

199 parent_name = QCowUtil._read_qcow2_backingfile(file, backing_file_offset, backing_file_size)

200

201 return {

202 "version": version,

203 "backing_file_offset": backing_file_offset,

204 "backing_file_size": backing_file_size,

205 "virtual_disk_size": size,

206 "cluster_bits": cluster_bits,

207 "l1_size": l1_size,

208 "l1_table_offset": l1_table_offset,

209 "refcount_table_offset": refcount_table_offset,

210 "snapshots_offset": snapshots_offset,

211 "parent": parent_name,

212 }

213

214 @staticmethod

215 def _is_l1_allocated(entry: int) -> bool:

216 """Checks if the given L1 entry is allocated.

217

218 If the offset is 0 then the L2 table and all clusters described

219 by this L2 table are unallocated.

220

221 Args:

222 entry: L1 entry

223

224 Returns:

225 bool: True if the L1 entry is allocated (ie has a valid offset).

226 False otherwise.

227 """

228 return (entry & QCowUtil.L2_OFFSET_MASK) != 0

229

230 @staticmethod

231 def _is_l2_allocated(entry: int) -> bool:

232 """Checks if a given entry is allocated.

233

234 Currently we only support standard clusters. And for standard clusters

235 the bit 63 is set to 1 for allocated ones or offset is not 0.

236

237 Args:

238 entry: L2 entry

239

240 Returns:

241 bool: Returns True if the L2 entry is allocated, False otherwise

242

243 Raises:

244 raise an exception if the cluster is not a standard one.

245 """

246 assert entry & QCowUtil.CLUSTER_TYPE_BIT == 0

247 return (entry & QCowUtil.ALLOCATED_ENTRY_BIT != 0) or (

248 entry & QCowUtil.STANDARD_CLUSTER_OFFSET_MASK != 0

249 )

250

251 @staticmethod

252 def _get_allocated_clusters(l2_entries: List[int]) -> List[int]:

253 """Get all allocated clusters in a given list of L2 entries.

254

255 Args:

256 l2_entries: A list of L2 entries.

257

258 Returns:

259 A list of all allocated entries

260 """

261 return [entry for entry in l2_entries if QCowUtil._is_l2_allocated(entry)]

262

263 @staticmethod

264 def _get_cluster_to_byte(clusters: int, cluster_bits: int) -> int:

265 # (1 << cluster_bits) give cluster size in byte

266 return clusters * (1 << cluster_bits)

267

268 def _get_number_of_allocated_clusters(self) -> int:

269 """Get the number of allocated clusters.

270

271 Args:

272 self: A QcowInfo object.

273

274 Returns:

275 An integer that is the list of allocated clusters.

276 """

277 assert(self.qcow_read)

278

279 allocated_clusters = 0

280

281 for l2_entries in self.l1_to_l2.values():

282 allocated_clusters += len(self._get_allocated_clusters(l2_entries))

283

284 return allocated_clusters

285

286 @staticmethod

287 def _move_backing_file(

288 f: BinaryIO, old_offset: int, new_offset: int, data_size: int

289 ) -> None:

290 """Move a number of bytes from old_offset to new_offset and replaces the old

291 value by 0s. It is up to the caller to save the current position in the file

292 if needed.

293

294 Args:

295 f: the file the will be modified

296 old_offset: the current offset

297 new_offset: the new offset where we want to move data

298 data_size: Size in bytes of data that we want to move

299

300 Returns:

301 Nothing but the file f is modified and the position in the file also.

302 """

303 # Read the string at backing_file_offset

304 f.seek(old_offset)

305 data = f.read(data_size)

306

307 # Write zeros at the original location

308 f.seek(old_offset)

309 f.write(b"\x00" * data_size)

310

311 # Write the string to the new location

312 f.seek(new_offset)

313 f.write(data)

314

315 def _add_or_find_custom_header(self) -> int:

316 """Add custom header at the end of header extensions

317

318 It finds the end of the header extensions and add the custom header.

319 If the header already exists nothing is done.

320

321 Args:

322

323 Returns:

324 It returns the data offset where custom header is found or created.

325 If data offset is 0 something weird happens.

326 The qcow2 file in self.filename can be modified.

327 """

328 assert self.qcow_read

329

330 header_length = 72 # This is the default value for version 2 images

331

332 custom_header_type = 0x76617465 # vate: it is easy to recognize with hexdump -C

333 custom_header_length = 8

334 custom_header_data = 0

335 # We don't need padding because we are already aligned

336 custom_header = struct.pack(

337 ">IIQ", custom_header_type, custom_header_length, custom_header_data

338 )

339

340 with open(self.filename, "rb+") as qcow2_file:

341 if self.header["version"] == 3:

342 qcow2_file.seek(100) # 100 is the offset of header_length

343 header_length = int.from_bytes(qcow2_file.read(4), "big")

344

345 # After the image header we found Header extension. So we need to find the end of

346 # the header extension area and add our custom header.

347 qcow2_file.seek(header_length)

348

349 custom_data_offset = 0

350

351 while True:

352 ext_type = int.from_bytes(qcow2_file.read(4), "big")

353 ext_len = int.from_bytes(qcow2_file.read(4), "big")

354

355 if ext_type == custom_header_type:

356 # A custom header is already there

357 custom_data_offset = qcow2_file.tell()

358 break

359

360 if ext_type == 0x00000000:

361 # End mark found. If we found the end mark it means that we didn't find

362 # the custom header. So we need to add it.

363 custom_data_offset = qcow2_file.tell()

364

365 # We will overwrite the end marker so rewind a little bit to

366 # write the new type extension and the new length. But if there is

367 # a backing file we need to move it to make some space.

368 if self.header["backing_file_offset"]:

369 # Keep current position

370 saved_pos = qcow2_file.tell()

371

372 bf_offset = self.header["backing_file_offset"]

373 bf_size = self.header["backing_file_size"]

374 bf_new_offset = bf_offset + len(custom_header)

375 self._move_backing_file(

376 qcow2_file, bf_offset, bf_new_offset, bf_size

377 )

378

379 # Update the header to match the new backing file offset

380 self.header["backing_file_offset"] = bf_new_offset

381 qcow2_file.seek(self.QCOW2_BACKING_FILE_OFFSET)

382 qcow2_file.write(struct.pack(">Q", bf_new_offset))

383

384 # Restore saved position

385 qcow2_file.seek(saved_pos)

386

387 qcow2_file.seek(-8, 1)

388 qcow2_file.write(custom_header)

389 break

390

391 # Round up the header extension size to the next multiple of 8

392 ext_len = (ext_len + 7) & 0xFFFFFFF8

393 qcow2_file.seek(ext_len, 1)

394

395 return custom_data_offset

396

397 def _set_l1_zero(self):

398 zero = int(0).to_bytes(1, "little")

399 nb_of_entries_per_cluster = QCOW2_DEFAULT_CLUSTER_SIZE/8

400 return list(zero * int(nb_of_entries_per_cluster/8))

401

402 def _set_l2_zero(self, b, i):

403 return b & ~(1 << i)

404

405 def _set_l2_one(self, b, i):

406 return b | (1 << i)

407

408 def _create_bitmap(self) -> bytes:

409 idx: int = 0

410 bitmap = list()

411 b = 0

412 for l1_entry in self.l1:

413 if not self._is_l1_allocated(l1_entry):

414 bitmap.extend(self._set_l1_zero())

415 continue

416

417 l2_table = self.l1_to_l2[l1_entry] #L2 is cluster_size/8 entries of cluster_size page

418 for l2_entry in l2_table:

419 if self._is_l2_allocated(l2_entry):

420 b = self._set_l2_one(b, idx)

421 else:

422 b = self._set_l2_zero(b, idx)

423 idx += 1

424 if idx == 8:

425 bitmap.append(b)

426 b = 0

427 idx = 0

428 return struct.pack("B"*len(bitmap), *bitmap)

429

430 # ----

431 # Implementation of CowUtil

432 # ----

433

434 @override

435 def getMinImageSize(self) -> int:

436 return MIN_QCOW_SIZE

437

438 @override

439 def getMaxImageSize(self) -> int:

440 return MAX_QCOW_SIZE

441

442 @override

443 def getBlockSize(self, path: str) -> int:

444 self._read_qcow2(path)

445 return 1 << self.header["cluster_bits"]

446

447 @override

448 def getFooterSize(self) -> int:

449 return 0

450

451 @override

452 def getDefaultPreallocationSizeVirt(self) -> int:

453 """vhdutil answer max size (2TiB) here but we don't want to allocate for max size in QCOW2, it would make small LV a lot bigger."""

454 return MIN_QCOW_SIZE

455

456 @override

457 def getMaxChainLength(self) -> int:

458 return MAX_QCOW_CHAIN_LENGTH

459

460 @override

461 def calcOverheadEmpty(self, virtual_size: int, block_size: Optional[int] = None) -> int:

462 if block_size:

463 cluster_size = block_size

464 else:

465 cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE

466 cmd = [QEMU_IMG, "measure", "-O", "qcow2", "--output", "json", "-o", f"cluster_size={cluster_size}", "--size", f"{virtual_size}"]

467 output = json.loads(self._ioretry(cmd))

468 return int(output["required"])

469

470 @override

471 def calcOverheadBitmap(self, virtual_size: int) -> int:

472 return 0 #TODO: What do we send back?

473

474 @override

475 def getInfo(

476 self,

477 path: str,

478 extractUuidFunction: Callable[[str], str],

479 includeParent: bool = True,

480 resolveParent: bool = True,

481 useBackupFooter: bool = False

482 ) -> CowImageInfo:

483 #TODO: handle resolveParent

484 self._read_qcow2(path)

485 uuid = extractUuidFunction(path)

486 cowinfo = CowImageInfo(uuid)

487 cowinfo.path = path

488 cowinfo.sizeVirt = self.header["virtual_disk_size"]

489 cowinfo.sizePhys = self.getSizePhys(path)

490 cowinfo.hidden = self.getHidden(path)

491 cowinfo.sizeAllocated = self.getAllocatedSize(path)

492 if includeParent:

493 parent_path = self.header["parent"]

494 if parent_path != "":

495 cowinfo.parentPath = parent_path

496 cowinfo.parentUuid = extractUuidFunction(parent_path)

497

498 return cowinfo

499

500 @override

501 def getInfoFromLVM(

502 self, lvName: str, extractUuidFunction: Callable[[str], str], vgName: str

503 ) -> Optional[CowImageInfo]:

504 lvcache = LVMCache(vgName)

505 return self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName)

506

507 def _getInfoLV(

508 self, lvcache: LVMCache, extractUuidFunction: Callable[[str], str], vgName: str, lvName: str

509 ) -> Optional[CowImageInfo]:

510 lvPath = "/dev/{}/{}".format(vgName, lvName)

511 lvcache.refresh()

512 if lvName not in lvcache.lvs:

513 util.SMlog("{} does not exist anymore".format(lvName))

514 return None

515

516 vdiUuid = extractUuidFunction(lvPath)

517 srUuid = vgName.replace(VG_PREFIX, "")

518

519 ns = NS_PREFIX_LVM + srUuid

520 lvcache.activate(ns, vdiUuid, lvName, False)

521 try:

522 cowinfo = self.getInfo(lvPath, extractUuidFunction)

523 finally:

524 lvcache.deactivate(ns, vdiUuid, lvName, False)

525 return cowinfo

526

527 @override

528 def getAllInfoFromVG(

529 self,

530 pattern: str,

531 extractUuidFunction: Callable[[str], str],

532 vgName: Optional[str] = None,

533 parents: bool = False,

534 exitOnError: bool = False

535 ) -> Dict[str, CowImageInfo]:

536 result: Dict[str, CowImageInfo] = dict()

537 #TODO: handle exitOnError

538 if vgName: 538 ↛ 539line 538 didn't jump to line 539, because the condition on line 538 was never true

539 reg = re.compile(pattern)

540 lvcache = LVMCache(vgName)

541 lvcache.refresh()

542 # We get size in lvcache.lvs[lvName].size (in bytes)

543 # We could read the header from the PV directly

544 lvList = list(lvcache.lvs.keys())

545 for lvName in lvList:

546 # lvinfo = lvcache.lvs[lvName]

547 if reg.match(lvName):

548 cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, lvName)

549 if cowinfo is None: #We get None if the LV stopped existing in the meanwhile

550 continue

551 cowinfo.path = lvName # Function CowUtil.getParentChain expect lvName here, otherwise blktap.{_activate,_deactivate} crashes

552 result[cowinfo.uuid] = cowinfo

553 if parents:

554 parentUuid = cowinfo.parentUuid

555 parentPath = cowinfo.parentPath

556 while parentUuid != "":

557 parentLvName = parentPath.split("/")[-1]

558 parent_cowinfo = self._getInfoLV(lvcache, extractUuidFunction, vgName, parentLvName)

559 if parent_cowinfo is None: #Parent disappeared while scanning

560 raise util.SMException("Parent of {} wasn't found during scan".format(lvName))

561 parentUuid = parent_cowinfo.parentUuid

562 parentPath = parent_cowinfo.parentPath

563 parent_cowinfo.path = parentLvName #Same reason as above, some users expect LvName here instead of path

564 result[parent_cowinfo.uuid] = parent_cowinfo

565

566 return result

567 else:

568 pattern_p: Path = Path(pattern)

569 list_qcow = list(pattern_p.parent.glob(pattern_p.name))

570 for qcow in list_qcow: 570 ↛ 571line 570 didn't jump to line 571, because the loop on line 570 never started

571 qcow_str = str(qcow)

572 info = self.getInfo(qcow_str, extractUuidFunction)

573 result[info.uuid] = info

574 return result

575

576 @override

577 def getParent(self, path: str, extractUuidFunction: Callable[[str], str]) -> Optional[str]:

578 parent = self.getParentNoCheck(path)

579 if parent:

580 return extractUuidFunction(parent)

581 return None

582

583 @override

584 def getParentNoCheck(self, path: str) -> Optional[str]:

585 self._read_qcow2(path)

586 parent_path = self.header["parent"]

587 if parent_path == "":

588 return None

589 return parent_path

590

591 @override

592 def hasParent(self, path: str) -> bool:

593 if self.getParentNoCheck(path):

594 return True

595 return False

596

597 @override

598 def setParent(self, path: str, parentPath: str, parentRaw: bool) -> None:

599 pid_openers = util.get_openers_pid(path)

600 if pid_openers:

601 util.SMlog("Rebasing while process {} has the VDI opened".format(pid_openers))

602

603 parentType = QCOW2_TYPE

604 if parentRaw:

605 parentType = RAW_TYPE

606 cmd = [QEMU_IMG, "rebase", "-u", "-f", QCOW2_TYPE, "-F", parentType, "-b", parentPath, path]

607 self._ioretry(cmd)

608

609 @override

610 def getHidden(self, path: str) -> bool:

611 """Get hidden property according to the value b

612

613 Args:

614

615 Returns:

616 True if hidden is set, False otherwise

617 """

618 self._read_qcow2(path)

619 custom_data_offset = self._add_or_find_custom_header()

620 if custom_data_offset == 0:

621 raise util.SMException("Custom data offset not found... should not reach this")

622

623 with open(path, "rb") as qcow2_file:

624 qcow2_file.seek(custom_data_offset)

625 hidden = qcow2_file.read(1)

626 if hidden == b"\x00":

627 return False

628 return True

629

630 @override

631 def setHidden(self, path: str, hidden: bool = True) -> None:

632 """Set hidden property according to the value b

633

634 Args:

635 bool: True if you want to set the property. False otherwise

636

637 Returns:

638 nothing. If the custom headers is not found it is created so the

639 qcow file can be modified.

640 """

641 self._read_qcow2(path)

642 custom_data_offset = self._add_or_find_custom_header()

643 if custom_data_offset == 0:

644 raise util.SMException("Custom data offset not found... should not reach this")

645

646 with open(self.filename, "rb+") as qcow2_file:

647 qcow2_file.seek(custom_data_offset)

648 if hidden:

649 qcow2_file.write(b"\x01")

650 else:

651 qcow2_file.write(b"\x00")

652

653 @override

654 def getSizeVirt(self, path: str) -> int:

655 self._read_qcow2(path)

656 return self.header['virtual_disk_size']

657

658 @override

659 def setSizeVirt(self, path: str, size: int, jFile: str) -> None:

660 """

661 size: byte

662 jFile: a journal file used for resizing with VHD, not useful for QCOW2

663 """

664 cmd = [QEMU_IMG, "resize", path, str(size)]

665 self._ioretry(cmd)

666

667 @override

668 def setSizeVirtFast(self, path: str, size: int) -> None:

669 self.setSizeVirt(path, size, "")

670

671 @override

672 def getMaxResizeSize(self, path: str) -> int:

673 return 0

674

675 @override

676 def getSizePhys(self, path: str) -> int:

677 size = os.stat(path).st_size

678 if size == 0:

679 size = int(self._ioretry(["blockdev", "--getsize64", path]))

680 return size

681

682 @override

683 def setSizePhys(self, path: str, size: int, debug: bool = True) -> None:

684 pass #TODO: Doesn't exist for QCow2, do we need to use it?

685

686 @override

687 def getAllocatedSize(self, path: str) -> int:

688 cmd = [QCOW2_HELPER, "allocated", path]

689 return int(self._ioretry(cmd))

690

691 @override

692 def getResizeJournalSize(self) -> int:

693 return 0

694

695 @override

696 def killData(self, path: str) -> None:

697 """Remove all data and reset L1/L2 table.

698

699 Args:

700 self: The QcowInfo object.

701

702 Returns:

703 nothing.

704 """

705 self._read_qcow2(path, read_clusters=True)

706 # We need to reset L1 entries and then just truncate the file right

707 # after L1 entries

708 with open(self.filename, "r+b") as file:

709 l1_table_offset = self.header["l1_table_offset"]

710 file.seek(l1_table_offset)

711

712 l1_table_size = (

713 self.header["l1_size"] * 8

714 ) # size in bytes, each entry is 8 bytes

715 file.write(b"\x00" * l1_table_size)

716 file.truncate(l1_table_offset + l1_table_size)

717

718 @override

719 def getDepth(self, path: str) -> int:

720 cmd = [QEMU_IMG, "info", "--backing-chain", "--output=json", path]

721 ret = str(self._ioretry(cmd))

722 depth = len(re.findall("\"backing-filename\"", ret))+1

723 #chain depth is beginning at one for VHD, meaning a VHD without parent has depth = 1

724 return depth

725

726 @override

727 def getBlockBitmap(self, path: str) -> bytes:

728 cmd = [QCOW2_HELPER, "bitmap", path]

729 text = cast(bytes, self._ioretry(cmd, text=False))

730 return zlib.compress(text)

731

732 def _getTapdisk(self, path: str) -> Tuple[int, int]:

733 """

734 Return a tuple of (PID, Minor) for the given path

735 """

736 pid_openers = util.get_openers_pid(path)

737 if pid_openers:

738 if len(pid_openers) > 1:

739 raise xs_errors.XenError("Multiple openers for {}".format(path)) # TODO: There might be multiple PID? Yes, we can have the chain enabled for multiple leaf (i.e. after a clone), taken into account in cleanup.py

740 pid = pid_openers[0]

741 tapdiskList = TapCtl.list(pid=pid)

742 if len(tapdiskList) > 1: #TODO: There might more than one minor for this blktap?

743 raise xs_errors.XenError("TapdiskAlreadyRunning", "There is multiple minor for this tapdisk process")

744 minor = tapdiskList[0]["minor"]

745 return (pid, minor)

746 raise xs_errors.XenError("TapdiskFailed", "No tapdisk process found for {}".format(path))

747

748 @override

749 def coalesceOnline(self, path: str) -> int:

750 pid, minor = self._getTapdisk(path)

751 logger = util.LoggerCounter(10)

752

753 try:

754 TapCtl.commit(pid, minor, QCOW2_TYPE, path)

755 # We need to wait for query to return concluded

756 # We are technically ininterruptible since being interrupted will only stop checking if the job is done.

757 # We need to call `tap-ctl cancel` if we are interrupted, it is done in cleanup.py code.

758 except TapCtl.CommandFailure:

759 util.SMlog(f"Commit command failed on tapdisk instance {pid} (m: {minor}). Raising...")

760 raise

761

762 try:

763 status, nb, _ = TapCtl.query(pid, minor)

764 if status == "undefined":

765 util.SMlog("Tapdisk {} (m: {}) coalesce status undefined for {}".format(pid, minor, path))

766 return 0

767

768 while status != "concluded":

769 time.sleep(1)

770 status, nb, _ = TapCtl.query(pid, minor, quiet=True)

771 logger.log("Got status {} for tapdisk {} (m: {})".format(status, pid, minor))

772 return nb

773 except TapCtl.CommandFailure:

774 util.SMlog(f"Query command failed on tapdisk instance {pid} (m: {minor}). Raising...")

775 raise

776

777 @override

778 def cancelCoalesceOnline(self, path: str) -> None:

779 pid, minor = self._getTapdisk(path)

780

781 try:

782 TapCtl.cancel_commit(pid, minor)

783 except TapCtl.CommandFailure:

784 util.SMlog("Cancel command failed on tapdisk instance {}. Raising...".format(pid))

785 raise

786

787 @override

788 def coalesce(self, path: str) -> int:

789 # -d on commit make it not empty the original image since we don't intend to keep it

790 cmd = [QEMU_IMG, "commit", "-f", QCOW2_TYPE, path, "-d"]

791 ret = cast(str, self._ioretry(cmd)) # Allows to parse for byte coalesced, our qemu-img is supposed to be patched to output it.

792 lines = ret.splitlines()

793 if re.match("Image committed.", lines[-1]):

794 res_line = lines[-2]

795 else:

796 res_line = lines[-1]

797

798 results = re.match(r"\((\d+)/(\d+)\)", res_line)

799 if results:

800 committed_bytes = int(results.group(1))

801 return committed_bytes

802 raise xs_errors.XenError("TapdiskFailed", "Couldn't get commited size from qemu-img commit call") # TODO: We might not want to raise in this case, it would break if the qemu-img called isn't modified to print the coalesce result even if it succeeded in coalesceing

803

804 @override

805 def create(self, path: str, size: int, static: bool, msize: int = 0, block_size: Optional[int] = None) -> None:

806 cmd = [QEMU_IMG, "create", "-f", QCOW2_TYPE, path, str(size)]

807 if static:

808 cmd.extend(["-o", "preallocation=full"])

809 if block_size:

810 cmd.extend(["-o", f"cluster_size={str(block_size)}"])

811 self._ioretry(cmd)

812 self.setHidden(path, False) #We add hidden header at creation

813

814 @override

815 def snapshot(

816 self,

817 path: str,

818 parent: str,

819 parentRaw: bool,

820 msize: int = 0,

821 checkEmpty: bool = True,

822 is_mirror_image: bool = False

823 ) -> None:

824 # TODO: msize, it's use to preallocate metadata, could we honor this too?

825 # TODO: checkEmpty? If it is False, then the parent could be empty and should still be used for snapshot

826 # But if True, if the parent is empty, we do what? vhd would just use the parent of parent as base, should we emulate this behavior?

827

828 cmd = [QEMU_IMG, "create"]

829

830 if parentRaw:

831 parent_type = RAW_TYPE

832 cluster_size = QCOW2_DEFAULT_CLUSTER_SIZE

833 else:

834 parent_type = QCOW2_TYPE

835 cluster_size = self.getBlockSize(parent)

836 args = ["-f", QCOW2_TYPE, "-F", parent_type, "-b", parent]

837

838 if is_mirror_image:

839 # is_mirror_image override the cluster size to ensure that we have a write of 512b to avoid having to read the parent during a migration.

840 # This is needed because the blkif blocksize is only 512b, as such it will try to only write blocks smaller than the cluster size.

841 # To write a smaller block, we would need to read the parent image cluster then change the 512b block.

842 # The parent being empty during the mirroring phase, reading from it would read zeros and corrupt the cluster.

843 # It also enable extended_l2 for this purpose, this is only done in the snapshot used for the mirror, this configuration will be lost when coalesced in its parent.

844 # Ensuring we go back to a better cluster_size for performance reasons.

845 # This limit our images max size to 64TiB.

846 cluster_size = 16 * 1024 # 16KiB

847 args.extend(["-o", "extended_l2=on"])

848

849 args.extend(["-o", f"cluster_size={cluster_size}"])

850 cmd.extend(args)

851 cmd.append(path)

852

853 self._ioretry(cmd)

854 self.setHidden(path, False) #We add hidden header at creation

855