#!/usr/bin/python
########################################################################
# Copyright (c) 2010-2021 VMware, Inc.                                 #
# All rights reserved.                                                 #
########################################################################

import gzip
import logging
import os
import platform
import shutil
from socket import gethostname
import tarfile

from vmware.runcommand import runcommand, RunCommandError

from .. import Database
from .. import Errors
from .. import ImageProfile
from .. import Vib
from . import Installer
from ..PayloadTar import PayloadTar
from ..Utils import BootCfg
from ..Utils import HashedStream
from ..Utils import HostInfo
from ..Utils import LockFile
from ..Utils import Ramdisk
from ..Utils.Misc import byteToStr

from .. import CHECKSUM_EXEMPT_PAYLOADS, MIB
# Feature switch from esximage/__init__.py
from .. import SYSTEM_STORAGE_ENABLED

from systemStorage.upgradeUtils import getTempBootbank, TempBbMarker

BACKUP_SH = '/sbin/backup.sh'
TMP_DIR = '/tmp'

class InvalidBootbankError(Exception):
   pass

log = logging.getLogger(os.path.basename(__file__))

class BootBankBootCfg(BootCfg.BootCfg):
   """BootBankBootCfg encapsulates the boot.cfg file of a bootbank.
      This class extends the common util BootCfg class to enclose path,
      provide bootstate update method, and set appropriate default attributes.

      See Utils.BootCfg for bootstates and attributes.
   """
   INSTALLER_BOOTSTATES = (BootCfg.BootCfg.BOOTSTATE_EMPTY,
                           BootCfg.BootCfg.BOOTSTATE_STAGED,
                           BootCfg.BootCfg.BOOTSTATE_UPDATED)

   def __init__(self, path):
      """
         Parameters:
            * path - File path to boot.cfg.
      """
      # Not parsing on init
      BootCfg.BootCfg.__init__(self)
      self._path = path
      self.clear()

   path = property(lambda self: self._path)

   def clear(self):
      """Set default values appropriate for installer.
      """
      super(BootBankBootCfg, self).clear()
      self.bootstate = self.BOOTSTATE_EMPTY
      self.title = 'Loading VMware ESXi'

   def parse(self):
      """Parse bootcfg with expected attributes checked.
      """
      expectedKeys = ['updated', 'bootstate', 'kernel', 'kernelopt', 'modules',
                      'build']
      super(BootBankBootCfg, self).parse(self._path, expectedKeys=expectedKeys)
      self.validate()

   def write(self):
      """Save bootcfg in bootbank.
      """
      super(BootBankBootCfg, self).write(self._path)

   def updatePrefix(self, prefix):
      """Change the prefix option in boot.cfg.
      """
      log.info("%s: setting prefix to %s" % (self._path, prefix))
      if not prefix.endswith('/'):
         prefix += '/'
      self.prefix = prefix

   def updateState(self, state, liveimage=False):
      """Change bootstate value to BOOTSTATE_EMPTY, BOOTSTATE_STAGED or
         BOOTSTATE_UPDATED, which are managed by Installer.

            Parameters:
               * state - one of BOOTSTATE_EMPTY, BOOTSTATE_STAGED,
                         BOOTSTATE_UPDATED, BOOTSTATE_SUCCESS (for live image)
            Exceptions:
               * ValueError - If the given state is not managed by Installer.

      """
      if state in self.INSTALLER_BOOTSTATES:
         log.info("%s: bootstate changed from %s to %s"
            % (self._path, str(self.bootstate), str(state)))
         self.bootstate = state
      else:
         if liveimage and state == self.BOOTSTATE_SUCCESS:
            log.info("%s: bootstate changed from %s to %s"
               % (self._path, str(self.bootstate), str(state)))
            self.bootstate = state
         else:
            msg = 'Can not change state to %s, only %s are allowed' % (
                  state, self.INSTALLER_BOOTSTATES)
            raise ValueError(msg)

   def copyAttrs(self, other, attrs):
      """Copy fields from another BootCfg object.
      """
      for attr in attrs:
         assert hasattr(self, attr) and hasattr(other, attr)
         setattr(self, attr, getattr(other, attr))

class BootBank(object):
   """Encapsulates attributes of a single bootbank.

      Class Variables:
         * DB_FILE  - File name of bootbank package database.
         * BOOT_CFG - File name of the config file for bootloader
         * BACKUP_STATE_TGZ - File name of the backed up pre-upgrade state.tgz

      Attributes:
         * db      - An instance of Database.TarDatabase representing the
                     bootbank package database.
         * updated - "updated" serial number from boot.cfg.  If the bootbank
                     contains an invalid image or boot.cfg, this should be -1.
         * path    - The full file path for the mounted bootbank.
   """
   DB_FILE = 'imgdb.tgz'
   BOOT_CFG = 'boot.cfg'
   BACKUP_STATE_TGZ = 'preupgrade-state.tgz'
   BASEMISC_PAYLOADTAR = 'basemisc.tgz'

   def __init__(self, bootbankpath):
      """
         Exceptions:
            InvalidBootbankError - thrown if a valid bootbank could not be found
                              at the bootbankpath
      """
      if not os.path.isdir(bootbankpath):
         raise InvalidBootbankError("%s is not a directory!" % (bootbankpath))

      self._path = bootbankpath
      # Folder to stage misc esx-base payloads; they will be tar'ed when staging
      # finishes.
      self.baseMiscPayloadsDir = os.path.join(bootbankpath, 'baseMiscPayloads')
      dbpath = os.path.join(bootbankpath, self.DB_FILE)
      # Load existing database or create a new in-ram database, do not commit
      # to the disk/folder.
      self.db = Database.TarDatabase(dbpath, dbcreate=False)
      bootcfgpath = os.path.join(bootbankpath, self.BOOT_CFG)
      self.bootcfg = BootBankBootCfg(bootcfgpath)

   def __str__(self):
      return "<BootBank:%s>" % (self.path)

   @property
   def path(self):
      return self._path

   def _getupdated(self):
      return self.bootcfg.updated

   def _setupdated(self, updated):
      self.bootcfg.updated = updated

   updated = property(_getupdated, _setupdated)

   @property
   def bootstate(self):
      return self.bootcfg.bootstate

   def _changePath(self, path):
      """Change this bootbank's path, without reloading.
         In a legacy upgrade, we need to swap the altbootbank and the temporary
         bootbank, without the need to reload the database or boot.cfg.
         Otherwise, this method should not be used since it may lead to
         inconsistencies.
      """
      self._path = path
      self.db._dbpath = os.path.join(path, self.DB_FILE)
      self.bootcfg._path = os.path.join(path, self.BOOT_CFG)

   def _UpdateVib(self, newvib):
      """Update missing properties of vib metadata

         Parameters:
            * newvib   - The new vib to use as source
         Returns:
            None if the update succeeds, Exception otherwise
         Exceptions:
            VibFormatError
      """
      try:
         self.db.vibs[newvib.id].SetSignature(newvib.GetSignature())
         self.db.vibs[newvib.id].SetOrigDescriptor(newvib.GetOrigDescriptor())
      except Exception as e:
         msg = ("Failed to set signature or original descriptor of VIB "
                "%s: %s" % (newvib.id, e))
         raise Errors.VibFormatError(newvib.id, msg)

   def Load(self, raiseerror=True, createprofile=False):
      """Load database and boot.cfg from the bootbank
         Exceptions:
            * InvalidBootbankError - If raiseerror is True and there is an error
                                     loading boot.cfg or database.
      """
      try:
         self.bootcfg.parse()
      except (IOError, BootCfg.BootCfgError) as e:
         self.bootcfg.clear()
         msg = 'Error in loading boot.cfg from bootbank %s: %s' % \
               (self._path, e)
         if raiseerror:
            raise InvalidBootbankError(msg)
         else:
            log.warn('Ignoring error when loading bootbank: %s' % msg)

      if self.bootcfg.bootstate in (BootBankBootCfg.BOOTSTATE_UPDATED,
                                    BootBankBootCfg.BOOTSTATE_STAGED,
                                    BootBankBootCfg.BOOTSTATE_SUCCESS):
         dbLockFile = '/var/run/%simgdb.pid' % self._path.replace('/', '')
         dbLock = None
         try:
            dbLock = LockFile.acquireLock(dbLockFile)
            self.db.Load()
         except (LockFile.LockFileError, Errors.DatabaseFormatError,
                 Errors.DatabaseIOError) as e:
            self.db.Clear()
            self.bootcfg.updateState(BootBankBootCfg.BOOTSTATE_EMPTY)
            if isinstance(e, LockFile.LockFileError):
               msg = 'Unable to obtain a lock for database I/O: %s' % str(e)
               exClass = Errors.LockingError
            else:
               msg = 'Error in loading database for bootbank %s: %s' % \
                     (self._path, e)
               exClass = InvalidBootbankError
            if raiseerror:
               raise exClass(msg)
            else:
               log.warn('Ignoring error when loading bootbank: %s' % msg)
         finally:
            if dbLock:
               dbLock.Unlock()

         if self.db.profile is None and createprofile:
            log.debug('Creating an empty ImageProfile for bootbank %s' % \
                      self._path)
            self.db.Clear()
            pname = 'Empty ImageProfile'
            creator = gethostname()
            p = ImageProfile.ImageProfile(pname, creator)
            self.db.profiles.AddProfile(p)

         # Populate profile.vibs and profile.bulletins to keep everything in the
         # image profile for the transaction.
         if self.db.profile is not None:
            self.db.profile.PopulateWithDatabase(self.db)

   def Reset(self):
      """Reset bootbank DB to empty and flag the bootbank as empty.
         The caller should call Save to persist the change, or skip Reset
         to call Clear directly to empty the bootbank.
      """
      self.bootcfg.updateState(self.bootcfg.BOOTSTATE_EMPTY)
      self.db.Clear()

   def Save(self):
      """Flush bootcfg and database to persistent storage.
      """
      dbLockFile = '/var/run/%simgdb.pid' % self._path.replace('/', '')
      dbLock = None
      try:
         dbLock = LockFile.acquireLock(dbLockFile)
         self.db.Save(savesig=True)
         self.bootcfg.write()
      except LockFile.LockFileError as e:
         msg = 'Unable to obtain a lock for database I/O: %s' % str(e)
         raise Errors.LockingError(msg)
      except (IOError, BootCfg.BootCfgError) as e:
         msg = 'Failed to save bootcfg to file %s: %s' % (self.bootcfg.path, e)
         raise Errors.InstallationError(e, None, msg)
      finally:
         if dbLock:
            dbLock.Unlock()

   def Clear(self, purge=False):
      """Clear the contents of the bootbank.
         Parameter:
            * purge - when set, remove all files in the bootbank rather
                      than marking the bootbank as empty in the database
                      and boot.cfg. Default is False.
      """
      try:
         self.Reset()
         self.Save()
      except Exception as e:
         msg = 'Error in clearing bootcfg or DB for bootbank %s: %s' % (
               self._path, e)
         raise Errors.InstallationError(e, None, msg)

      excludes = ([BootBank.BOOT_CFG, BootBank.DB_FILE,
                  BootBank.BACKUP_STATE_TGZ, TempBbMarker.MARKER_FILE] if not
                  purge else None)
      lockfile = '/tmp/%s.lck' % self._path.replace("/", "_")
      flock = LockFile.acquireLock(lockfile)
      try:
         self._clearFiles(excludes)
      except EnvironmentError as e:
         msg = 'Failed to clear bootbank content %s: %s' % (self._path, e)
         raise Errors.InstallationError(e, None, msg)
      finally:
         flock.Unlock()

   def FileCopy(self, srcbank, purge=True):
      """Copy all srcbank files to this bootbank.
         Parameter:
            purge - when set, remove all files in the bootbank first
                    before copying. Default is True.
      """
      lockfile = '/tmp/%s.lck' % self._path.replace("/", "_")
      flock = LockFile.acquireLock(lockfile)
      try:
         if purge:
            # Remove all files of this bootbank
            self._clearFiles()
         # Copy all files from srcbank to this bootbank
         log.info('Copying bootbank %s to %s' % (srcbank.path, self.path))
         for item in os.listdir(srcbank.path):
            src = os.path.join(srcbank.path, item)
            dest = os.path.join(self.path, item)
            try:
               shutil.copy2(src, dest)
            except IsADirectoryError:
               # Usually there should be no directory in bootbanks, handle it
               # anyway.
               shutil.copytree(src, dest)
      except EnvironmentError as e:
         msg = 'Failed to copy files from %s to %s: %s' % (
                 srcbank.path, self._path, e)
         raise Errors.InstallationError(e, None, msg)
      finally:
         flock.Unlock()

   def Verify(self):
      """ Verify whether the content of bootbank is consistent with imgprofile
      in DB.
      """
      #TODO: verify payloads according to DB

      #Verify vib payload

   def Stage(self, srcbank, keeps, imgprofile):
      """Stage the initial content of this bootbank. If srcbank is different
         from this bootbank, for VIBs in keeps list, payloads will be
         copied to this bootbank and renamed according to imgprofile. If
         srcbank is this bootbank, VIBs not in keeps list will be removed from
         the bootbank and the VIBs in keeps list will be renamed according to target
         image profile.

         Parameters:
            * srcbank    - BootBank instance, either 'bootbank' or
                           'altbootbank', the content is used to build target
                           image profile as source input.
            * keeps      - List of id of VIBs which need to be kept from srcbank
                           to construct target imageprofile.
            * imgprofile - Target image profile for this bootbank.

         Exceptions:
            * InstallationError - If there is an error to stage the imgprofile
                                  to this bootbank
      """
      if srcbank.path == self.path:
         self._inplaceStage(keeps, imgprofile)
      else:
         self._copyStage(srcbank, keeps, imgprofile)

      self._stageBaseMiscPayloads(srcbank, keeps)

      # Update database with the staged image profile
      self.db.PopulateWith(imgProfile=imgprofile)

   def _clearFiles(self, exemptFiles=None):
      '''Remove all files and directories in the bootbank, except when a
         file is exempted (e.g. boot.cfg). Exemption is only applicable to
         files in the root of the bootbank.
      '''
      exemptFiles = exemptFiles or []
      log.info('Clearing bootbank %s with files %s exempted in root'
               % (self.path, exemptFiles))
      for root, dirs, files in os.walk(self.path):
         for dn in dirs:
            # Usually we should not be seeing folders except temp folder
            # created by backup.sh, clean them and ignore errors (e.g.
            # FileNotFound exception caused by a race).
            shutil.rmtree(os.path.join(root, dn), ignore_errors=True)
         for name in files:
            if name not in exemptFiles:
               try:
                  os.unlink(os.path.join(root, name))
               except FileNotFoundError:
                  # Handle removal race
                  pass

   def _stageBaseMiscPayloads(self, srcBank, keeps):
      """Extract misc esx-base payloads to be carried over to the stage
         directory.
      """
      baseMiscPayloads = []
      for vibId in keeps:
         for pl in srcBank.db.vibs[vibId].payloads:
            if (pl.payloadtype in
                BootBankInstaller.SUPPORTED_NONGZIP_PAYLOADS):
               baseMiscPayloads.append((pl.name, pl.payloadtype))

      srcTar = os.path.join(srcBank.path, self.BASEMISC_PAYLOADTAR)
      if not os.path.exists(srcTar):
         # Updating an ESXi without basemisc.tgz, could be a prior version
         # or an image created by legacy tools.
         log.info('Skip staging misc payloads, %s does not exist', srcTar)
         return

      try:
         os.makedirs(self.baseMiscPayloadsDir)
      except EnvironmentError as e:
         msg = ('Failed to create directory %s to stage misc payloads: %s'
                % (self.baseMiscPayloadsDir, str(e)))
         raise Errors.InstallationError(None, keeps, msg)

      # Extract misc payloads to keep into staging dir, sorted by their types.
      with tarfile.open(srcTar, 'r') as tar:
         for plName, plType in baseMiscPayloads:
            extractDir = os.path.join(self.baseMiscPayloadsDir,
                                      plType)
            try:
               if not os.path.exists(extractDir):
                  os.makedirs(extractDir)
               memberName = os.path.join(PayloadTar.PAYLOADTAR_PREFIX, plType,
                                         plName)
               m = tar.getmember(memberName)
               tar.extract(m, path=extractDir)
            except (tarfile.TarError, EnvironmentError) as e:
               msg = ('Failed to extract payload file %s from %s: %s'
                      % (plName, srcTar, str(e)))
               raise Errors.InstallationError(None, keeps, msg)

   def _inplaceStage(self, keeps, imgprofile):
      #NOTE: This only uses current bootbank VIB content, we might do better in some
      # case to use VIBs from /bootbank.
      self.bootcfg.updateState(self.bootcfg.BOOTSTATE_EMPTY)
      try:
         self.bootcfg.write()
      except (IOError, BootCfg.BootCfgError) as e:
         msg = 'Failed to save bootcfg to file %s: %s' % (self.bootcfg.path, e)
         raise Errors.InstallationError(e, None, msg)
      kept = set()

      for vibid in self.db.profile.vibstates:
         # In place renaming to a temporary name
         vibstate = self.db.profile.vibstates[vibid]
         if vibid in keeps:
            deststate = imgprofile.vibstates[vibid]
            for pl in self.db.vibs[vibid].payloads:
               if (pl.payloadtype not in
                   BootBankInstaller.SUPPORTED_GZIP_PAYLOADS):
                  continue

               plname = pl.name
               srcname = None
               if plname in vibstate.payloads:
                  srcname = vibstate.payloads[plname]

               if srcname is not None:
                  destname = deststate.payloads[plname] + '.stg'
                  dst = os.path.join(self._path, destname)
                  src = os.path.join(self._path, srcname)

                  try:
                     os.rename(src, dst)
                  except OSError as e:
                     msg = 'Failed to '
                     raise Errors.InstallationError(e, imgprofile.vibIDs, msg)
               else:
                  msg = ('Can not locate source for payload %s of VIB %s '
                         ' from bootbank %s' % (plname, vibstate.id, self._path))
                  raise Errors.InstallationError(None, imgprofile.vibIDs, msg)

               vibstate.payloads[plname] = destname
            kept.add(vibstate.id)
         else:
            # Remove unwanted vibs
            log.info('Removing VIB %s from %s' % (vibid, self._path))
            for plname in vibstate.payloads:
               localfile = os.path.join(self.path, vibstate.payloads[plname])
               if not os.path.isfile(localfile):
                  continue

               log.info('Removing payload %s at %s' % (plname, localfile))
               try:
                  os.unlink(localfile)
               except EnvironmentError as e:
                  msg = ('Can not remove payload %s in VIB %s: %s' %
                        (localfile, vibid, e))
                  raise Errors.InstallationError(e, None, msg)
            del self.db.vibs[vibid]

      if kept != set(keeps):
         left = set(keeps) - kept
         msg = 'VIBs %s are not found in bootbank %s' % (left, self._path)
         raise Errors.InstallationError(None, imgprofile.vibIDs, msg)

      # re-establish local filename
      # In order to avoid name collision between original localname and
      # target localname, payload is renamed to a name space ends with
      # '.stg' and then move to the final name.
      for vibid in keeps:
         vibstate = self.db.profile.vibstates[vibid]
         for pl in self.db.vibs[vibid].payloads:
            if pl.payloadtype not in BootBankInstaller.SUPPORTED_PAYLOADS:
               continue

            plname = pl.name
            src = os.path.join(self.path, vibstate.payloads[plname])
            dst = src[:-4]
            if src.endswith('.stg'):
               try:
                  os.rename(src, dst)
               except OSError as e:
                  msg = ('Failed to rename temporary payload file %s '
                          'to final name %s: %s.' % (src, dst, e))
                  raise Errors.InstallationError(e, imgprofile.vibIDs, msg)
            else:
               msg = 'Unexpected temporary payload file name %s' % (src)
               raise Errors.InstallationError(None, imgprofile.vibIDs, msg)
            vibstate.payloads[plname] = vibstate.payloads[plname][:-4]

   def _copyStage(self, srcbank, keeps, imgprofile):
      # Preparation altbootbank based on srcbank
      # TODO: we can do better if there is a partial databse
      # Let's try the hard way first - just copy src from bootbank
      self.Clear()
      for vibid in keeps:
         if vibid not in srcbank.db.profile.vibIDs:
            msg = 'Can not find VIB %s in bootbank %s.' % (vibid,
                  srcbank.path)
            raise Errors.InstallationError(None, imgprofile.vibIDs, msg)

         log.info('Copying payloads of %s from %s to %s' % (vibid,
            srcbank.path, self._path))
         deststate = imgprofile.vibstates[vibid]
         vibstate = srcbank.db.profile.vibstates[vibid]
         for pl in srcbank.db.vibs[vibid].payloads:
            if pl.payloadtype not in BootBankInstaller.SUPPORTED_GZIP_PAYLOADS:
               continue

            plname = pl.name
            srcfile = None
            if plname in vibstate.payloads:
               srcfile = os.path.join(srcbank.path, vibstate.payloads[plname])

            if srcfile is not None:
               dstfile = os.path.join(self.path, deststate.payloads[plname])
               log.info('Copying %s to %s' % (srcfile, dstfile))
               try:
                  shutil.copy2(srcfile, dstfile)
               except EnvironmentError as e:
                  msg = 'Error in copying from %s to %s: %s' % (srcfile,
                        dstfile, e)
                  raise Errors.InstallationError(e, imgprofile.vibIDs, msg)
               #TODO: verify copy
               #log.debug('Verifying the file %s' % (dstfile))
            else:
               msg = ('Can not locate source for payload %s of VIB %s '
                      'from bootbank %s' % (plname, vibid, srcbank.path))
               raise Errors.InstallationError(None, imgprofile.vibIDs, msg)

class BootBankInstaller(Installer):
   """BootBankInstaller is the Installer class for building
      a ESXi VFAT bootbank image.

      Class Variables:
         * BOOTBANK_STATE_FRESH   - altbootbank is either empty or a previous boot
                                    image.
         * BOOTBANK_STATE_STAGED  - altbootbank is staged with new ImageProfile,
                                    and VIBs has been installed to the bootbank,
                                    but the host will still boot from current
                                    bootbank in next reboot.
         * BOOTBANK_STATE_UPDATED - altbootbank is upgraded. The host will boot
                                    from altbootbank in next reboot.
         * SUPPORTED_PAYLOADS     - tuple of payload types that will be handled
                                    by the installer.

      Attributes:
         * bootbank    - A BootBank instance representing /bootbank, the bank
                         from which the host booted
         * altbootbank - A BootBank instance representing /altbootbank, the
                         bank holding the future (or previous) boot image
         * stagebootbank - A BootBank instance representing STAGEBOOTBANK,
                           the ramdisk holding staged boot image.
         * bootbankstate - Indicate whether altbootbank is staged and / or
                           updated, combination of BOOTBANK_STATE_* bit.
   """
   installertype = "boot"
   priority = 10

   BOOTBANK = '/bootbank'
   ALTBOOTBANK = '/altbootbank'

   # GetContainerId() retrieves the simulator name to generate a unique ramdisk
   # name and path for each simulator environment. It returns an empty string
   # if not in the simulator environment. This is similar to
   # LiveImageInstaller's staging area.
   STAGEBOOTBANK_NAME = HostInfo.GetContainerId() + 'stagebootbank'
   STAGEBOOTBANK = os.path.join(os.path.sep, "tmp", STAGEBOOTBANK_NAME)

   # The staging area would match the bootbank in size, except during
   # an upgrade from 6.x, smaller 250MB bootbanks are present, where
   # a temporary bootbank will be used to transition to the new layout.
   # In that case, our stage limit would be 500MB.
   LEGACY_BOOTBANK_SIZE = 250
   MIN_STAGEBOOTBANK_SIZE = 500

   _BOOTBANK_SIZE = None
   _STAGEBOOTBANK_SIZE = None
   # Refer hostd-simulator-start2015._init_chroot_ramdisk().
   _SIMULATOR_BOOTBANK_SIZE_MB = 1024

   @classmethod
   def InitBootBankSizes(cls):
      """Initialize bootbank and stagebootbank sizes of the system.
      """
      if SYSTEM_STORAGE_ENABLED:
         try:
            # If simulator, then return a hardcoded value for bootbank size
            if HostInfo.HostOSIsSimulator():
               cls._BOOTBANK_SIZE = cls._SIMULATOR_BOOTBANK_SIZE_MB
            else:
               cls._BOOTBANK_SIZE = HostInfo.GetFsSize(cls.BOOTBANK) // MIB
         except Exception:
            # PXE or installer
            cls._BOOTBANK_SIZE = 0
         cls._STAGEBOOTBANK_SIZE = \
            (cls._BOOTBANK_SIZE
             if cls._BOOTBANK_SIZE + 10 > cls.MIN_STAGEBOOTBANK_SIZE
             else cls.MIN_STAGEBOOTBANK_SIZE)
      else:
         cls._BOOTBANK_SIZE = cls.LEGACY_BOOTBANK_SIZE
         cls._STAGEBOOTBANK_SIZE = cls.LEGACY_BOOTBANK_SIZE

   @classmethod
   def GetBootBankSize(cls):
      if cls._BOOTBANK_SIZE is None:
         cls.InitBootBankSizes()
      return cls._BOOTBANK_SIZE

   @classmethod
   def GetStageBootBankSize(cls):
      if cls._STAGEBOOTBANK_SIZE is None:
         cls.InitBootBankSizes()
      return cls._STAGEBOOTBANK_SIZE

   # Used during an legacy upgrade from 6.x.
   TMPBOOTBANK = '/tmpbootbank'
   OLDALTBOOTBANK = '/oldaltbootbank'

   BUFFER_SIZE = MIB
   BTLDR_PAYLOAD = 'btldr'
   BTLDR_RAMDISK_NAME = 'bootloader-installer'

   # NOTE: 10MB for database, state.tgz
   BOOTBANK_PADDING_MB = 10

   # BOOTBANK_STATE should be considered as bit pattern state
   # because we can have staged and updated state.
   # The first bit ( int 1) is for staged or not.
   # The second bit ( int 2 ) is for updated or not.
   # 'staged' and 'updated' state is that the both bits are on:
   # > BOOTBANK_STATE_STAGED | BOOTBANK_STATE_UPDATED
   # We can check whether the stage is 'staged',
   # > state & BOOTBANK_STATE_STAGED
   # To check whether it's updated,
   # > state & BOOTBANK_STATE_UPDATED
   BOOTBANK_STATE_FRESH = 0
   BOOTBANK_STATE_STAGED = 1
   BOOTBANK_STATE_UPDATED = 2

   SUPPORTED_VIBS = set([Vib.BaseVib.TYPE_BOOTBANK,])
   # BootBank installer supports both gzip types and misc types that
   # can be packaged in a bootbank VIB, the former are placed directly
   # in a bootbank and the latter are stored in a tar on bootbank.
   SUPPORTED_GZIP_PAYLOADS = (Vib.Payload.TYPE_TGZ, Vib.Payload.TYPE_VGZ,
      Vib.Payload.TYPE_BOOT)
   SUPPORTED_NONGZIP_PAYLOADS = Vib.Payload.NON_GZIP_TYPES
   SUPPORTED_PAYLOADS = tuple(list(SUPPORTED_GZIP_PAYLOADS) +
                              list(SUPPORTED_NONGZIP_PAYLOADS))

   def __init__(self, bootbank=BOOTBANK, altbootbank=ALTBOOTBANK):
      """Constructor for this Installer class.
         /bootbank and /altbootbank must both exist. /bootbank must points to
         valid bootbank with boot.cfg file.  If not, this class is not
         instantiated.

         Exceptions:
            InstallerNotAppropriate - Environment is not appropriate for
               this type of installer.
      """
      #TODO: add probe to host environment, raise error if bootbank is not
      # supported.
      try:
         self.bootbank = BootBank(bootbank)
      except InvalidBootbankError as e:
         msg = 'bootbank is invalid: %s' % (e)
         raise Errors.InstallerNotAppropriate(self.installertype, msg)

      try:
         self.bootbank.Load(createprofile = True)
         self.altbootbank = BootBank(altbootbank)
         # Altbootbank is not always available.
         self.altbootbank.Load(raiseerror=False)
         if os.path.exists(self.STAGEBOOTBANK):
            self.stagebootbank = BootBank(self.STAGEBOOTBANK)
            self.stagebootbank.Load(raiseerror=False)
         else:
            self.stagebootbank = None
      except InvalidBootbankError as e:
         msg = 'altbootbank is invalid: %s' % (e)
         raise Errors.InstallationError(e, None, msg)
      # Unassigned unless a 6.x upgrade is in remediation.
      self.tempBootBank = None
      self.problems = list()

      self._isARM = (platform.machine() == 'aarch64')

   @property
   def bootbankstate(self):
      staged = 0
      if self.stagebootbank and \
            self.stagebootbank.bootstate & BootBankBootCfg.BOOTSTATE_STAGED:
         staged = self.BOOTBANK_STATE_STAGED
      if self.bootbank.updated < self.altbootbank.updated and \
            self.altbootbank.bootstate in (BootBankBootCfg.BOOTSTATE_SUCCESS,
                                           BootBankBootCfg.BOOTSTATE_UPDATED):
         # When a live vib is installed, updated is incremented,
         # but bootstate remains to 0 (BOOTSTATE_SUCCESS).
         return self.BOOTBANK_STATE_UPDATED | staged
      return self.BOOTBANK_STATE_FRESH | staged

   @property
   def currentbootbank(self):
      if self.bootbankstate & self.BOOTBANK_STATE_UPDATED:
         return self.altbootbank
      else:
         return self.bootbank

   @property
   def database(self):
      return self.currentbootbank.db

   @property
   def stagedatabase(self):
      if self.stagebootbank and \
            self.stagebootbank.bootstate & BootBankBootCfg.BOOTSTATE_STAGED:
         return self.stagebootbank.db
      else:
         return None

   def CheckInstallationSize(self, imgprofile):
      '''Calculate the installation size of imgprofile on bootbank.
         Verify bootbank can hold the image. BOOTBANK_PADDING_MB is reserved for
         state.tgz and other extra files.

         NOTE: No attempt is made to estimate the size of state.tgz, transaction
               might still fail if state.tgz needs more than the reserved padding
               space.

         Parameters:
            * imgprofile - Target image profile

         Exceptions:
            * InstallationError - Installation size of this image profile can't
                                  fit in bootbank partition.
      '''
      totalsize = self.GetInstallationSize(imgprofile)
      capacity = (self.GetStageBootBankSize() - self.BOOTBANK_PADDING_MB) * MIB
      log.debug('The pending transaction requires %u bytes free space, the '
                'staging area can hold %u bytes', totalsize, capacity)
      if totalsize > capacity:
         totalInMib = totalsize // MIB
         capacityInMib = capacity // MIB
         msg = ('The pending transaction requires %u MB free space, however '
                'the maximum supported size is %u MB.')
         log.error(msg, totalInMib, capacityInMib)
         raise Errors.InstallationError(None, None,
                                        msg % (totalInMib, capacityInMib))

   def CheckBootState(self):
      '''Check the bootstate of current bootbank. If bootstate is not
         BOOTSTATE_SUCCESS, raise InstallationError exception.
      '''
      if self.bootbank.bootstate != BootBankBootCfg.BOOTSTATE_SUCCESS:
         msg = ('Current bootbank %s is not verified and most likely a '
                'serious problem was encountered during boot, it is not safe '
                'to continue altbootbank install. bootstate is %d, expected '
                'value is %s.' % (self.bootbank.path, self.bootbank.bootstate,
                 BootBankBootCfg.BOOTSTATE_SUCCESS))
         raise Errors.InstallationError(None, None, msg)

   def PreInstCheck(self, imgprofile):
      '''Perform sanity check for new image profile change.
         Parameters:
            * imgprofile - Target image profile

         Exceptions:
            * InstallationError - /bootbank is not in BOOTSTATE_SUCCESS state or
                                  installation size can't fit in bootbank
                                  partition.
      '''
      self.CheckBootState()
      self.CheckInstallationSize(imgprofile)

   def _SetupStageBootbank(self, imgProfile):
      """Set self.stagebootbank."""
      ramdiskSize = self.GetStageBootBankSize()

      createRamdisk = True
      if os.path.exists(self.STAGEBOOTBANK):
         try:
            size = Ramdisk.GetRamdiskSizeInMiB(self.STAGEBOOTBANK_NAME)
         except Errors.InstallationError as e:
            log.info('Stage bootbank exists but failed to get ramdisk size: '
                     ' %s', e)
            size = 0

         if size >= ramdiskSize - 1:
            # Actual minimum bootbank is 499MB in size, not exactly 500MB.
            createRamdisk = False
         else:
            # 250MB ramdisk created by 6.x before upgrade will be removed.
            log.debug('Removing stagebootbank with a size of %s MB', size)
            self._CleanupStageBootbank(raiseException=True)

      if createRamdisk:
         installSizeMb = int(round(self.GetInstallationSize(imgProfile) / MIB))
         try:
            Ramdisk.CreateRamdisk(ramdiskSize, self.STAGEBOOTBANK_NAME,
                                  self.STAGEBOOTBANK,
                                  reserveSize=installSizeMb)
         except Errors.InstallationError:
            # we need to set stagebootbank None when ramdisk create fail
            self.stagebootbank = None
            raise
      self.stagebootbank = BootBank(self.STAGEBOOTBANK)

   def _CleanupStageBootbank(self, raiseException=False):
      """Empty stagebootbank and remove ramdisk. If raiseException is set,
         allow RemoveRamdisk() to raise an exception when an error occurs.
      """
      Ramdisk.RemoveRamdisk(self.STAGEBOOTBANK_NAME, self.STAGEBOOTBANK,
                            raiseException=raiseException)
      self.stagebootbank = None

   def StartTransaction(self, imgprofile, preparedest=True, **kwargs):
      """Initiates a new installation transaction. Calculate what actions
         need to be taken.  Prepare /altbootbank by copying all contents
         from /bootbank.
         This method may change the installation destination.

         Parameters:
            * imgprofile - The ImageProfile instance representing the
                           target set of VIBs for the new image
            * preparedest - Boolean, if True, then prepare the destination.
                           Set to false for a "dry run", to avoid changing
                           the destination.
            * forcebootbank - Boolean, if True, skip install of live image
                           even if its eligible for live install
         Returns:
            A tuple (installs, removes, staged), installs and removes are list
            of VIB IDs for HostImage.Stage() to install to the destination and
            to remove from the destination, in order to make it compliant
            with imgprofile. If altbootbank has already staged the imgprofile,
            staged is True.
            If there is nothing to do, (None, None, False) is returned.
         Exceptions:
            InstallationError
      """
      imgprofile = self.GetInstallerImageProfile(imgprofile)

      if preparedest:
         # Create/load stagebootbank with a remediation.
         self._SetupStageBootbank(imgprofile)
         try:
            self.stagebootbank.Load(raiseerror=False)
         except Exception as e:
            # Unknown contents, empty the stage bootbank.
            log.debug('Unexpected error when loading stagebootbank: %s, '
                      'clearing stagebootbank', str(e))
            self.stagebootbank.Clear()

      staged = False
      if (self.stagebootbank and
          self.stagebootbank.bootstate == BootBankBootCfg.BOOTSTATE_STAGED):
         if (self.stagebootbank.db.profile is not None and
             self.stagebootbank.db.profile.HasSameInventory(imgprofile)):
            # stagebootbank has exact same image staged.
            staged = True
         elif preparedest:
            # stagebootbank exists but it's not for the current transaction,
            # clean its contents.
            self.stagebootbank.Clear()

      adds, removes = imgprofile.Diff(self.database.profile)

      if staged:
         return (adds, removes, staged)

      if preparedest and (removes or adds):
         if self.bootbankstate & self.BOOTBANK_STATE_UPDATED:
            srcbootbank = self.altbootbank
         else:
            srcbootbank = self.bootbank

         keeps = set(imgprofile.vibIDs) - set(adds)
         self.stagebootbank.Stage(srcbootbank, keeps, imgprofile)

      return (adds, removes, staged)

   def UpdateVibDatabase(self, newvib):
      """Update missing properties of vib metadata
         New vibs are always installed in altbootbank

         Parameters:
            * newvib   - The new vib to use as source
         Returns:
            None if the update succeeds, Exception otherwise
         Exceptions:
            VibFormatError
      """
      try:
         self.stagebootbank._UpdateVib(newvib)
      except Exception as e:
         log.debug('failed updating bootbank for %s', newvib.name)
         raise

   def VerifyPayloadChecksum(self, vibid, payload, rebootPending=False):
      """Verify the checksum of a given payload.

         Parameters:
            * vibid   - The Vib id containing the payload
            * payload - The Vib.Payload instance to read or write
            * rebootPending - Whether the system is pending reboot from a
                              bootbank-only change.
         Returns:
            None if verification succeeds, Exception otherwise
         Exceptions:
            ChecksumVerificationError
            InstallationError
      """
      if payload.payloadtype in self.SUPPORTED_GZIP_PAYLOADS:
         # Tardisks and boot type payloads, both are loaded during the
         # boot process. Other payloads are excluded here, they are stored in
         # basemisc.tgz only for VLCM host seeding where their checksums are
         # verified.

         # Special case for payloads that encapsulate state.
         # These payloads can change. Hence their checksums
         # will not match the ones in the descriptor.
         # TODO: Since OEMs can add additional useropts modules,
         # we will need to get this list from the kernel.
         if any(payload.name.startswith(p) for p in CHECKSUM_EXEMPT_PAYLOADS):
            return

         gunzipChecksums = []
         for checksum in payload.checksums:
            if checksum.verifyprocess == 'gunzip':
               # In a stateful installed host, the bootbank payloads for
               # tardisks are generated by reading the nodes in the /tardisks
               # directory and gzip'ing the contents. Such compressed payload
               # may have a different hash than the one in the original
               # descriptor file, if the VIB was originally created by legacy
               # vibauthor/build where gzip added timestamp and other file
               # metadata to the payload. Hence we calculate hash on the
               # uncompressed version of a payload and compare with "gunzip"
               # checksum to verify the payload.
               gunzipChecksums.append((checksum.checksumtype,
                                       checksum.checksum))

         if not gunzipChecksums:
            msg = "Failed to find checksum for payload %s" % payload.name
            log.error(msg)
            raise Errors.ChecksumVerificationError(msg)

         # Prefer more secure hash, e.g. sha-256 over sha-1, but accepting both
         # since old VIBs may only have sha-1.
         algo, checksum = sorted(gunzipChecksums, reverse=True)[0]
         try:
            if rebootPending:
               # When a reboot is pending, finding a VIB that is in the live
               # imageDB is not straightforward:
               # 1) VIBs not live installed/updated should be in /bootbank.
               # 2) If after a live VIB install, a bootbank-only change
               #    occurred, this live VIB is in /altbootbank.
               # 3) If after a live VIB install, a bootbank-only change that
               #    either removes/updates this VIB occurred, it won't be found
               #    in both bootbanks. In this case, do not panic and return.
               if vibid in self.altbootbank.db.profile.vibIDs:
                  # Prefer altbootbank since it will be booted on reboot.
                  bootbank = self.altbootbank
               elif vibid in self.bootbank.db.profile.vibIDs:
                  bootbank = self.bootbank
               else:
                  log.info('VIB %s is not found in both bootbanks, it may have '
                           'been removed in a reboot-required transaction. '
                           'Skip checking payload checksums.', vibid)
                  return
            else:
               # currentbootbank matches the live image and should contain the
               # VIB.
               bootbank = self.currentbootbank

            if vibid in bootbank.db.profile.vibIDs:
               vibstate = bootbank.db.profile.vibstates[vibid]
               rootpath = bootbank.path
            else:
               msg = 'Cannot find VIB %s in BootbankInstaller' % vibid
               raise Errors.InstallationError(None, [vibid], msg)

            filepath = None
            if payload.name in vibstate.payloads:
               filepath = os.path.join(rootpath,
                                       vibstate.payloads[payload.name])
            else:
               msg = ("VIB %s found in BootbankInstaller (%s), but payload %s "
                      "is not found in the VIB"
                      % (vibid, rootpath, payload.name))
               raise Errors.InstallationError(None, [vibid], msg)

            if os.path.isfile(filepath):
               with gzip.open(filepath, 'rb') as sourcefp:
                  log.info("Verifying hash on vibid %s, payload %s, expected "
                           "%s hash %s", vibid, payload.name, algo, checksum)
                  hashedfp = HashedStream.HashedStream(sourcefp,
                                                       checksum,
                                                       algo.replace('-', ''))
                  inbytes = hashedfp.read(self.BUFFER_SIZE)
                  while inbytes:
                     inbytes = hashedfp.read(self.BUFFER_SIZE)
                  hashedfp.close()
            else:
               msg = ("Payload '%s' of VIB %s is missing at '%s'"
                      % (payload.name, vibid, filepath))
               raise Errors.InstallationError(None, None, msg)
         except Exception as e:
            msg = ("Failed to verify checksum for payload %s: %s"
                   % (payload.name, e))
            log.error(msg)
            raise Errors.ChecksumVerificationError(msg)

   def OpenPayloadFile(self, vibid, payload, read = True, write = False,
                       fromBaseMisc=False):
      """Creates and returns a File-like object for either reading from
         or writing to a given payload.  One of read or write must be True, but
         ready and write cannot both be true.

         Parameters:
            * vibid   - The Vib id containing the payload
            * payload - The Vib.Payload instance to read or write
            * read    - Set to True to get a File object for reading
                        from the payload.
            * write   - Set to True to get a File object for writing
                        to the payload.
         Returns:
            A File-like object, must support read (for read), write (for
            write), close methods.
            None if the desired read/write is not supported.
         Exceptions:
            AssertionError    - neither read nor write is True, or both are true
            InstallationError - Cannot open file to write
      """
      Installer.OpenPayloadFile(self, vibid, payload, read=read, write=write,
                                fromBaseMisc=False)

      if payload.payloadtype not in self.SUPPORTED_PAYLOADS:
         log.debug("Payload %s of type '%s' in VIB '%s' is not supported by "
                   "BootbankInstaller.", payload.name, payload.payloadtype,
                   vibid)
         return None

      if read == True:
         def openPayloadForRead(vibId, payload, filePath):
            if not os.path.exists(filePath):
               msg = ('Payload %s of VIB %s does not exist at path %s'
                      % (payload.name, vibId, filePath))
               raise Errors.InstallationError(None, [vibId], msg)

            try:
               return open(filePath, 'rb')
            except EnvironmentError as e:
               msg = ('Error in opening payload %s of VIB %s at %s for read: %s'
                      % (payload.name, vibid, filePath, e))
               raise Errors.InstallationError(e, [vibid], msg)

         localfile = None
         if vibid in self.bootbank.db.profile.vibIDs:
            vibstate = self.bootbank.db.profile.vibstates[vibid]
            rootpath = self.bootbank.path
         elif self.bootbankstate & self.BOOTBANK_STATE_UPDATED and \
               vibid in self.altbootbank.db.profile.vibIDs:
            vibstate = self.altbootbank.db.profile.vibstates[vibid]
            rootpath = self.altbootbank.path
         elif self.bootbankstate & self.BOOTBANK_STATE_STAGED and \
               vibid in self.stagebootbank.db.profile.vibIDs:
            vibstate = self.stagebootbank.db.profile.vibstates[vibid]
            rootpath = self.stagebootbank.path
         else:
            msg = 'Cannot find VIB %s in BootbankInstaller' % vibid
            raise Errors.InstallationError(None, [vibid], msg)

         if fromBaseMisc or \
            payload.payloadtype not in self.SUPPORTED_GZIP_PAYLOADS:
            # Misc payloads are in basemisc.tgz for bootbanks or in a sub-dir
            # for stagebootbank.
            if self.stagebootbank and rootpath == self.stagebootbank.path:
               filePath = os.path.join(self.stagebootbank.baseMiscPayloadsDir,
                                       payload.payloadtype, payload.name)
               return openPayloadForRead(vibid, payload, filePath)

            baseMiscTarPath = os.path.join(rootpath,
                                           BootBank.BASEMISC_PAYLOADTAR)
            if not os.path.exists(baseMiscTarPath):
               msg = ('VIB %s found in BootbankInstaller (%s), but %s does not'
                      'exist to provide payload %s', vibid, rootpath,
                      BootBank.BASEMISC_PAYLOADTAR, payload.name)
               raise Errors.InstallationError(None, [vibid], msg)

            baseMiscTar = tarfile.open(baseMiscTarPath, 'r')
            tarFilePath = os.path.join(PayloadTar.PAYLOADTAR_PREFIX,
                                       payload.payloadtype,
                                       payload.name)
            return baseMiscTar.extractfile(tarFilePath)

         if payload.name in vibstate.payloads:
            localfile = os.path.join(rootpath, vibstate.payloads[payload.name])
         else:
            msg = ("VIB %s found in BootbankInstaller (%s), but payload %s is "
                   "not found in the VIB" % (vibid, rootpath, payload.name))
            raise Errors.InstallationError(None, [vibid], msg)

         return openPayloadForRead(vibid, payload, localfile)

      elif write == True:
         def openPayloadForWrite(dirPath, fileName):
            fullPath = os.path.join(dirPath, fileName)
            try:
               if not os.path.exists(dirPath):
                  # Misc payloads go into sub-dirs that may not exist yet.
                  os.makedirs(dirPath)
               return open(fullPath, 'wb')
            except EnvironmentError as e:
               self.Cleanup()
               msg = 'Can not open %s to write payload %s: %s' % (fullPath,
                     fileName, e)
               raise Errors.InstallationError(e, None, msg)

         if payload.payloadtype in self.SUPPORTED_GZIP_PAYLOADS:
            # BOOT, VGZ and TGZ payload
            return openPayloadForWrite(self.stagebootbank.path,
                                       payload.localname)
         else:
            # Write misc payloads to keep into staging dir, sorted by their
            # types.
            writeDir = os.path.join(self.stagebootbank.baseMiscPayloadsDir,
                                    payload.payloadtype)
            return openPayloadForWrite(writeDir, payload.name)

      return None

   def SaveDatabase(self):
      """Write out the updated database of the installer.
      """
      self.database.Save(savesig=True)

   def Cleanup(self):
      """Cleans up after a Transaction that has been started, perhaps from
         a previous instantiation of the Installer class.
         Unmount and remove stagebootbank.
      """
      self._CleanupStageBootbank()

   def CompleteStage(self):
      """Complete the staging bootbank: writing out the new imgdb, boot.cfg
         and nongzip.tgz.

         Exceptions:
            InstallationError
      """
      imgprofile = self.stagebootbank.db.profile
      bootorder = imgprofile.GetBootOrder(self.SUPPORTED_GZIP_PAYLOADS,
                                          self.SUPPORTED_VIBS)

      #TODO: need to accomodate custom boot option bewteen kernel and regular
      # module
      self.stagebootbank.bootcfg.build = imgprofile.GetEsxVersion()

      try:
         _, p = bootorder[0]
         self.stagebootbank.bootcfg.kernel = p.localname
         modules = [p.localname for (vibid, p) in bootorder[1:]]
         self.stagebootbank.bootcfg.modules = modules

         self.stagebootbank.bootcfg.updateState(
                                             BootBankBootCfg.BOOTSTATE_STAGED)
         self.stagebootbank.updated = self.bootbank.updated + 1
         self.stagebootbank.Save()

         # Create misc esx-base payload tar, the staging folder does not exist
         # when no payloads are staged.
         if os.path.exists(self.stagebootbank.baseMiscPayloadsDir):
            tarPath = os.path.join(self.stagebootbank._path,
                                   self.stagebootbank.BASEMISC_PAYLOADTAR)
            baseMiscTar = PayloadTar(tarPath)
            try:
               baseMiscTar.FromDirectory(self.stagebootbank.baseMiscPayloadsDir)
            finally:
               baseMiscTar.Close()

            shutil.rmtree(self.stagebootbank.baseMiscPayloadsDir)
      except Exception as e:
         self.Cleanup()
         msg = 'Failed to finish bootbank stage: %s'  % (e)
         raise Errors.InstallationError(e, None, msg)

   def CacheNewImage(self, srcinstlr):
      """Cache image to stagebootbank and remediate altbootbank using another
         installer.
      """
      log.info('Caching LiveImage to stagebootbank...')
      self._SetupStageBootbank(srcinstlr.database.profile)

      # Copy preupgrade-state.tgz to staged bootbank. This file needs to be
      # copied over to altbootbank during a live VIB install. The copy is
      # not required for reboot required VIB install because we regenerate
      # preupgrade-state.tgz during jumpstart of the next boot. Today, we do
      # not have a way to go from reboot required state to no reboot required
      # state so it is suffcient to have this change in one place here.
      # We will have to revisit if this assumption changes.
      src = os.path.join(self.bootbank.path, BootBank.BACKUP_STATE_TGZ)
      if os.path.exists(src):
         dst = os.path.join(self.stagebootbank.path, BootBank.BACKUP_STATE_TGZ)
         shutil.copy2(src, dst)

      self.stagebootbank.Clear()
      self.stagebootbank.bootcfg.updateState(BootBankBootCfg.BOOTSTATE_EMPTY)
      try:
         self.stagebootbank.bootcfg.write()
      except (IOError, BootCfg.BootCfgError) as e:
         msg = 'Failed to save bootcfg to file %s: %s' % \
               (self.stagebootbank.bootcfg.path, e)
         raise Errors.InstallationError(e, None, msg)

      profile = srcinstlr.database.profile.Copy()
      self.stagebootbank.db.PopulateWith(imgProfile=profile)

      bbNonGzipTarPath = os.path.join(self.bootbank.path,
                                      BootBank.BASEMISC_PAYLOADTAR)
      bbNonGzipTar = None
      if os.path.isfile(bbNonGzipTarPath):
         bbNonGzipTar = tarfile.open(bbNonGzipTarPath, 'r')
      else:
         log.info('%s does not exist, misc payloads will not be extracted',
                  bbNonGzipTarPath)

      # Read payloads from live image
      for vibid in profile.vibIDs:
         for payload in srcinstlr.database.vibs[vibid].payloads:
            if (not bbNonGzipTar and
                payload.payloadtype in self.SUPPORTED_NONGZIP_PAYLOADS):
               # Skip when there is no source for the misc payloads.
               continue

            payload.localname = profile.vibstates[vibid].payloads[payload.name]

            # Get dest file obj, skip if payload is not supported
            dst = self.OpenPayloadFile(vibid, payload, write=True, read=False)
            if dst is None:
               continue

            log.debug("About to write payload '%s' of VIB %s to '%s'",
                      payload.name, vibid, self.stagebootbank.path)

            if payload.payloadtype in self.SUPPORTED_GZIP_PAYLOADS:
               # Gzip payloads
               src = None
               compress = False
               # Get src from bootbank
               if vibid in self.bootbank.db.profile.vibstates \
                   and  (payload.name in
                         self.bootbank.db.profile.vibstates[vibid].payloads):
                  vibstate = self.bootbank.db.profile.vibstates[vibid]
                  srcfile = os.path.join(self.bootbank.path,
                        vibstate.payloads[payload.name])
                  log.debug('Using %s from bootbank as source' % (srcfile))
                  src = open(srcfile, 'rb')
               # Get src from live image for newly installed module
               elif payload.payloadtype != payload.TYPE_BOOT:
                  log.debug('Using source from %s' % (
                     srcinstlr.__class__.__name__))
                  src = srcinstlr.OpenPayloadFile(vibid, payload, read=True,
                        write=False)
                  # When using live image as source, compress the tardisks
                  compress = (srcinstlr.installertype == 'live')
               else:
                  log.error('Could not found payload %s from sources' % (
                     payload.name))

               if src == None:
                  dst.close()
                  if bbNonGzipTar:
                     bbNonGzipTar.close()
                  msg = ('Error in opening payload %s of VIB %s from installer '
                         '%s' % (payload.name, vibid, srcinstlr.installertype))
                  raise Errors.InstallationError(None, [vibid], msg)

               try:
                  Vib.copyPayloadFileObj(payload, src, dst, compress=compress)
               except Exception:
                  if bbNonGzipTar:
                     bbNonGzipTar.close()
                  raise
               finally:
                  src.close()
                  dst.close()
            else:
               # Misc payload, extract into the staging area.
               tarFilePath = os.path.join(PayloadTar.PAYLOADTAR_PREFIX,
                                          payload.payloadtype, payload.name)

               try:
                  m = bbNonGzipTar.getmember(tarFilePath)
                  bbNonGzipTar.extract(m, path=os.path.dirname(dst.name))
               except KeyError:
                  log.info('Misc payload %s not found in %s',
                        payload.name, bbNonGzipTarPath)
               except tarfile.TarError as e:
                  if bbNonGzipTar:
                     bbNonGzipTar.close()
                  msg = ('Failed to extract misc payload %s of VIB %s: %s'
                         % (payload.name, vibid, e))
                  raise Errors.InstallationError(None, [vibid], msg)
               finally:
                  dst.close()

      if bbNonGzipTar:
         bbNonGzipTar.close()

      self.CompleteStage()
      return self.Remediate(liveimage=True)

   def _RunInstallBootloader(self, btldrRamdiskPath, baseVibId):
      """Execute install-bootloader script to update the bootloader.
      """
      cmdPath = os.path.join(btldrRamdiskPath,
                             'usr/lib/vmware/bootloader-installer',
                             'install-bootloader')
      cmd = [cmdPath, '--prefix', btldrRamdiskPath]
      try:
         rc, out = runcommand(' '.join(cmd))
      except RunCommandError as e:
         msg = ('Failed to execute bootloader install script: %s' % str(e))
         raise Errors.InstallationError(e, [baseVibId], msg)
      if rc != 0:
         msg = ('Execution of install-bootloader script failed with status %d\n'
                'output: %s' % (rc, byteToStr(out)))
         raise Errors.InstallationError(None, [baseVibId], msg)

   def _SysStorageBootPartAction(self, disk, btldrRamdiskPath, baseVibId):
      """Install bootloader with SystemStorage.
      """
      from systemStorage.esxboot import BOOTLOADER_DIR, installBootloader

      if disk.hasLegacyBootPart:
         raise NotImplementedError('SystemStorage does not support legacy small'
                                   'boot partition.')
      else:
         # Use SystemStorage with new large boot partition.
         try:
            srcRoot = os.path.join(btldrRamdiskPath,
                                   BOOTLOADER_DIR.lstrip(os.path.sep))
            installBootloader(disk, srcRoot=srcRoot, isARM=self._isARM)
         except Exception as e:
            msg = 'Failed to update bootloader: %s' % str(e)
            raise Errors.InstallationError(e, [baseVibId], msg)

   def _UpdateBootloader(self, imgprofile, dstBootBank):
      """Update the bootloader during an upgrade.
         For each bootbank remediation check to see if the esx-base vib is
         being installed/upgraded and if the vib has the bootloader payload.
         If the payload is present in the vib, then this function mounts the
         the payload tardisk and runs a script that updates the bootloader.
      """
      log.info('Updating ESX bootloader')
      try:
         for vibid in imgprofile.vibstates:
            vib = dstBootBank.db.vibs[vibid]
            if vib.name != 'esx-base':
               continue
            payload = None
            for pl in vib.payloads:
               if pl.name == self.BTLDR_PAYLOAD:
                  payload = pl
                  break
            if not payload:
               break
            log.info('Mounting bootloader tardisk to update bootloader')
            # If the bootloader installer payload is found in
            # the esx-base VIB, run the internal script to update
            # the bootloader.
            srcName = imgprofile.vibstates[vibid].payloads[self.BTLDR_PAYLOAD]
            srcPath = os.path.join(dstBootBank.path, srcName)
            dstName = 'tmp-' + self.BTLDR_PAYLOAD
            dstPath = os.path.join(TMP_DIR, dstName)

            # Decompress the payload to tardisk in the tmp directory
            with open(srcPath, 'rb') as srcfObj:
               with open(dstPath, 'wb') as destfObj:
                  Vib.copyPayloadFileObj(payload, srcfObj, destfObj,
                                         decompress=True)

            # Create a ramdisk and mount the tardisk in it
            btldrRamdiskPath = os.path.join(TMP_DIR, self.BTLDR_RAMDISK_NAME)
            # The ramdisk is used for mounting and possibily extraction.
            ramdiskSize = (payload.size // MIB + 1) * 5
            try:
               Ramdisk.CreateRamdisk(ramdiskSize,
                                     self.BTLDR_RAMDISK_NAME,
                                     btldrRamdiskPath)
               Ramdisk.MountTardiskInRamdisk(vibid, self.BTLDR_PAYLOAD, dstPath,
                                             self.BTLDR_RAMDISK_NAME,
                                             btldrRamdiskPath,
                                             checkAcceptance=False)

               runScript = True # If to use install-bootloader script
               if SYSTEM_STORAGE_ENABLED:
                  from systemStorage.esxboot import getActiveBootDisk
                  disk = getActiveBootDisk()
                  disk.scanPartitions()

                  if not disk.hasLegacyBootPart:
                     self._SysStorageBootPartAction(disk, btldrRamdiskPath,
                                                    vibid)
                     runScript = False

               if runScript:
                  # Run the bootloader-installer script for small boot
                  # partition during an upgrade.
                  self._RunInstallBootloader(btldrRamdiskPath, vibid)
            finally:
               Ramdisk.UnmountManualTardisk(dstName)
               Ramdisk.RemoveRamdisk(self.BTLDR_RAMDISK_NAME, btldrRamdiskPath)
      except Errors.InstallationError:
         raise
      except Exception as e:
         msg = 'Failed to update bootloader: %s' % str(e)
         raise Errors.InstallationError(e, None, msg)

   def _copyStagedImage(self):
      """Copy the stage bootbank to the target bootbank and return the bootbank
         object for further remediation actions.
         For most cases, the target bootbank is the altbootbank.
         In case of an 6.x upgrade, create a larger temporary bootbank as the
         target bootbank.
      """
      if not SYSTEM_STORAGE_ENABLED:
         self.altbootbank.FileCopy(self.stagebootbank)
         self.altbootbank.Load(raiseerror=False)
         return self.altbootbank

      # If simulator, then return a hardcoded value for altbootbank size
      if HostInfo.HostOSIsSimulator():
         altBbSize = self._SIMULATOR_BOOTBANK_SIZE_MB
      else:
         altBbSize = HostInfo.GetFsSize(self.ALTBOOTBANK) // MIB

      if altBbSize + self.BOOTBANK_PADDING_MB > \
         self.GetStageBootBankSize():
         # The altbootbank has equal or more space than the staging allowance.
         # This means one of these two:
         # 1) We have the new disk layout, bootbank size matches the
         #    stage bookbank.
         # 1) The temp bootbank has already been created and pointed to as
         #    /altbootbank (in case of retrying a failed upgrade), the space is
         #    more than or equal to 500MB.
         # As a result, we can just return the altbootbank.
         # Use the 10MB padding to counter any rounding issue.
         self.altbootbank.FileCopy(self.stagebootbank)
         self.altbootbank.Load(raiseerror=False)
         return self.altbootbank
      else:
         # We are in a legacy 6.x upgrade, prepare a temporary bootbank.
         tempBootBank = getTempBootbank(self.bootbank.path,
                                        self.altbootbank.path)
         log.info('Temporary bootbank %s has been prepared' % tempBootBank)
         # Create a symlink to help lookup.
         if os.path.exists(self.TMPBOOTBANK):
            os.remove(self.TMPBOOTBANK)
         os.symlink(tempBootBank, self.TMPBOOTBANK, target_is_directory=True)
         # Create the BootBank object and return, no need to clean as
         # getTempBootbank() already did so.
         self.tempBootBank = BootBank(self.TMPBOOTBANK)
         self.tempBootBank.FileCopy(self.stagebootbank, purge=False)
         self.tempBootBank.Load(raiseerror=False)
         return self.tempBootBank

   def _relinkAltBootBank(self):
      '''When we are finishing an upgrade from 6.x, the temporary bootbank is
         ready to be linked as /altbootbank and replace the original volume.
         Updating the symlink allows profile/vib --rebooting-image commands
         and backup.sh to work properly before the next reboot.
      '''
      if not SYSTEM_STORAGE_ENABLED:
         return

      # Lock /altbootbank from backup.sh access
      altbbLockFile = '/tmp/%s.lck' % self.altbootbank.path.replace('/', '_')
      flock = LockFile.acquireLock(altbbLockFile)
      try:
         altbbPath = os.path.realpath(self.altbootbank.path)
         log.info('Repointing /altbootbank to %s'
                  % os.path.realpath(self.TMPBOOTBANK))
         os.rename(self.TMPBOOTBANK, self.ALTBOOTBANK)
         # Update paths of the bootbanks.
         # We don't swap these objects as we are finishing.
         self.altbootbank._changePath(altbbPath)
         self.tempBootBank._changePath(self.ALTBOOTBANK)
      except Exception as e:
         try:
            os.remove(self.TMPBOOTBANK)
         except FileNotFoundError:
            pass
         msg = 'Failed to update altbootbank symlink: %s' % str(e)
         raise Errors.InstallationError(e, None, msg)
      finally:
         flock.Unlock()

   def _updateAltBootbank(self, dstBootBank):
      ''' Purge the contents of altbootbank, then give it a new boot.cfg
          based on dstBootBank's boot.cfg, but with the path to dstBootBank
          as a prefix. This allows loadESX to boot from altbootbank while
          getting the modules from dstBootBank. Also add quickboot=1 to the
          new boot.cfg, so that if loadESX is not used or fails, a full boot
          will not attempt to boot from the altbootbank.
      '''
      if not SYSTEM_STORAGE_ENABLED:
         return

      from systemStorage.upgradeUtils import backupBootCfg, restoreBootCfg

      # Clear previous altbootbank ESXi image since it's no longer needed
      self.altbootbank.Clear(purge=True)

      dstBootCfg = dstBootBank.bootcfg
      altBootCfg = self.altbootbank.bootcfg

      attrs = (['bootstate', 'kernel', 'kernelopt', 'modules', 'build',
                'updated'] + list(BootBankBootCfg.INT_OPTS))
      altBootCfg.copyAttrs(dstBootCfg, attrs)
      altBootCfg.updatePrefix(os.path.realpath(dstBootBank.path))
      altBootCfg.quickboot = 1

      try:
         altBootCfg.write()
      except (IOError, BootCfg.BootCfgError) as e:
         log.warn("Error while updating %s: %s", altBootCfg.path, str(e))
         restoreBootCfg(self.altbootbank.path)

   def Remediate(self, checkmaintmode=False, liveimage=False, **kwargs):
      """For bootbank installs, the remediation consists of a reboot. Since
         the core engine does not carry out the reboot, we simply NOP.
         Returns:
            A Boolean, True if a reboot is needed.
         Exceptions:
            HostNotChanged    - If no staged ImageProfile
            InstallationError -
      """
      if not self.bootbankstate & self.BOOTBANK_STATE_STAGED:
         msg = "Bootbank is not yet staged, nothing to remediate."
         raise Errors.HostNotChanged(msg)

      adds, _ = self.stagedatabase.profile.Diff(self.database.profile)

      dstBootBank = self._copyStagedImage()
      imgprofile = dstBootBank.db.profile

      # TODO: Verify bootbank with respect to imgprofile
      dstBootBank.Verify()

      dstBootBank.bootcfg.modules.append(BootBank.DB_FILE)
      if os.path.isfile(
            os.path.join(dstBootBank.path, BootBank.BASEMISC_PAYLOADTAR)):
         # basemisc.tgz may not exist without source files to form it.
         dstBootBank.bootcfg.modules.append(BootBank.BASEMISC_PAYLOADTAR)

      # If esx-base is in adds list, esxi version is updated.
      # This is useful for bootloader update and featurestate reset.
      esx_updated = False
      for vibid in adds:
         vib = dstBootBank.db.vibs[vibid]
         if vib.name == 'esx-base':
            esx_updated = True
            break

      if os.path.isfile(BACKUP_SH):
         try:
            rc, out = runcommand('%s 0 %s' % (BACKUP_SH, dstBootBank.path))
         except RunCommandError as e:
            msg = ('Failed to run %s: %s' % (BACKUP_SH, e))
            raise Errors.InstallationError(e, None, msg)

         if rc != 0:
            msg = ('Error in running backup script: non-zero code returned.'
                   '\nreturn code: %d\noutput: %s' % (rc, byteToStr(out)))
            raise Errors.InstallationError(None, None, msg)

         # When esxi is updated, we want to drop the features.gz written by
         # backup.sh, and let the profile's default features.gz take effect.
         # This will allow featurestate change to show up after reboot.
         if esx_updated:
            features_gz = 'features.gz'
            staged_features = os.path.join(self.stagebootbank.path, features_gz)
            altbootbank_features = os.path.join(dstBootBank.path,
                                                features_gz)
            if os.path.exists(staged_features):
               log.debug('Copying feature config file %s to %s.' %
                         (staged_features, altbootbank_features))
               shutil.copy2(staged_features, altbootbank_features)
      else:
         log.debug("Skipping backup script '%s', which was not found"
                   % (BACKUP_SH))

      statefile = os.path.join(dstBootBank.path, 'state.tgz')
      if os.path.isfile(statefile):
         dstBootBank.bootcfg.modules.append('state.tgz')

      log.info("boot config of '%s' is being updated, %d" %
               (dstBootBank.path, dstBootBank.updated))
      dstBootBank.updated = self.bootbank.updated + 1
      if liveimage:
         dstBootBank.bootcfg.updateState(BootBankBootCfg.BOOTSTATE_SUCCESS,
                                              liveimage=True)
      else:
         dstBootBank.bootcfg.updateState(BootBankBootCfg.BOOTSTATE_UPDATED)

      # Keep kernel option a customer might have specified, and mboot specific
      # flags in current bootbank's boot.cfg.
      attrs = ['kernelopt'] + list(BootBankBootCfg.INT_OPTS)
      dstBootBank.bootcfg.copyAttrs(self.bootbank.bootcfg, attrs)

      try:
         dstBootBank.bootcfg.write()
      except (IOError, BootCfg.BootCfgError) as e:
         msg = 'Failed to save bootcfg to file %s: %s' % \
               (dstBootBank.bootcfg.path, e)
         raise Errors.InstallationError(e, None, msg)

      # Check for bootloader update when esxi is updated
      if esx_updated:
         self._UpdateBootloader(imgprofile, dstBootBank)

      # The temp bootbank is only used for a legacy 6.x upgrade.
      # It should become the actual altbootbank now that we are completing.
      if self.tempBootBank:
         self._relinkAltBootBank()
         # Clear the old altbootbank and update its boot.cfg to prefix to the
         # temp bootbank for loadESX.
         self._updateAltBootbank(dstBootBank)

      return True
