########################################################################
# Copyright (C) 2010-2021 VMware, Inc.
# All Rights Reserved
########################################################################

"""This module contains the HostImage class,
   used for applying and extracting an image
   profile from the host.
"""

import datetime
import logging
import os
import struct
import tarfile
import time
from operator import attrgetter

from vmware.runcommand import runcommand, RunCommandError

from .ImageProfile import AcceptanceChecker
from .Installer import BootBankInstaller, LiveImageInstaller, LockerInstaller
from .PayloadTar import PayloadTar
from . import Bulletin
from . import Database
from . import Errors
from . import Downloader
from . import Vib
from . import VibCollection

from .Utils import LockFile, Ramdisk, HostInfo
from .Utils.XmlUtils import _utctzinfo
from .Utils.Misc import byteToStr, configStoreLogInit

try:
   import spicy
   HAVE_SPICY = True
except ImportError:
   HAVE_SPICY = False

try:
   import libconfigstorepy as cs
   HAVE_CS = True
except ImportError:
   HAVE_CS = False

# Please don't rearrange the sequence of the following
# DEFAULT_INSTALLERS tuple

DEFAULT_INSTALLERS = (LiveImageInstaller.LiveImageInstaller,
                      BootBankInstaller.BootBankInstaller,
                      LockerInstaller.LockerInstaller,)
INSTALLERTYPE_TO_NAME = {'boot' : 'BootBankInstaller',
      'live' : 'LiveImageInstaller',
      'locker': 'LockerInstaller'}

ADVCFG = '/sbin/esxcfg-advcfg'
VMKVOB = '/usr/lib/vmware/vob/bin/vmkvob'
LOCKFILE = '/var/run/esximg.pid'

# This is the default acceptance level that gets returned if the configuration
# cannot be found.
DEFAULT_HOST_ACCEPTANCE = Vib.ArFileVib.ACCEPTANCE_PARTNER

ACCEPTANCE_LEVEL_CS_OBJ_ID = ("acceptance_level", "software", "esx_update")

log = logging.getLogger('HostImage')

def verifyVibChecksum(vib, vibFile):
   """Compare the checksum present in VIB metadata (if present) with the actual
      downloaded VIB's checksum.
      Parameters:
         * vib - VIB object.
         * vibFile - The actual VIB file which is downloaded.

      Exceptions:
         VibChecksumError - if checksum couldn't be calculated or doesn't
                            match with the expected one.
   """
   if not vib.checksum.checksum:
      # This should happen only in case of esxcli vib install path.
      log.warning('VIB %s checksum is not available, skip verification',
                  vib.id)
   else:
      # If the overall VIB checksum doesn't match with the one stored in
      # metadata, installation should fail.
      try:
         checksum = VibCollection._getdigest(
                       vibFile, vib.checksum.checksumtype.replace('-', ''))
      except Exception as e:
         raise Errors.VibChecksumError(vib.id, "Failed to calculate VIB "
                                       "checksum: %s" % str(e))
      if checksum != vib.checksum.checksum:
         raise Errors.VibChecksumError(vib.id, "VIB checksum does not match:"
                                       " calculated: %s, expected: %s" %
                                       (checksum, vib.checksum.checksum))

class HostImage(object):
   """The HostImage class enables the basic operation of applying
      an image profile to an ESX host via the Stage and Remediate
      methods.  It also provides a way to obtain the current
      VIB inventory and current image profile.

      Class Variables:
         * IMGSTATE_FRESH_BOOT - The host is local booted and VisorFS VIBs
                        are the same as /bootbank VIBs (no VIBs have been
                        updated); or the host is PXE booted
         * IMGSTATE_LIVE_UPDATED - VIBs have been live installed or removed
                        but no reboot-required VIBs have been installed
                        since the last boot.  The VIBs in VisorFS are diff-
                        erent from the /bootbank VIBs, but should be the same
                        as /altbootbank VIBs.
         * IMGSTATE_BOOTBANK_UPDATED - VIBs requiring a reboot have been
                        installed, so that the VIBs in /altbootbank are diff-
                        erent than the VIBs in VisorFS.  Once a host enters
                        this state, live installs are not permitted until
                        after a reboot.

      Attributes:
         * installers    - A dict of supported Installer classes.
         * imgstate      - Current ESX image state, one of the IMGSTATE_* values
         * livevibs      - A VibCollection instance of live VIBs
         * livestagedvibs - A VibCollection instance of VIBs staged for
                            LiveImageInstaller.
         * bootbankupdatedvibs - A VibCollection of VIBs, which are installed
                                 into altbootbank and will be live after reboot.
         * bootbankstagedvibs  - A VibCollection of VIBs, which are in
                                 altbootbank, but altbootbank is in staged
                                 state.
   """
   IMGSTATE_UNKNOWN = -1
   IMGSTATE_FRESH_BOOT = 0
   IMGSTATE_LIVE_UPDATED = 1
   IMGSTATE_BOOTBANK_UPDATED = 2

   DB_AUTO = 0
   DB_VISORFS = 1
   DB_BOOTBANK = 2
   DB_CURBOOTBANK = 3
   DBS = (DB_AUTO, DB_VISORFS, DB_BOOTBANK, DB_CURBOOTBANK)

   # Ramdisk for vib download.
   # GetContainerId() retrieves simulator name to generate unique ramdisk name
   # and path for each simulator environment. It returns an empty string if not
   # in simulator environment.
   VIB_DOWNLOAD_NAME = HostInfo.GetContainerId() + 'vibdownload'
   VIB_DOWNLOAD = os.path.join(os.path.sep, 'tmp', VIB_DOWNLOAD_NAME)

   # Audit events and messages
   AUDIT_START_EVENTID = 'system.update.start'
   AUDIT_END_EVENTID = 'system.update.end'
   AUDIT_NOTE_NOSIG = 'The user has specified to bypass signature verification.'
   AUDIT_NOTE_NOSIG_IGNORED = 'The user has specified to bypass signature ' \
                              'verification, but the request has been ' \
                              'ignored since SecureBoot is enabled.'

   # VIB signature verification result, lower number is more benign.
   SIG_VERIFY_PASSED = 0
   SIG_VERIFY_LOCKER = 1
   SIG_VERIFY_MISSING = 2
   SIG_VERIFY_UNKNOWN = 3
   SIG_VERIFY_ERROR = 4

   def __init__(self, installerclasses=DEFAULT_INSTALLERS,
                initInstallers=True):
      """HostImage class constructor.
         Initialize all the installers.
         Note, on PXE systems the BootbankInstaller and LockerInstaller
         will not initialize and will not be present.
         Parameters:
            * installerclasses - a list or iterable of class objects
                                 inherited from Installer.Installer
            * intInstallers - whether to initiate all installers, use False
                              with a transaction that would call _getLock()
                              at the start.
         TODO: Exceptions:
            SetupError - if no installer classes could be initialized
      """
      self._lock = LockFile.LockFile(LOCKFILE)
      self._hosthw = None
      self._installerClasses = installerclasses
      if initInstallers:
         self._initInstallers()
      else:
         log.debug('Deferring initiating installers')

   def _initInstallers(self):
      """Initiate the installer instances.
      """
      self.installers = dict()

      # in DEFAULT_INSTALLERS tuple, LockerInstaller is arranged
      # next to BootBankInstaller in sequence,  if for some reason
      # the BootBankInstaller is not initiated,  we don't need to
      # initiate LockerInstaller.

      for cls in self._installerClasses:
         try:
            installer = cls()
            self.installers[installer.installertype] = installer
         except Exception as e:
            if hasattr(e, 'msg'):
               log.info('Installer %s was not initiated - reason: %s',
                        str(cls), e.msg)
            else:
               log.info('Installer %s was not initiated - reason: %s',
                        str(cls), str(e))

            # Live installer is supposed to initiate in all configurations
            if cls == LiveImageInstaller.LiveImageInstaller:
               raise Errors.InstallationError(e, None,
                  "Failed to initiate installer: %s" % str(e))

            # Host requires both bootbank and locker installers in stateful
            # env or none of them in stateless env. But Simulators run
            # neither completely stateful nor completely stateless - they
            # require locker partition to allow scan to complete, but do not
            # require all tardisks to be in bootbank and altbootbank.
            if 'BootBankInstaller' in str(cls) and \
               not HostInfo.HostOSIsSimulator():
               break

      log.info('Installers initiated are %s', str(self.installers))

   def _getImgState(self):
      if 'boot' not in self.installers:
         if 'live' not in self.installers:
            return self.IMGSTATE_UNKNOWN
         # Probably PXE
         # XXX: if we support fresh install, there might be a drift
         return self.IMGSTATE_FRESH_BOOT
      elif 'live' not in self.installers:
         # bootbank exists but not live installer, probably offline
         # State depends on whether /altbootbank image has been updated
         bootinst = self.installers['boot']
         if bootinst.bootbankstate & bootinst.BOOTBANK_STATE_UPDATED:
            return self.IMGSTATE_BOOTBANK_UPDATED
         else:
            return self.IMGSTATE_FRESH_BOOT
      else:
         # Both bootbank and live images exist.
         bootinst = self.installers['boot']
         liveinst = self.installers['live']
         #
         # Altbootbank contains a valid, newer image than bootbank.
         # If altbootbank image is the same as the live image, then
         # the live image has been updated as well. If they are different,
         # then altbootbank is newer than the live image.  It is not possible
         # for the live image to be newer than /altbootbank once /altbootbank
         # is updated, we prevent that scenario.
         if bootinst.bootbankstate & bootinst.BOOTBANK_STATE_UPDATED:
            if liveinst.database.vibIDs == bootinst.database.vibIDs:
               return self.IMGSTATE_LIVE_UPDATED
            else:
               return self.IMGSTATE_BOOTBANK_UPDATED
         # Altbootbank contains an older image than /bootbank.
         # Most likely this is a fresh boot, so the live image is the same as
         # the bootbank image.
         # It is possible that a live install happened, but for some reason the
         # altbootbank was not updated.  In this case, we still return live
         # updated, but if a user reboots, they will lose their live image changes.
         elif bootinst.database.vibIDs == liveinst.database.vibIDs:
            return self.IMGSTATE_FRESH_BOOT
         else:
            log.info('Live image has been updated but /altbootbank image has '
                     'not. A reboot will disgard live changes.')
            # Logging VIB diff of the databases to help debugging
            bootinstunique = list(bootinst.database.vibIDs -
                                  liveinst.database.vibIDs)
            liveinstunique = list(liveinst.database.vibIDs -
                                  bootinst.database.vibIDs)
            log.debug('Live image has unique VIB %s, bootbank has unique VIB '
                      '%s' % (liveinstunique, bootinstunique))
            return self.IMGSTATE_LIVE_UPDATED

   imgstate = property(_getImgState)

   def _getDatabase(self, database = DB_AUTO):
      if database not in self.DBS:
         raise ValueError("Database value of %s not valid" % (database))

      #
      # Pick database automatically.  Pick from bootbank database if
      # bootbank-only remediation has occured; otherwise from live database
      #
      if database == self.DB_AUTO:
         if self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED:
            database = self.DB_BOOTBANK
         elif 'live' in self.installers:
            database = self.DB_VISORFS
         else:
            database = self.DB_CURBOOTBANK

      if database == self.DB_BOOTBANK and 'boot' in self.installers:
         return self.installers['boot'].database
      elif database == self.DB_VISORFS and 'live' in self.installers:
         return self.installers['live'].database
      elif database == self.DB_CURBOOTBANK and 'boot' in self.installers:
         return self.installers['boot'].bootbank.db
      else:
         # When the intended DB is unavailable, we may have ran into an issue.
         # Stopping is not necessary for all transactions, here we return an
         # empty database without creating it in visorfs.
         log.warn('Intended database location %d is not available, database '
                  'read issue suspected. Current host image state is %d.'
                  % (database, self.imgstate))
         visordbdir = os.path.join('/', LiveImageInstaller.LiveImage.DB_DIR)
         return Database.Database(visordbdir, dbcreate=False)

   def _verifyVibSignatureAndPayloads(self, vib, imgprofile, skiplive=False,
                                      rebootPending=False):
      """Verifies VIB signature and checksums on the payloads of a given vib.
         The checksums in the vib metadata are compared with the checksums
         of the modules actually installed.
         Caller of this method needs to wrap with _getLock() and _freeLock()
         when verifying VIBs of interest.
         Parameters:
            * vib        - The vib whose payloads need to be verified
            * imgprofile - The image profile to use
            * skiplive   - Skip payload verification for live image
            * rebootPending - Whether the system is pending reboot from a
                              bootbank-only change.

         Exceptions:
            A sub-type of VibSignatureError - Error in verifying VIB signature
            ChecksumVerificationError - Error in verifying the checksums
            InstallationError - Error in the installer
      """
      vib.VerifyAcceptanceLevel()
      for payload in vib.payloads:
         # if the payload is not in this image profile, it is assumed
         # to be a locker vib
         if payload.name in imgprofile.vibstates[vib.id].payloads:
            payload.localname = \
               imgprofile.vibstates[vib.id].payloads[payload.name]
            for installer in self._GetOrderedInstallers():
               installertype = installer.installertype
               if installertype == 'live' or installertype == 'boot':
                  if not skiplive or installertype != 'live':
                     installer.VerifyPayloadChecksum(
                        vib.id, payload, rebootPending=rebootPending)
               else:
                  # Skip the locker installer
                  assert installertype == 'locker'

   def GetComponents(self, database=DB_AUTO):
      """Obtains the collection of components installed on the ESX host.
         If the host is PXE-booted or in the 'Fresh Boot' and
         'Live Image Updated' states, then the bulletin inventory
         will be taken from the VisorFS database. If the host
         is in the 'Boot Image Updated' state, then the VIB
         inventory is taken from the /altbootbank database. Components from
         locker will be added to the return value.

         Parameters:
            * database - Override the automatic database selection algorithm
                         and read the inventory from a specific database.
                         Should be one of the DB_* class variables.

         Returns:
            A ComponentCollection instance, or None if no database was found

         Exceptions:
            ValueError - Illegal database value passed
            DatabaseFormatError - Database files not formatted correctly
            DatabaseIOError - Database could not be read
            ComponentFormatError - One or more files were not a valid Bulletin
                                   xml.
      """
      allbulletins = Bulletin.BulletinCollection()
      allbulletins += self._getDatabase(database).bulletins
      # Locker components are only kept in locker installer
      if 'locker' in self.installers:
         allbulletins += self.installers['locker'].database.bulletins
      return Bulletin.ComponentCollection(allbulletins, True)

   def GetInventory(self, database = DB_AUTO):
      """Obtains the collection of VIBs installed on the ESX host.
         If the host is PXE-booted or in the 'Fresh Boot' and
         'Live Image Updated' states, then the VIB inventory
         will be taken from the VisorFS database.  If the host
         is in the 'Boot Image Updated' state, then the VIB
         inventory is taken from the /altbootbank database. If database is
         DB_AUTO, VIBs from locker will be added to the return value.

         Parameters:
            * database - Override the automatic database selection algorithm
                         and read the inventory from a specific database.
                         Should be one of the DB_* class variables.

         Returns:
            A VibCollection instance, or None if no database was found

         Exceptions:
            ValueError - Illegal database value passed
            DatabaseFormatError - Database files not formatted correctly
            DatabaseIOError - Database could not be read
      """
      allvibs = VibCollection.VibCollection()
      allvibs += self._getDatabase(database).vibs
      # Locker VIBs are only kept in locker installer
      if 'locker' in self.installers:
         allvibs += self.installers['locker'].database.vibs
      return allvibs

   def GetProfile(self, database = DB_AUTO):
      """Obtains the last image profile applied to the ESX host.
         If the host is PXE-booted or in the 'Fresh Boot' and
         'Live Image Updated' states, then the image profile
         will be taken from the VisorFS database.  If the host
         is in the 'Boot Image Updated' state, then the image
         profile is taken from the /altbootbank database.
         Locker VIBs are added back to image profile because they are saved
         separately in product locker.

         Parameters:
            * database - Override the automatic database selection algorithm
                         and read the profile from a specific database.
                         Should be one of the DB_* class variables.

         Returns:
            An ImageProfile instance, or None if no image profile
            is in the database.

         Exceptions:
            ValueError  - Illegal database value passed
            DatabaseFormatError - Database files not formatted correctly
            DatabaseIOError     - Database could not be read
      """
      db = self._getDatabase(database)
      profile = None
      if db.profile:
         profile = db.profile.Copy()
         profile.creationtime = db.profile.creationtime
         profile._modifiedtime = db.profile._modifiedtime

      # Adding back locker VIBs and components as live/boot installer
      # doesn't have locker database contents.
      # Modify the profile._modifiedtime to the greater of
      # db.profile._modifiedtime or the latest of the vib installdate
      # in the locker database.
      if profile and 'locker' in self.installers:
         lockerDb = self.installers['locker'].database
         lockerComps = Bulletin.ComponentCollection(lockerDb.bulletins, True)
         prevprofilemodifiedtime = profile._modifiedtime
         for vib in lockerDb.vibs.values():
            profile.AddVib(vib)
            if prevprofilemodifiedtime < vib.installdate:
               prevprofilemodifiedtime = vib.installdate
         for comp in lockerComps.IterComponents():
            try:
               profile.AddComponent(comp)
            except KeyError as e:
               # After an upgrade from 7.0 pre-U1, if the bootbank image
               # database was last written by legacy code, e.g. when another
               # esxcli command was called after upgrade or vib command was used
               # for the upgrade, we will find that the locker component is
               # already in the image profile.
               log.warning('Cannot add locker component: %s. The database was '
                           'likely created by legacy code. Another image apply '
                           'should resolve this warning.', e)

         profile._modifiedtime = prevprofilemodifiedtime

      # Update acceptance level to reflect system config.
      # Always retrieve the acceptance level each time since it may have
      # changed in the background.
      hostAcl = self.GetHostAcceptance()
      if profile and hostAcl in Vib.ArFileVib.ACCEPTANCE_LEVELS:
         profile.acceptancelevel = hostAcl

      return profile

   @property
   def livevibs(self):
      vibs = VibCollection.VibCollection()
      if 'live' in self.installers:
         vibs.update(self.GetInventory(self.DB_VISORFS))
      return vibs

   @property
   def livestagedvibs(self):
      vibs = VibCollection.VibCollection()
      if 'live' in self.installers:
         livevibids = set(self.livevibs.keys())
         db = self.installers['live'].stagedatabase
         if db is not None:
            for k, v in db.vibs.items():
               if k not in livevibids:
                  vibs.AddVib(v)
      return vibs

   @property
   def bootbankupdatedvibs(self):
      vibs = VibCollection.VibCollection()
      if self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED:
         # Altbootbank has the latest ImageProfile, GetInventory will return
         # vibs from latest DB by default.
         vibs.update(self.GetInventory())
      return vibs

   @property
   def bootbankstagedvibs(self):
      vibs = VibCollection.VibCollection()
      if 'boot' in  self.installers:
         db = self.installers['boot'].stagedatabase
         if db is not None:
            vibs.update(db.vibs)
      return vibs

   def GetHostHwPlatform(self):
      if self._hosthw is None:
         self._hosthw = list()
         vendor, model = HostInfo.GetBiosVendorModel()
         self._hosthw.append(Vib.HwPlatform(vendor, model))
         # translate OEM Strings into vendor contraints
         # PR1086517
         for vendor in HostInfo.GetBiosOEMStrings():
            self._hosthw.append(Vib.HwPlatform(vendor, model=''))
      return self._hosthw

   def GetHostAcceptance(self):
      """Returns the host acceptance level setting.  If there is no setting
         or if the configured value is invalid, then a default acceptance
         level will be returned.
         Parameters:
            None
         Returns:
            One of the Vib.ArFileVib.ACCEPTANCE_LEVELS settings.
         Raises:
            ValueError -- Illegal value retrieved
            Errors.AcceptanceGetError -- An error occured while fetching
                                         acceptance level
      """
      # Config store is not running in a simulator, fall back to read from
      # esx.conf instead.
      if not HostInfo.HostOSIsSimulator():
         def getAcceptanceFromImageProfile():
            """Get the acceptance level kept in the image profile.
            """
            # Read value from raw profile instead of using GetProfile
            # as GetProfile calls GetHostAcceptance and a recursion
            # happens.
            db = self._getDatabase()
            if not db.profile:
               log.error("No image profile is present to provide acceptance " \
                         "level.")
               return ""

            return db.profile.acceptancelevel

         # The integration of acceptance level with config store has been
         # enabled. In this case, the code tries to get the acceptance level
         # from config store, falling back to the image profile value if the
         # value is unset in the config store.
         # The config store/image profile approach is taken only if:
         # - config store library is present on the system.
         # - config store schema for the 'acceptance_level' config is present
         #   on the system.
         #
         # Otherwise, use the esx.conf approach in order to be compatible with
         # legacy ESXi during an upgrade.
         if HAVE_CS:
            # If this check passes, assume that this is a 7.0 or later ESXi and
            # attempt to get value of config from config store.
            try:
               configStoreLogInit()
               csoi = cs.ConfigStoreObjectId(*ACCEPTANCE_LEVEL_CS_OBJ_ID)
               store = cs.ConfigStore.GetStore()
               # Assume that the schema is present. An exception would be raised
               # otherwise.
               cso = store.Get(csoi)
               if cso:
                  return Vib.ArFileVib.FROM_CS_ACCEPTANCE[cso.GetValue("level")]
               else:
                  # No entry for the config was found in the store. Fall back
                  # to return the value held in the image profile.
                  return getAcceptanceFromImageProfile()
            except cs.ConfigStoreException as e:
               # Check whether the exception was raised because config store
               # could not connect to the database. In case it is, fall back
               # to the acceptance level in the image profile.
               #
               # PR 2678680
               # This is a workaround, at the time of the early boot secure boot
               # VIB acceptance check, the config store database does not exist
               # yet.
               if "Failed to connect to database" in str(e):
                  # TODO: Check the errorCode after one is added for the error.
                  return getAcceptanceFromImageProfile()

               # Check whether the exception is because the schema is missing.
               # If that's the case fall back to the old esx.conf approach
               # instead of raising an exception.
               if e.customExp.errorCode != 1037 and \
                  "Schema not found for object:" not in str(e):
                  msg = "Failed to get 'acceptance_level' from config " \
                        "store: %s" % str(e)
                  log.error(msg)
                  raise Errors.AcceptanceGetError(msg)
            except KeyError as e:
               msg = "Failed to get 'acceptance_level' from config store: " \
                     "missing value of %s." % str(e)
               log.error(msg)
               raise Errors.AcceptanceGetError(msg)

      return self._getHostAcceptance()

   def SetHostAcceptance(self, newlevel):
      """Sets the host acceptance level to a new level.  Validates that
         the new level still works with the current image profile, ie that no
         VIBs in the current image has a lower level than the new level.
         To prevent the acceptance level from changing while an image is
         being updated, locking will be used.
         Parameters:
            * newlevel - the new acceptance level, should be one of the
                         constants Vib.ArFileVib.ACCEPTANCE_LEVELS
         Returns:
            None
         Raises:
            LockError   - Image is being updated, it's not safe to change the
                          level
            AcceptanceConfigError - new level invalidates the image profile, or
                                    unable to run esxcfg-advcfg
            ValueError  - "newlevel" is not a valid acceptance level
      """
      if newlevel not in Vib.ArFileVib.ACCEPTANCE_LEVELS:
         raise ValueError("Invalid acceptance level of '%s' was passed in" \
                          % (newlevel))

      if newlevel == Vib.ArFileVib.ACCEPTANCE_COMMUNITY and \
         HostInfo.IsHostSecureBooted():
         log.info("Secure Boot enabled: Installation of %s "
                  "VIBs is not allowed." % (Vib.ArFileVib.ACCEPTANCE_COMMUNITY))
         msg = "Secure Boot enabled: Cannot change acceptance level to %s."\
               % (newlevel)
         raise Errors.AcceptanceConfigError(msg)

      self._getLock()
      try:
         prof = self.GetProfile()
         oldlevel = prof.acceptancelevel
         log.info("Attempting to change the host acceptance level from %s to %s"
                  % (oldlevel, newlevel))
         prof.acceptancelevel = newlevel
         prof.vibs = self.GetInventory()

         # Focus on only acceptance level problems
         # TODO: Disable extensibility rule checks
         problems = prof.Validate(nodeps=True, noconflicts=True,
                                  allowobsoletes=True,
                                  allowfileconflicts=True)
         if problems:
            badvibs = [getattr(p, 'vibid', '') for p in problems]
            msg = "Unable to set acceptance level of %s due to installed " \
                  "VIBs %s having a lower acceptance level." \
                  % (newlevel, ', '.join(badvibs))
            raise Errors.AcceptanceConfigError(msg)

         if not HostInfo.HostOSIsSimulator():
            # Assume that if the FSS is enabled we are goot to write directly
            # into config store and treat its absense as an error.
            if not HAVE_CS:
               msg = "Unable to set acceptance level: config store is not " \
                     "present."
               log.error(msg)
               raise Errors.AcceptanceConfigError(msg)

            try:
               csLevel = Vib.ArFileVib.TO_CS_ACCEPTANCE[newlevel]
               configStoreLogInit()
               csoi = cs.ConfigStoreObjectId(*ACCEPTANCE_LEVEL_CS_OBJ_ID)
               cso = cs.ConfigStoreObject(csoi)
               cso.SetValue("level", csLevel)
               store = cs.ConfigStore.GetStore()
               store.SetUserData(cso)

               # Update installer's image profile to reflect the newly set
               # acceptance level.
               #
               # PR 2678680
               # This is needed to avoid the lag in image profile's acceptance
               # level compared to the one in the config store. Secure boot VIB
               # acceptance check requires image profile to lock-step with the
               # config store value.
               #
               # ? Can we reuse the profile above
               profile = self.GetProfile()
               profile.acceptancelevel = newlevel
               for installer in self._GetOrderedInstallers():
                  installer.UpdateImageProfile(profile)
            except (cs.ConfigStoreException, KeyError) as e:
               msg = ("Failed to set acceptance level '%s' in config " \
                      "store: %s" % (csLevel, str(e)))
               log.error(msg)
               raise Errors.AcceptanceConfigError(msg)
            except Exception as e:
               msg = ("Failed to set acceptance level '%s': %s" %
                      (csLevel, str(e)))
               log.error(msg)
               raise Errors.AcceptanceConfigError(msg)
         else:
            # Config store is not running in a simulator, fall back to read
            # from esx.conf instead.
            self._setHostAcceptance(newlevel)

         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.SendVob("hostacceptance.changed", oldlevel, newlevel)
      finally:
         self._freeLock()

   def _GetOrderedInstallers(self):
      return sorted(self.installers.values(), key=attrgetter('priority'))

   def _verify_and_write_payload(self, installer, vib, payload, sourcefp):
      """Write payload to installer staging area and verify checksum."""
      dfp = installer.OpenPayloadFile(vib.id, payload, write=True, read=False)
      if not dfp:
         log.debug('Payload %s from vib %s skipped by %s' %
                   (payload.name, vib.id, installer.__class__.__name__))
         return
      # LiveImageInstaller requires decompressed payloads (tardisks)
      decompress = (installer.installertype == 'live')
      try:
         # If the installer is locker type, we need to save the payload
         # filelist mapping as well.
         if installer.installertype == 'locker':
            installer._SavePayloadFileList(payload, vib, sourcefp)
         Vib.copyPayloadFileObj(payload, sourcefp, dfp, decompress=decompress,
                                checkdigest=True, checksize=True)
      finally:
         dfp.close()

   def _stage_from_deploy_dir(self, imgprofile, adds, installer, deploydir):
      """Stage payloads of VIBs from deploy format folder to installer."""
      for vibid in adds:
         vib = imgprofile.vibs[vibid]

         # Validate vib signature and schema
         vib.VerifyAcceptanceLevel()

         # Update installer database
         try:
            # Update original descriptor and signature of the VIB. Locker
            # installer does not support signature verification, however we
            # need to preserve them for host depot seeding.
            installer.UpdateVibDatabase(vib)
         except Exception as e:
            raise Errors.InstallationError(e, [vibid], str(e))

         # Only support payloads for bootbank or locker installer
         SUPPORTED_PAYLOADS = set(list(BootBankInstaller.BootBankInstaller. \
                                       SUPPORTED_PAYLOADS) + \
                                  list(LockerInstaller.LockerInstaller. \
                                       SUPPORTED_PAYLOADS))

         for payload in vib.payloads:
            if not payload.localname:
               raise Errors.InstallationError(None,
                                              [vibid],
                                              'No local name available for '
                                              'payload %s' % payload.name)

            # Skip unsupported payloads
            if not payload.payloadtype in SUPPORTED_PAYLOADS:
               continue

            sourcepath = self.FindPayloadInDeployDir(vib, payload, deploydir)
            if not sourcepath:
               raise Errors.InstallationError(None, [vibid],
                  'Failed to locate payload %s filename %s in directory %s'
                  % (payload.name, payload.localname, deploydir))

            # Update localname to reflect final name on bootbank.
            payload.localname = \
               imgprofile.vibstates[vibid].payloads[payload.name]

            with open(sourcepath, 'rb') as sourcefp:
               self._verify_and_write_payload(installer, vib, payload, sourcefp)

         # Add install date for newly installed vib
         if vib.installdate is None:
            vib.installdate = datetime.datetime.now(_utctzinfo)

   def _download_and_stage(self, imgprofile, vibid, installer,
                           checkacceptance = True, schema = None):
      """Download a vib, verify and stage the payloads."""
      arvib = src = None
      vib = imgprofile.vibs[vibid]
      log.info('Attempting to download VIB %s', vib.name)
      download_path = self.VIB_DOWNLOAD + '/' + vib.id + '.vib'
      for url in vib.remotelocations:
         try:
            d = Downloader.Downloader(url, download_path)
            actual_path = d.Get()
            src = open(actual_path, 'rb')
            arvib = Vib.ArFileVib.FromFile(src, schema)
            break
         except Exception as e:
            if src:
               src.close()
               src = None
            log.info('Unable to download from %s, error [%s]. Trying next '
                     'url...', url, str(e))
            # remove downloaded data if there is any
            if os.path.isfile(download_path):
               os.unlink(download_path)
            continue
      if not arvib:
         urls = ', '.join(vib.remotelocations)
         raise Errors.VibDownloadError(urls, download_path,
                  "Unable to download VIB from any of the URLs %s" % urls)

      # Verify complete VIB checksum.
      verifyVibChecksum(vib, actual_path)

      # Verify descriptor matches metadata:
      try:
         arvib.MergeVib(vib)
         if checkacceptance:
            arvib.VerifyAcceptanceLevel()
      except Exception as e:
         src.close()
         raise Errors.InstallationError(e, [arvib.id], str(e))

      # The vib metadata maintained by the installer does not contain
      # information that is not part of the image profile metadata.
      # New information like the original vib descriptor and the signature
      # information must be updated in that database.
      try:
         # Update original descriptor and signature of the VIB. Locker
         # installer does not support signature verification, however we
         # need to preserve them for host depot seeding.
         installer.UpdateVibDatabase(arvib)
      except Exception as e:
         raise Errors.InstallationError(e, [arvib.id], str(e))

      # TODO:
      #   - verify hardware platform

      try:
         # In the locker parition, there could be extra files like vSAN traces.
         # Make sure the parition has enough free space to install the VIB.
         # This check has to be late because we need the arctual VIB file to
         # calculate space usage. As a result, old locker VIB has already been
         # removed. User has to retry the transaction, or install the old
         # VIB again after clean up the partition.
         if installer.installertype == 'locker':
            installer._CheckDiskSpaceForVib(arvib)

         # Loop through each payload, open dest payload file, and start buffer
         # copy.  Compute digest.
         for payload, sourcefp in arvib.IterPayloads():
            if payload.name in imgprofile.vibstates[vibid].payloads:
               vibstate = imgprofile.vibstates[vibid]
               payload.localname = vibstate.payloads[payload.name]
            else:
               payload.localname = None
            self._verify_and_write_payload(installer, arvib, payload, sourcefp)
      except Exception as e:
         raise Errors.InstallationError(e, [arvib.id], str(e))
      finally:
         src.close()

      # Explicitly check for installdate is None. If installdate is already
      # set, we do not want to reset it.
      if vib.installdate is None:
         vib.installdate = datetime.datetime.now(_utctzinfo)

      # If the vib was downloaded from HTTP, remove it
      if os.path.isfile(download_path):
         os.unlink(download_path)

   def _get_tardisk_payload(self, imgprofile, vibid, vib):
      """Stage compressed payloads of a VIB for stagebootbank from uncompressed
         tardisks in /tardisks.
      """
      vibstate = imgprofile.vibstates[vibid]
      for pl in vib.payloads:
         if pl.payloadtype not in BootBankInstaller.\
               BootBankInstaller.SUPPORTED_PAYLOADS:
            continue

         plname = pl.name
         srcfile = None
         if plname in vibstate.payloads:
            srcname = vibstate.payloads[plname]
            srcfile = os.path.join('/tardisks', srcname)
            destfile = os.path.join(BootBankInstaller.\
                                   BootBankInstaller.STAGEBOOTBANK, srcname)

         if srcfile is not None and os.path.isfile(srcfile):
            log.info('Compressing %s to %s' % (srcfile, destfile))
            with open(srcfile, 'rb') as srcfObj:
               with open(destfile, 'wb') as destfObj:
                  Vib.copyPayloadFileObj(pl, srcfObj, destfObj, compress=True)
         else:
            msg = 'Cannot locate source for payload %s of '\
                  'VIB %s ' % (plname, vibid)
            if HostInfo.HostOSIsSimulator():
               log.info('HostSimulator: %s' % (msg))
            else:
               raise Errors.InstallationError(None, [vibid], msg)

   def _stage_with_url(self, imgprofile, adds, installer, checkacceptance,
                       vibschema):
      """Stage VIBs with URLs in vib/profile transaction"""
      # Use the largest VIB size to create download ramdisk
      maxVibSize = 0
      for vibid in adds:
         vib = imgprofile.vibs[vibid]
         vibPackedSize = 0
         for payload in vib.payloads:
            vibPackedSize += payload.size
         vibPackedSize = int(vibPackedSize / 1024 / 1024) + 1
         if vibPackedSize > maxVibSize:
            maxVibSize = vibPackedSize
      # Add a buffer for descriptor and signature
      maxVibSize += 5

      Downloader.Downloader.setEsxupdateFirewallRule('true')
      Ramdisk.CreateRamdisk(maxVibSize, self.VIB_DOWNLOAD_NAME,
                            self.VIB_DOWNLOAD, reserveSize=maxVibSize)
      try:
         for vibid in adds:
            vib = imgprofile.vibs[vibid]

            if not vib.remotelocations:
               # for vib installed after the host boots up,
               # get vib payloads from /tardisks.
               # This is only applicable to BootBankInstaller since
               # for LiveImageInstaller and LockerInstaller
               # new vibs always have vib.remotelocations.
               log.info("Constructing payloads from tardisks for vib %s"
                        % vibid)
               self._get_tardisk_payload(imgprofile, vibid, vib)
            else:
               self._download_and_stage(imgprofile, vibid, installer,
                                        checkacceptance, vibschema)
      finally:
         Downloader.Downloader.setEsxupdateFirewallRule('false')
         Ramdisk.RemoveRamdisk(self.VIB_DOWNLOAD_NAME, self.VIB_DOWNLOAD)

   def Stage(self, imgprofile, nosigcheck = False,
                   forcebootbank = False, dryrun = False,
                   vibschema = None, stageonly = False, checkacceptance = True,
                   deploydir=None):
      """Stage prepares or 'stages' a new ESX image for application
         based on the specified image profile.  If the current image
         or last staged image is composed of the same VIBs as the
         passed in image profile, then the Stage method will have
         no work to do.
         Maintenance Mode is not required during a stage operation.
         Stage also checks for any hwplatform dependencies.

         The VIBs are downloaded and a new ESXi image is prepared.  The
         StartTransaction, OpenPayloadFile, and CompleteStage methods of
         *Installer subclasses actually carry out the work.  This Stage method
         is responsible for coordinating between different Installers,
         VIB signature validation, and driving each Installer to install
         successive VIB payloads.  The databases will be locked to prevent
         another process from calling Stage() at the same time.

         Stage manages the sourcing of VIBs, either initiating downloads or
         sourcing them from previous Installers for chained installation.

         TBD: Cycling may not be practical.
         We want to skip LiveImageInstaller once the image state becomes
         BOOTBANK_UPDATED or forcebootbank is True.

         Parameters:
            * imgprofile   - The ImageProfile instance to stage. It must have
                             a populated vibs attribute.
            * nosigcheck   - Boolean, skips signature validation if True
            * forcebootbank - Force an install using BootbankInstaller even
                              if the LiveImageInstaller could be used
            * dryrun       - Dry run only, report what will be changed
                             but do not make changes to the system.
            * vibschema    - A file path to an XML schema for validation
                             of the VIB descriptor. Set to None to disable
                             validation.
            * checkacceptance - If True (the default), VIB acceptance levels
                                 will be validated. A failed validation raises
                                 an exception.
            * deploydir    - Directory which contain image in deploy format,
                             such as ISO and PXE. When supplied, database and
                             payloads will be fetched from the directory.

         Returns:
            None

         Exceptions:
            InstallationError
            LockingError      - Unable to lock database for writing. Most
                                likely another thread has invoked Stage().
            VibDownloadError  - error during download of a VIB
            VibFormatError    - improper VIB or descriptor structure
            VibSignatureError - invalid VIB signature
            DatabaseFormatError - Database files not formatted correctly
            DatabaseIOError     - Database could not be read
            VibHwPlatformError - VIB hwplatform dependency could not
                                  be satisfied
      """
      assert len(imgprofile.vibs) >= len(imgprofile.vibIDs)
      log.debug('Staging image profile [%s]' % (imgprofile.name))
      log.debug('VIBs in image profile: %s' % (', '.join(imgprofile.vibIDs)))

      # Check potential installation problems before hand
      #   - current bootbank bootstate must be 0
      #   - installation size must fit in bootbank partition
      if 'boot' in self.installers:
         self.installers['boot'].PreInstCheck(imgprofile)

      # hwplatform requirement check
      hwmatchproblems = list()
      hosthws = self.GetHostHwPlatform()
      for imghw in imgprofile.GetHwPlatforms():
         for hw in hosthws:
            prob = imghw.MatchProblem(hw)
            if prob is None:
               # We have a match, forget any other mismatches
               hwmatchproblems = list()
               break
            else:
               hwmatchproblems.append(prob)
         else:
            continue
         break # if we break out of the inner loop, break out of the outer too

      if len(hwmatchproblems) > 0:
         reasons = ["BIOS %s %s is required but host BIOS %s is %s" \
                    % (p[0], p[1], p[0], p[2]) for p in hwmatchproblems]
         msg = "Host doesn't meet image profile '%s' hardware requirements:" \
               "\n%s" % (imgprofile.name, '\n'.join(reasons))
         log.error(msg)
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.SendVob("install.invalidhardware", imgprofile.name, \
                         '\n'.join(reasons))
         raise Errors.VibHwPlatformError(msg)

      # Send VOB warning if installing image profile with nosigcheck or
      # checkacceptance is False
      if nosigcheck or not checkacceptance:
         reasons = list()
         if nosigcheck:
            reasons.append("signature validation disabled")
         if not checkacceptance:
            reasons.append("acceptance level checking disabled")
         msg = "SECURITY ALERT: Installing image profile '%s' with %s." \
               % (imgprofile.name, " and ".join(reasons))
         log.warn(msg)
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.SendVob("install.securityalert", imgprofile.name, \
                         " and ".join(reasons))

      try:
         #
         # Cycle through each installer. Installers are ordered by its
         # priority. Current design is to run live installer first
         #
         installerstate = {'unsupported' : list(),
                           'nochange'    : list(),
                           'finished'    : list()}
         for installer in self._GetOrderedInstallers():
            # Skip bootbank stage if live staged
            installertype = installer.installertype
            if installertype == 'boot' and \
                  'live' in installerstate['finished']:
               log.debug("Already staged to live image, skipping "
                         "BootbankInstaller.")
               installerstate['finished'].append(installertype)
               continue

            try:
               adds, removes, staged = installer.StartTransaction(imgprofile,
                                 imgstate = self.imgstate,
                                 forcebootbank = forcebootbank,
                                 stageonly = stageonly,
                                 preparedest = not dryrun)
            except EnvironmentError as e:
               raise Errors.InstallationError(e, None,
                                 "Failed to start installation: %s" % str(e))

            # (None, None, False) is returned if installer could not handle the
            # transaction
            if adds is None and removes is None:
               log.info('%s is not supported, skipping.' %
                        (installer.__class__.__name__))
               installerstate['unsupported'].append(installertype)
               continue

            if not adds and not removes:
               log.info('Nothing for %s to do, skipping.' %
                        (installer.__class__.__name__))
               installerstate['nochange'].append(installertype)
               continue

            log.debug(' --- Stage: %s adding [%s], removing [%s]' % (
                      installer.__class__.__name__,
                      ', '.join(adds), ', '.join(removes)))

            if not (staged or dryrun):
               if deploydir:
                  # for upgrade using deploy format
                  self._stage_from_deploy_dir(imgprofile, adds, installer,
                                              deploydir)
               else:
                  # for tranasction with vib/profile url
                  self._stage_with_url(imgprofile, adds, installer,
                                       checkacceptance, vibschema)

               installer.CompleteStage()

            installerstate['finished'].append(installertype)

         if len(installerstate['unsupported']) == len(self.installers):
            msgs = list()
            for installer in self.installers.values():
               for problem in installer.problems:
                  msgs.append(str(problem))
            if 'boot' not in self.installers:
               raise Errors.StatelessError(
                  'The transaction is not supported: %s' % (' '.join(msgs)))
            else:
               raise Errors.InstallationError(None, None,
                  'The transaction is not supported: %s' % (' '.join(msgs)))

         if dryrun:
            installers = [INSTALLERTYPE_TO_NAME[inst] for inst in
                  installerstate['finished']]
            msg = ('Dryrun only, host not changed. '
                   'The following installers will be applied: [%s]' %
                   (', '.join(installers)))
            # If only boot bank installer is applied, then
            # live image is not changed - reboot is required
            if 'boot' in installerstate['finished'] and \
               'live' not in installerstate['finished']:
               raise Errors.NeedsRebootResult(msg)
            else:
               raise Errors.NormalExit(msg)
      except Errors.NormalExit:
         raise
      except Exception as e:
         msg = "Could not stage image profile '%s': %s" % (imgprofile.name, e)
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.SendVob("install.stage.error", imgprofile.name, e)
         # cleanup installers
         for installer in self._GetOrderedInstallers():
            # locker install is done in-place, should not cleanup
            if installer.installertype in ['live', 'boot']:
               installer.Cleanup()
         raise

   def Remediate(self, imgprofile, checkmaintmode=True,
                 hasConfigDowngrade=False):
      """Remediate applies a previously staged image.  The exact
         behavior depends on the *Installer subclasses carrying out
         the work. Maintenance Mode is checked if any VIBs require it.
         The databases will be locked for writing; therefore only one
         process may invoke Remediate() at a time.
         Stage must be called before Remediate().
         If not, Remediate may not do anything.

         Parameters:
            * imgprofile     - The ImageProfile that has been staged.
            * checkmaintmode - Boolean, check that the ESX host is in
                               maintenance mode if the VIBs require it.
                               Note that for bootbank-only installs,
                               Remediate is a NOP, so maintenance mode is
                               not required.
            * hasConfigDowngrade - Boolean, set when the transaction contains
                                   downgrade of config, which will affect
                                   behavior of the installers.
         Returns:
            None

         Exceptions:
            InstallationError
            LockingError      - Unable to lock database for writing. Most
                                likely another thread has invoked Remediate().
            MaintenanceModeError - Maintenance mode was required but the host
                                is not in maintenance mode.
            HostNotChanged    - If host is not changed.
            NeedsRebootResult - Remediation is bootbank only install, reboot is
                                required.
      """
      finishedinstallers = set()
      try:
         for installer in self._GetOrderedInstallers():
            try:
               if 'live' in finishedinstallers \
                     and installer.installertype == 'boot':
                  installer.CacheNewImage(self.installers['live'])
               else:
                  installer.Remediate(checkmaintmode=checkmaintmode,
                                      hasConfigDowngrade=hasConfigDowngrade)
            except Errors.HostNotChanged as e:
               continue
            except Exception as e:
               log.debug('installer %s failed: %s. Clean up the installation.',
                  installer.__class__.__name__, e)
               raise

            finishedinstallers.add(installer.installertype)
      except Errors.NormalExit:
         raise
      except Exception as e:
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.SendVob("install.error", e)
         raise
      finally:
         # no more action for installers, perform cleanup
         for installer in self._GetOrderedInstallers():
            # locker install is done in-place, should not cleanup
            if installer.installertype in ['live', 'boot']:
               installer.Cleanup()

      log.debug('Host is remediated by installer: %s' % (
         ', '.join(finishedinstallers)))

      # The image profile in the live/bootbank/locker installers may need to be
      # updated even if there is no VIB change in the respective installer.
      # 1) In a locker/bootbank installers, update description and component
      #    metadata.
      # 2) In a transaction with no VIB change at all, base image, addon and
      #    component metadata may have changed.
      for installer in self._GetOrderedInstallers():
         # VIB change in installer, no need to check metadata update.
         if installer.installertype in finishedinstallers:
            continue

         # If the host is already pending reboot, modify altbootbank and locker
         # database only. Otherwise both current bootbank and visorfs databases
         # will be updated.
         if (installer.installertype == 'live'
             and self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED):
            continue

         log.debug('Check for image profile metadata update in installer '
                   '%s', installer.installertype)
         installer.UpdateImageProfile(imgprofile)

      if len(finishedinstallers) == 0:
         if self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED:
            # return NeedsRebootResult when host is pending reboot
            raise Errors.NeedsRebootResult('Host is not changed. '
                                'Reboot is pending from previous transaction.')
         else:
            raise Errors.HostNotChanged('Host is not changed.')
      elif 'live' not in finishedinstallers \
            and 'boot' in finishedinstallers:
         raise Errors.NeedsRebootResult()
      elif 'live' in finishedinstallers \
            and 'boot' not in finishedinstallers:
         raise Errors.LiveInstallOnlyResult()

   def GetVibPayloadMap(self, tardiskOnly=False):
      """Returns a map of installed VIB ID to local file names of payloads in
         the VIB. When tardiskOnly is set, only VisorFS mountable payloads are
         returned.
      """
      vibPayloads, payloadMap = dict(), dict()

      # Load signed VIB descriptor rather than the regular descriptor to get
      # names of known VIB payloads to cross-check with the image profile.
      imgProfile = self.GetProfile()
      for vib in imgProfile.vibs.values():
         vibPayloads[vib.id] = set()
         origDesc = vib.GetOrigDescriptor()
         if not origDesc:
            log.warning('VIB %s does not have signed XML data', vib.id)
            continue
         origVib = Vib.ArFileVib.FromXml(origDesc)
         for p in origVib.payloads:
            if not tardiskOnly or p.payloadtype in p.TARDISK_TYPES:
               vibPayloads[origVib.id].add(p.name)

      for vibId, vibState in imgProfile.vibstates.items():
         payloadMap[vibId] = list()
         for name, localName in vibState.payloads.items():
            if name in vibPayloads.get(vibId, set()):
               # A payload file name is returned only when the VIB mentions
               # the payload.
               payloadMap[vibId].append(localName)
         payloadMap[vibId] = sorted(payloadMap[vibId])
      return payloadMap

   def _getDBSourceForSigVerify(self, rebootingImage):
      """Returns database source to fetch VIBs/components in a signature
         verify call.
      """
      if (rebootingImage and self.imgstate in
          (self.IMGSTATE_LIVE_UPDATED, self.IMGSTATE_BOOTBANK_UPDATED)):
         # Specified rebooting image and /altbootbank is actually updated.
         return self.DB_BOOTBANK
      else:
         # Verifying live image or /bootbank will be the rebooting image.
         return self.DB_VISORFS

   def GetCompSigVerifyMap(self, rebootingImage=False):
      """Verifies component VIB signatures and payload checksums, returns a map
         from component IDs to component info like name, version, vendor,
         acceptance level and a verification result as a map from VIB IDs
         to its verification result returned from GetVibSigVerifyMap().
         esximage lock is held while the image DB is being accessed.
      """
      res = dict()

      self._getLock()
      try:
         database = self._getDBSourceForSigVerify(rebootingImage)
         comps = self.GetComponents(database=database)
         vibResult = self.GetVibSigVerifyMap(rebootingImage, getLock=False)
      finally:
         self._freeLock()

      if comps and vibResult:
         # When there are components on host and rebootingImage, if set, is
         # used properly.
         for comp in comps.IterComponents():
            compRes = dict(name=comp.compNameStr,
                           version=comp.compVersionStr,
                           vendor=comp.vendor,
                           vibs=dict())
            # Acceptance level of a component is the minimum level of the VIBs.
            minAccLevel = Vib.ArFileVib.ACCEPTANCE_CERTIFIED
            for vibId in comp.vibids:
               vibInfo = vibResult[vibId]
               accLevel = vibInfo['acceptance_level']
               if (AcceptanceChecker.TRUST_ORDER[accLevel] <
                   AcceptanceChecker.TRUST_ORDER[minAccLevel]):
                  minAccLevel = accLevel
               compRes['vibs'][vibId] = vibInfo['verification_result']
            compRes['acceptance_level'] = minAccLevel
            res[comp.id] = compRes
      return res

   def GetVibSigVerifyMap(self, rebootingImage=False, getLock=True):
      """Verifies VIB signature and payload checksums, returns a map from VIB
         IDs to VIB info like name, version, vendor, acceptance level and a
         verification result as a tuple of one of SIG_VERIFY_* states and an
         error message for SIG_VERIFY_ERROR.
         Parameter:
            rebootingImage - when set, check VIBs in the altbootbank image.
            getLock - when set, get esximage lock while accessing image DB;
                      otherwise assume the lock is already held.
      """
      if getLock:
         self._getLock()
      try:
         res = dict()
         rebootPending = self.imgstate == self.IMGSTATE_BOOTBANK_UPDATED
         database = self._getDBSourceForSigVerify(rebootingImage)
         imgProfile = self.GetProfile(database=database)
         # Do not use vibs of image profile in case an VIB ID is mismatching
         # due to corruption/tampering, which leads to misreporting.
         vibs = self.GetInventory(database=database)
         for vib in vibs.values():
            accLevel = getattr(vib, 'acceptancelevel', 'None')
            vibRes = (self.SIG_VERIFY_UNKNOWN, None)
            if vib.vibtype == Vib.ArFileVib.TYPE_LOCKER:
               vibRes = (self.SIG_VERIFY_LOCKER, None)
            elif isinstance(vib, Vib.ArFileVib):
               try:
                  self._verifyVibSignatureAndPayloads(vib, imgProfile,
                                                      rebootingImage,
                                                      rebootPending)
                  vibRes = (self.SIG_VERIFY_PASSED, None)
               except Errors.VibSigMissingError:
                  vibRes = (self.SIG_VERIFY_MISSING, None)
               except (Errors.VibFormatError, Errors.VibValidationError,
                       Errors.VibSignatureError,
                       Errors.ChecksumVerificationError) as e:
                  vibRes = (self.SIG_VERIFY_ERROR, str(e))
            res[vib.id] = dict(name=vib.name, version=vib.versionstr,
                               vendor=vib.vendor, acceptance_level=accLevel,
                               verification_result=vibRes)
      finally:
         if getLock:
            self._freeLock()
      return res

   @staticmethod
   def SendVob(eventid, *args):
      """Sends a VOB (VmKernel Observation) event.   VOB events allow important
         host events to be easily tracked in various clients.  This method
         is designed to send VOBs related to esximage.

         Parameters:
            * eventid - The VOB ID relative to vob.user.esximage, and
              (more importantly) the event type ID relative to
              esx.problem.esximage of the event that will be posted to
              hostd.  That "esx.problem.esximage." + eventid string
              will then be associated with a KB article.
            * args - The arguments associated with the vob.  See
              #bora/apps/addvob/addvob.c for details.
      """
      # Avoid import esx module globally due to confusion to vmotion test
      # lib in testesx.
      import esx.vob
      vob = esx.vob.createVob("vob.user.esximage." + eventid, *args)
      try:
         vob.send()
      except Exception as e:
         # Not logging the arguments, as syslog has length limits.
         log.error('Failed to send vob %s: %s' % (eventid, e))

   @staticmethod
   def SendConsoleMsg(msg):
      """Sends msg to console. The message will be displayed in red on
         informational console to warn user about potential issues.

         Parameters:
            * msg  - A string describing the warning.
      """
      if HAVE_SPICY:
         spicy.set_sysalert(msg)

   @staticmethod
   def SendAuditEvent(eventID, note, error, adds, removes):
      """Send an system audit event.
         Read bora/lib/public/vmwAudit.h on system.update.start and
         system.update.end events.

         Parameters:
            eventID - AUDIT_START_EVENTID or AUDIT_END_EVENTID.
            note    - String. In a success case, supply extra information to
                      be included in the reason field. Must be defined as
                      an AUDIT_* variable.
            error   - An Exception object. In a failure case, message of this
                      object is used to form the reason field.
            adds    - VIB IDs that are being added to the image profile.
            removes - VIB IDs that are being removed from the image profile.
      """
      import pwd
      import socket

      # From RFC 5424:
      # https://tools.ietf.org/html/rfc5424
      PRIORITY_INFO = 6
      FACILITY_AUDIT = 13
      # VMware IANA number:
      # https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers
      VMWARE_IANA_NUM = 6876

      AUDIT_LOG_SOCKET = '/dev/auditlog'
      AUDIT_LOG_FORMAT = '%(facility)u\0%(priority)u\0%(timeStamp)s\0' \
                         'esxupdate\0%(pid)u\0' \
                         '[%(eventID)s@%(originator)u subject="%(userName)s" ' \
                         'object="" result="%(result)s" reason="%(reason)s" ' \
                         'vib="%(vib)s"]\0\0'
      if error:
         # Fomulate error reason. Try to use msg for esxupdate errors and
         # replace the escaped chars and new lines.
         msg = error.msg if hasattr(error, 'msg') and error.msg \
               else str(error)
         replaces = [('\n', ','), ('"', ''), ('\'', ''), ('[', '{'), (']', '}')]
         for old, new in replaces:
            msg = msg.replace(old, new)
      else:
         # Use the extra note for a success
         msg = note if note else ''
      for i in range(4):
         s = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
         try:
            s.connect(AUDIT_LOG_SOCKET)
            utcNow = datetime.datetime.utcnow()
            isoTimeStamp = utcNow.strftime('%Y-%m-%dT%H:%M:%S.') + \
                           "%03dZ" % (utcNow.microsecond // 1000)

            msgArgs = {
               'facility': FACILITY_AUDIT,
               'priority': PRIORITY_INFO,
               'timeStamp': isoTimeStamp,
               'pid': os.getpid(),
               'eventID': eventID,
               'originator': VMWARE_IANA_NUM,
               'userName': pwd.getpwuid(os.geteuid()).pw_name,
               'result': 'failure' if error else 'success',
               'reason': msg,
               'vib': 'add: {%s} remove: {%s}' \
                      % (','.join(adds), ','.join(removes)),
            }
            s.send((AUDIT_LOG_FORMAT % msgArgs).encode())
         except OSError as e:
            log.warn('Failed to send audit event, try #%d: %s'
                     % (i + 1, str(e)))
            time.sleep(2)
         else:
            break
         finally:
            s.close()


   @staticmethod
   def _getHostAcceptance():
      # If there is an error or the configured value does not make sense,
      # we return a default.
      try:
         _, out = runcommand([ADVCFG, '-U', 'host-acceptance-level', '-G'],
                             timeout=30.0)
      except RunCommandError as e:
         msg = 'Unable to execute %s: %s' % (os.path.basename(ADVCFG), str(e))
         log.error(msg)
         raise Errors.AcceptanceGetError(msg)

      hostaccept = byteToStr(out).strip()
      if hostaccept in Vib.ArFileVib.ACCEPTANCE_LEVELS:
         return hostaccept
      elif hostaccept:
         log.error("Illegal acceptance level '%s' obtained from "
                   "configuration" % (hostaccept))
         return hostaccept
      else:
         log.error("No host acceptance level is configured")
         return ""

   @staticmethod
   def _setHostAcceptance(acceptance):
      if acceptance not in Vib.ArFileVib.ACCEPTANCE_LEVELS:
         msg = "Cannot set acceptance level: unknown level '%s'." % acceptance
         log.error(msg)
         raise Errors.AcceptanceConfigError(msg)

      try:
         res, out = runcommand([ADVCFG, '-U', 'host-acceptance-level',
                              '-S', acceptance])
      except RunCommandError as e:
         msg = 'Unable to execute %s: %s' % (os.path.basename(ADVCFG), str(e))
         log.error(msg)
         raise Errors.AcceptanceConfigError(msg)

      if res != 0:
         msg = ('%s exited with non-zero status %d, output: %s'
                % (os.path.basename(ADVCFG), res, out))
         log.error(msg)
         raise Errors.AcceptanceConfigError(msg)

   @staticmethod
   def TryLowerUpperPath(deployDir, relPath):
      """Given relative path in the deploy directory, try lower and
         upper paths, return the path that exists or None if both
         are not present.
      """
      upperPath = os.path.join(deployDir, relPath.upper())
      lowerPath = os.path.join(deployDir, relPath.lower())
      if os.path.exists(upperPath):
         return upperPath
      elif os.path.exists(lowerPath):
         return lowerPath
      return None

   @staticmethod
   def FindPayloadInDeployDir(vib, payload, deployDir):
      """Find absolute path of a VIB payload in deploy directory.
      """
      tryLowerUpperPath = HostImage.TryLowerUpperPath

      def restoreIsoLinux():
         """Restore isolinux.bin to the original form in esx-base VIB, return
            path to the file.
         """
         isoLinuxPath = tryLowerUpperPath(deployDir, 'isolinux.bin')
         if not isoLinuxPath:
            return None
         # The extracted ISO folder will never be used for boot, we can
         # modify in place.
         log.info('Restoring boot info table of isolinux file %s',
                  isoLinuxPath)
         try:
            with open(isoLinuxPath, 'rb+') as fObj:
               # See Iso9660.py EmbedBootInfoTable() for boot info table format.
               fObj.seek(8)
               pvdLoc, extLoc, dataLen, checksum = struct.unpack(
                  '<IIII40x', fObj.read(56))
               # XXX: changes based on observation of comparing esx-base and
               # ISO contents. If the upstream isolinux source changes,
               # upgrade would fail with checksum error.
               extLoc = 0
               padding = 0xdeadbeef
               newInfoTable = (struct.pack('<IIII', pvdLoc, extLoc, dataLen,
                                           checksum) +
                               struct.pack("<L", padding) * 10)
               fObj.seek(8)
               fObj.write(newInfoTable)
            return isoLinuxPath
         except (IOError, OSError, ValueError):
            log.exception('Failed to restore isolinux.bin')
            return None

      # Reference ImageBuilder/EsxIsoImage.py for how payloads are named
      # and placed.
      if payload.payloadtype in payload.GZIP_TYPES:
         # Regular tgz/vgz/boot payloads for bootbank.
         return tryLowerUpperPath(deployDir, payload.localname)
      elif payload.payloadtype == payload.TYPE_UPGRADE:
         # Upgrade payloads are in upgrade sub-dir.
         return tryLowerUpperPath(deployDir,
                                  os.path.join('upgrade', payload.name))
      elif payload.payloadtype == payload.TYPE_BOOT_LOADER_EFI:
         # EFI bootloader payloads in efi/boot sub-dir.
         return tryLowerUpperPath(deployDir,
                                  os.path.join('efi', 'boot', payload.name))
      elif payload.payloadtype == payload.TYPE_TEXT:
         # Text files are named differently.
         return tryLowerUpperPath(
            deployDir,
            '-'.join((vib.vendor, vib.name, payload.name)))
      elif payload.payloadtype == payload.TYPE_BOOT_ISO_BIOS:
         baseMiscPath = tryLowerUpperPath(
                           deployDir,
                           BootBankInstaller.BootBank.BASEMISC_PAYLOADTAR)
         if baseMiscPath:
            # isolinux.bin is modified for ISO boot at ISO creation time,
            # extract an original copy from basemisc.tgz.
            try:
               with tarfile.open(baseMiscPath, 'r') as tar:
                  relPath = os.path.join(PayloadTar.PAYLOADTAR_PREFIX,
                                         payload.payloadtype,
                                         payload.name)
                  m = tar.getmember(relPath)
                  tar.extract(m, path=deployDir)
               # At this point the casing should match payload name.
               return os.path.join(deployDir, relPath)
            except (EnvironmentError, tarfile.TarError):
               log.exception('Failed to extract VIB %s payload %s from '
                             'basemisc.tgz', vib.id, payload.name)
               return None
         elif payload.name == 'isolinux.bin':
            # In an ISO that is customized by legacy tool, there is no
            # basemisc.tgz. We can only try our best to retore the
            # isolinux.bin file back to how it looked like in esx-base VIB.
            return restoreIsoLinux()
         else:
            # Currently there is no other ISO BIOS module other than
            # isolinux.bin, however allow lookup of the payload in
            # the ISO root.
            return tryLowerUpperPath(deployDir, payload.name)
      else:
         # Other ISO/BIOS specific payloads.
         return tryLowerUpperPath(deployDir, payload.name)

   def _getLock(self):
      """Lock esximage exclusive file lock. Should be called with write
         transactions at entry points. Most of such transactions happen
         in Transaction with a few exception in HostImage.
      """
      # We will try to acquire lock for one min with 5 sec interval retry
      retries = 13
      delay = 5
      for i in range(retries):
         try:
            self._lock.Lock()
            break
         except LockFile.LockFileError as e:
            log.warn('Failed to acquire lock: %s', str(e))
            if i < retries - 1:
               time.sleep(delay)
      else:
         raise Errors.LockingError("Another process is updating the ESX "
                                   "image. Please try again later.")

      # Refresh the installers to make sure we load the latest state.
      # Even without a single locking error the state might have been just
      # refreshed before this method is called.
      self._initInstallers()

   def _freeLock(self):
      """Free esximage exclusive file lock.
      """
      try:
         self._lock.Unlock()
      except Exception as e:
         raise Errors.LockingError("Unable to free lock: %s" % (str(e)))
