"""
Copyright (c) 2020-2021 VMware, Inc.
All rights reserved. -- VMware Confidential
"""
from datetime import datetime, timedelta
import glob
import json
import logging
import os
import shutil
import stat
import tarfile
import tempfile

from .Constants import *
from .DepotMgr import DepotMgr
from .Utils import getFormattedMessage, getCommaSepArg

from .. import Depot, Errors, HostImage, MIB, OfflineBundle, Vib
from ..Utils import EsxGzip, HostInfo
from ..Version import VibVersion
from ..VibCollection import VibCollection

try:
   # Calls to get image info or extract depot require bindings that aren't
   # found on earlier ESXi during upgrades.
   from .Scanner import getSolutionInfo, vapiStructToJson
   from com.vmware.esx.software_client \
      import (InstalledImage, SoftwareInfo, BaseImageInfo,
              AddOnInfo, ComponentInfo, HardwareSupportInfo, Notifications,
              Notification, SoftwareSpec, BaseImageSpec,
              AddOnSpec, HardwareSupportSpec)
   from com.vmware.vapi.std_client import LocalizableMessage
   VAPI_SUPPORT = True
except ImportError:
   VAPI_SUPPORT = False

isNotNone = lambda x: x is not None
EXTRACT_DEPOT_TASK_ID = 'com.vmware.esx.software.installedimage.extractdepot'
DEPOT_FILE_NAME = 'OfflineBundle'
LIFECYCLE_SCRACTCH = '/var/vmware/lifecycle'
_HOST_SEED_POSTFIX = 'hostSeed'
HOST_SEED_DIR_NAME = os.path.join(LIFECYCLE_SCRACTCH, _HOST_SEED_POSTFIX)

# TODOs:
#  Check if we have the required space on disk'

# Local exception classes for caller to be aware of the error context.
class ReserveVibCacheError(Exception):
   pass

class NoVibCacheError(ReserveVibCacheError):
   pass

class VibNotInCacheError(ReserveVibCacheError):
   pass

def getNotification(notificationId, msgId, msgArgs=None,
                     resArgs=None):
   """Helper function to compose the Notification Object
   """
   defMsg = getFormattedMessage(NOTIFICATION_MSG[msgId], msgArgs)
   msg = LocalizableMessage(id=msgId, default_message=defMsg,
                            args=msgArgs or [])
   resMsg = getFormattedMessage(
                  RESOLUTION_MSG.get(notificationId, ''), resArgs)
   if resMsg:
      # Populate the optional resolution when there is actually a message.
      resId = msgId + RESOLUTION_SUFFIX if resMsg else ''
      resolution = LocalizableMessage(id=resId,
                                      default_message=resMsg,
                                      args=resArgs or [])
   else:
      resolution = None

   return Notification(id=notificationId,
                       time=datetime.utcnow(),
                       message=msg,
                       resolution=resolution)

def _getImageNotification(msgId, info):
   """Helper function to form a notification for a components or VIBs.
   """
   if isinstance(info, dict):
      details = ['%s(%s)' % (name, version) for name, version in info.items()]
      return getNotification(msgId, msgId, msgArgs=[getCommaSepArg(details)])

   return getNotification(msgId, msgId, msgArgs=[getCommaSepArg(info)])

def _findVibPayload(vib, pName):
   """Returns payload object with the specified name in a VIB.
   """
   for payload in vib.payloads:
      if payload.name == pName:
         return payload
   raise ValueError('Payload %s is not found in VIB %s' % (pName, vib.id))

def _checkPayloadChecksum(fObj, pObj):
   """Checks if a payload file matches the expected checksum, an exception
      would be raised if the check does not pass.
   """
   for checksum in pObj.checksums:
      if checksum.verifyprocess == '':
         hashAlgo, expected = \
            checksum.checksumtype.replace('-', ''), checksum.checksum
         break
   else:
      raise ValueError('No checksum found for payload %s' % pObj.name)

   calculated = Vib.calculatePayloadChecksum(pObj, fObj, hashAlgo, False)
   logging.debug("Calculated %s checksum of payload %s '%s', expected '%s'",
                 hashAlgo, pObj.name, calculated, expected)

   if calculated != expected:
      raise RuntimeError('Calculated %s checksum of payload %s does not '
                         'match VIB metadata' % (hashAlgo, pObj.name))

def _getSeedImageProfile(profile):
   '''Orphan vibs, solutions and manifests are not needed in the extracted
      depot. Hence we remove them from profile and return the modifed profile.
   '''
   newProfile = profile.Copy()
   # Get list of orphan vibs and remove them from the cloned image profile.
   orphanVibs = newProfile.GetOrphanVibs()
   if orphanVibs:
      logging.info('Skipping orphan VIBs %s in depot extraction',
                   ', '.join(sorted(list(orphanVibs.keys()))))
   for vib in orphanVibs:
      newProfile.RemoveVib(vib)

   # Get list of solutions and remove them from the cloned image profile.
   sols = [s.nameSpec.name for s in newProfile.solutions.values()]
   if sols:
      logging.info('Skipping solutions %s in depot extraction',
                   ', '.join(sorted(sols)))
      newProfile.RemoveSolutions(sols)

   # Get list of manifests and remove them from the cloned image profile.
   if newProfile.manifestIDs:
      logging.info('Skipping manifests %s in depot extraction',
                   ', '.join(sorted(newProfile.manifestIDs)))
      for manifestID in newProfile.manifestIDs:
         newProfile.manifests.RemoveManifest(manifestID)
      newProfile._syncVibs()

   return newProfile

def getIsoUpgradePayloadPath(vib, pName, hostImage, isoDir, isoProfile):
   """Get path to a VIB payload in an extracted ISO directory.
   """
   # localname of payload needs to be filled in.
   pObj = _findVibPayload(vib, pName)
   try:
      pObj.localname = isoProfile.vibstates[vib.id].payloads[pName]
   except KeyError:
      raise KeyError('VIB %s payload %s is not found in ISO directory image '
                     'database' % (vib.id, pName))
   logging.info('Finding %s in %s', pObj.name, isoDir)
   stagePath = hostImage.FindPayloadInDeployDir(vib, pObj, isoDir)
   with open(stagePath, 'rb') as fObj:
      _checkPayloadChecksum(fObj, pObj)

   return stagePath

def getBootBankPayloadPath(vib, pName, hostImage, tmpDir):
   """Get path to a bootbank VIB payload.
      Direct path into the current bootbank will be returned for gzipped
      payloads, a temp file created in tmpDir will be returned for misc
      esx-base payloads in basemisc.tgz and empty gzip payloads that are
      since modified after installation.
   """
   def createEmptyGzFile():
      """Creates an temp empty gzip file.
      """
      tempfObj = tempfile.NamedTemporaryFile(dir=tmpDir, delete=False)
      try:
         EsxGzip.GzipFile(tempfObj.name, 'wb').close()
         tempfObj.seek(0)
      except Exception:
         tempfObj.close()
         raise
      return tempfObj

   def createTempFile(fObj):
      with fObj:
         # Write the payload file to a temp file.
         tempfObj = tempfile.NamedTemporaryFile(dir=tmpDir, delete=False)
         shutil.copyfileobj(fObj, tempfObj, MIB)
         tempfObj.seek(0)
         return tempfObj

   if 'boot' not in hostImage.installers:
      raise RuntimeError('BootBankInstaller is not initiated')

   bbInstaller = hostImage.installers['boot']

   pObj = _findVibPayload(vib, pName)
   fObj = bbInstaller.OpenPayloadFile(vib.id, pObj)
   if fObj is None:
      logging.debug('Payload %s type %s of VIB %s cannot be opened by '
                   'BootBankInstaller', pObj.name, pObj.payloadtype, vib.id)
      return None

   if isinstance(fObj, tarfile.ExFileObject):
      fObj = createTempFile(fObj)
   stagePath = fObj.name

   # Check checksum of the payload., if it does not match and payload type is
   # boot, it might be an empty gzip payload (e.g. jumpstrt/features.gz).
   try:
      _checkPayloadChecksum(fObj, pObj)
   except RuntimeError:
      if pObj.payloadtype != pObj.TYPE_BOOT:
         raise
      # For boot payload, when checksum does not match, it might be an
      # empty/useropts gzip payload, re-create it.
      logging.debug('Payload %s of VIB %s might be an empty gzip payload or '
                    'modified during runtime', pObj.name, vib.id)
      if pObj.name == "useropts":
         bbfObj = bbInstaller.OpenPayloadFile(vib.id, pObj, fromBaseMisc=True)
         with createTempFile(bbfObj) as tempfObj:
            _checkPayloadChecksum(tempfObj, pObj)
            stagePath = tempfObj.name
      else:
         with createEmptyGzFile() as tempfObj:
            _checkPayloadChecksum(tempfObj, pObj)
            stagePath = tempfObj.name
   finally:
      fObj.close()

   return stagePath

def getLockerPayloadPath(vib, pName, hostImage, tmpDir):
   """For locker vibs, payload is extracted in locker partition and kept there.
      To reconstruct a locker vib, we need to create the exact .tgz payload
      and we have to make sure that there are no timestamp or user information
      which can change checksum. Currently, there is only one locker vib i.e.
      tools-light which needs to be handled this way. This follows the similar
      implementation as done in scons build.
   """
   from ..Installer.LockerInstaller import LOCKER_ROOT, PAYLOAD_MAPPING_FILE
   if 'locker' not in hostImage.installers:
      raise Errors.VibRecreateError(vib.id,
                                    'LockerInstaller is not initiated')

   payloadPath = LOCKER_ROOT
   # Stage the .tar and .tgz payloads in hostSeed directory. These files have
   # to be deleted once the payload is added to the vib.
   payloadtar = os.path.join(tmpDir, pName + '.tar')
   gzipFile = os.path.join(tmpDir, pName)
   # Handle the case when multiple payloads are present for a given locker vib.
   # We have a file inside locker root which stores the payload to file mapping.
   if os.path.exists(PAYLOAD_MAPPING_FILE):
      try:
         with open(PAYLOAD_MAPPING_FILE, 'r') as f:
            payloadFileDict = json.load(f)
         fileList = payloadFileDict.get(vib.name, dict()).get(pName, []) or \
                    vib.filelist
      except Exception as e:
         logging.error('Failed to read from file %s: %s',
                       PAYLOAD_MAPPING_FILE, str(e))
         fileList = vib.filelist
   else:
      # If payload mapping file doesn't exist due to accidental delete or when
      # this host was upgraded without patch the patcher.
      # At present, we don't have any locker vib with more than one payload
      # and hence it is better to fall back on vib's filelist.
      fileList = vib.filelist

   try:
      # Order in which files are added to tar is important to ensure a
      # checksum match with descriptor.
      with tarfile.open(payloadtar, 'w') as tar:
         for root, dirs, files in os.walk(payloadPath):
            for f in dirs + files:
               fspath = os.path.join(root, f)
               arcname = os.path.relpath(fspath, payloadPath)
               # Do not add a file that is not in VIB's filelist, or a dir
               # that is not in any of the files' path.
               if os.path.isdir(fspath) and not any(f in x for x in fileList):
                  continue
               if os.path.isfile(fspath) and arcname not in fileList:
                  continue

               # After VUM upgrade from 6.x, some directories may be left
               # behind and individual files may be found with 700 permission.
               # Skip the directories and adjust permissions back to default.
               # TODO: Remove after 6.x releases hit EOL.
               if os.path.isdir(fspath) and '6.5.0' in fspath:
                  continue

               mode = os.stat(fspath).st_mode
               if os.path.isfile(fspath) and oct(stat.S_IMODE(mode)) != '0o644':
                  os.chmod(fspath, 0o644)

               ti = tar.gettarinfo(fspath, arcname)
               ti.mtime = 0
               ti.uid = ti.gid = 0
               ti.uname = ti.gname = 'root'
               if ti.islnk():
                  ti.type = tarfile.REGTYPE
                  ti.size = os.path.getsize(fspath)
               tar.addfile(ti, None if ti.isdir() or ti.issym() else
                           open(fspath, 'rb'))

      # Now Gzip the tar file to convert into a tgz. EsxGzip ensures that gzip
      # header doesn't contain timestamp which can affect checksum.
      with EsxGzip.GzipFile(gzipFile, 'wb') as out:
          with open(payloadtar, 'rb') as inp:
             data = inp.read(Vib.PAYLOAD_READ_CHUNKSIZE)
             while data:
                out.write(data)
                data = inp.read(Vib.PAYLOAD_READ_CHUNKSIZE)
   except Exception as e:
      raise Errors.VibRecreateError(vib.id, "Failed to create payload %s: %s" %
                                    (pName, str(e)))
   finally:
      # Delete the .tar file as it is not needed anymore
      if os.path.isfile(payloadtar):
         os.remove(payloadtar)
   # Return the final stage path which will be used to add payload to vib.
   return gzipFile

class InstalledImageInfo(object):
   """This class provides methods to get the ESX host's current
      software information.
   """

   def __init__(self):
      """Constructor
      """
      if not VAPI_SUPPORT:
         raise RuntimeError('VAPI support classes could not be imported')

      self.currHostImage = HostImage.HostImage()
      db = self.currHostImage.DB_VISORFS
      self.currImageProfile = self.currHostImage.GetProfile(database=db)

      if self.currImageProfile is None or len(self.currImageProfile.vibIDs)==0:
         msg = "Could not extract profile from host"
         logging.error(msg)
         raise ValueError(msg)

      self.biAddonCompNames = set()
      if self.currImageProfile.baseimage:
         self.baseImageComponents = self.currImageProfile.baseimage.components
         self.biAddonCompNames |= set(self.baseImageComponents.keys())

      if self.currImageProfile.addon:
         self.addOnComponents = self.currImageProfile.addon.components
         self.biAddonCompNames |= set(self.addOnComponents.keys())
      else:
         self.addOnComponents = None

      # Manifests info
      self.hspDict, self.allHspCompDict, self.allHspRmCompNames = \
         self.currImageProfile.GetHardwareSupportInfo()

   def getLocalSolutionInfo(self):
      """Get a tuple of:
         1) A dict of SolutionComponentDetails for local solution components,
            indexed by solution name and then component name.
         2) Names of components that are installed as part of solutions.
      """
      solInfoDict = dict()
      ipSolDict, ipSolComps = self.currImageProfile.GetSolutionInfo()
      for solId, solComps in ipSolDict.items():
         solution = self.currImageProfile.solutions[solId]
         solInfoDict[solution.nameSpec.name] = getSolutionInfo(solution,
                                                               solComps)
      return solInfoDict, list(ipSolComps.keys())

   def _getCompVersions(self, name):
      """Returns versions of a component in base image, addon and HSP,
         None is used whenever the component is not found in a release unit.
      """
      getCompVerObj = lambda d, n: (VibVersion.fromstring(d[n])
                                    if d and n in d else None)
      return [getCompVerObj(comps, name)
              for comps in (self.baseImageComponents, self.addOnComponents,
                 self.allHspCompDict)]

   # TODO: Reuse duplicate code from Scanner - Bug 2678306
   def _getRemovedDgAndUgComponents(self, baseImageComponents, addOnComponents,
                                    addOnRemovedCompNames):
      installedComps = self.currImageProfile.ListComponentSummaries()

      compUserAdded = dict()
      addOnRemovedCompNames = addOnRemovedCompNames or []
      hspRemovedCompNames = self.allHspRmCompNames or set()

      # All component names appear in the addon, base image and manifests.
      biAddonHspCompNames = set(baseImageComponents.keys())
      if addOnRemovedCompNames:
         biAddonHspCompNames -= set(addOnRemovedCompNames)
      if addOnComponents:
         biAddonHspCompNames |= set(addOnComponents.keys())
      if hspRemovedCompNames:
         biAddonHspCompNames -= hspRemovedCompNames
      if self.allHspCompDict:
         biAddonHspCompNames |= set(self.allHspCompDict.keys())

      ADD, UPGRADE, DOWNGRADE = 'add', 'upgrade', 'downgrade'
      # Base image components.
      baseImageCompSpec = dict()
      # A breakdown of addon components regarding base image components.
      addonCompSpec = {
         ADD: dict(),
         UPGRADE: {
            BASE_IMG: dict(),
         },
         DOWNGRADE: {
            BASE_IMG: dict(),
         },
      }
      # A breakdown of hardware support components' relationship related to
      # base image and addon.
      hspCompSpec = {
         ADD: dict(),
         UPGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict()
         },
         DOWNGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict(),
         },
      }
      # A breakdown of user components regarding manifest components.
      userCompSpec = {
         ADD: dict(),
         UPGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict(),
            HARDWARE_SUPPORT: dict(),
         },
         DOWNGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict(),
            HARDWARE_SUPPORT: dict(),
         },
      }
      def _addCompInSpec(compName, comp, spec, subject, hostVer, subjectVer):
         """Triage a component to one of add, upgrade and downgrade categories
            according to host's and the spec piece's component versions.
            An addition in the spec dict means the spec piece (e.g. addon)
            adds the component, or upgrades/downgrades the component of another
            image piece (subject), e.g. base image.
         """
         if hostVer > subjectVer:
            spec[UPGRADE][subject][compName] = comp
         elif hostVer < subjectVer:
            spec[DOWNGRADE][subject][name] = comp
         else:
            # Even if multiple pieces claim the same version, the component
            # will be treated as "add" in all of them. This makes sure upgrade
            # and downgrade components are calculated correctly.
            spec[ADD][name] = comp

      solDict, solutionCompNames = self.getLocalSolutionInfo()
      installedCompNames = set()
      for comp in installedComps:
         # Loop to figure out upgrade/downgrade relations.
         name = comp['component']
         installedCompNames.add(name)
         if name in solutionCompNames:
            # Solution component.
            continue
         if name not in self.biAddonCompNames:
            # Component introduced by the user, could be a new component,
            # or one that is removed by addon but re-added by user.
            compUserAdded[name] = comp
         else:
            # Find the provider of the component and if there is any
            # upgrade/downgrade took place.
            hostVersion = VibVersion.fromstring(comp[VERSION])
            biVersion, addonVersion, hspVersion = self._getCompVersions(name)

            if isNotNone(biVersion):
               # Base image originally has the component.
               if biVersion == hostVersion:
                  # No override.
                  baseImageCompSpec[name] = comp
               elif isNotNone(addonVersion) and addonVersion == hostVersion:
                  # Addon override base image, it should be upgrade only, but
                  # we will check all cases nevertheless.
                  _addCompInSpec(name, comp, addonCompSpec, BASE_IMG,
                                 hostVersion, biVersion)
               elif isNotNone(hspVersion) and hspVersion == hostVersion:
                  # HSP overrides base image, it should be upgrade only, but
                  # we will check all cases just like addon.
                  _addCompInSpec(name, comp, hspCompSpec, BASE_IMG,
                                 hostVersion, biVersion)
               else:
                  # User component override base image, "add" was already
                  # handled, so here should only be upgrade/downgrade.
                  _addCompInSpec(name, comp, userCompSpec, BASE_IMG,
                                 hostVersion, biVersion)
            elif isNotNone(addonVersion):
               # Addon originally has the component.
               if addonVersion == hostVersion:
                  # Addon adds the component.
                  addonCompSpec[ADD][name] = comp
               elif isNotNone(hspVersion) and hspVersion == hostVersion:
                  # HSP overrides addon, it should be upgrade only, but
                  # we will check all cases just like addon.
                  _addCompInSpec(name, comp, hspCompSpec, ADD_ON,
                                 hostVersion, addonVersion)
               else:
                  # User component override addon, "add" was already
                  # handled, so here should only be upgrade/downgrade.
                  _addCompInSpec(name, comp, userCompSpec, ADD_ON,
                                 hostVersion, addonVersion)
            else:
               # HSP originally has the component.
               if hspVersion == hostVersion:
                  # Manifest adds the component.
                  hspCompSpec[ADD][name] = comp
               else:
                  # User component upgrades/downgrades HSP.
                  _addCompInSpec(name, comp, userCompSpec, HARDWARE_SUPPORT,
                                 hostVersion, hspVersion)


      # Component on host may have been reserved due to VIB obsolete rather than
      # same component name replacement or manual removal. A catch-up component
      # scanning is required so we do not report release unit component removal
      # when a component was obsoleted as a result of the last apply.
      reservedComps = set(self.currImageProfile.reservedComponentIDs)
      depotMgrObj = DepotMgr(depotSpecs=None, connect=True)
      if reservedComps:
         fullComps = depotMgrObj.componentsWithVibs
         hostComps = self.currImageProfile.GetKnownComponents()

      # Sets for the components of interest.
      removedBIComps = ((set(baseImageComponents.keys()) - installedCompNames) -
                         set(addOnRemovedCompNames) - set(hspRemovedCompNames))
      downgradedBIComps = (set(addonCompSpec[DOWNGRADE][BASE_IMG].keys() |
                           set(hspCompSpec[DOWNGRADE][BASE_IMG].keys()) |
                            set(userCompSpec[DOWNGRADE][BASE_IMG].keys())) -
                            set(addOnRemovedCompNames) -
                            set(hspRemovedCompNames))

      removedAddonComps, downgradedAddonComps = set(), set()
      if addOnComponents:
         removedAddonComps = (set(addOnComponents.keys()) -
                              installedCompNames -
                              set(hspRemovedCompNames))
         downgradedAddonComps = (set(hspCompSpec[DOWNGRADE][ADD_ON].keys()) |
                                 set(userCompSpec[DOWNGRADE][ADD_ON].keys()) -
                                 set(hspRemovedCompNames))
      # All addon components
      allAddonComps = dict()
      allAddonComps.update(addonCompSpec[ADD])
      allAddonComps.update(addonCompSpec[UPGRADE][BASE_IMG])
      allAddonComps.update(addonCompSpec[DOWNGRADE][BASE_IMG])

      # Update the User Components with the Added, and Upgraded Components.
      compUserAdded.update(userCompSpec[ADD])

      compUserUpgraded = dict()
      compUserUpgraded.update(userCompSpec[UPGRADE][HARDWARE_SUPPORT])
      compUserUpgraded.update(userCompSpec[UPGRADE][ADD_ON])
      compUserUpgraded.update(userCompSpec[UPGRADE][BASE_IMG])
      compUserAddUpgrade = dict()
      compUserAddUpgrade.update(compUserAdded)
      compUserAddUpgrade.update(compUserUpgraded)

      compInfo = dict()
      # Name to info dictionaries.
      compInfo[BASEIMAGE_COMPS_KEY] = baseImageCompSpec
      compInfo[ADDON_COMPS_KEY] = allAddonComps
      # Lists of names.
      compInfo[REMOVED_DG_BI_COMP_KEY] = removedBIComps | downgradedBIComps
      compInfo[REMOVED_DG_ADDON_COMP_KEY] = (removedAddonComps |
                                             downgradedAddonComps)
      compInfo[USER_ADD_UPGRADE_COMPS_KEY] = list(compUserAddUpgrade.values())
      return compInfo

   def addNotification(self, notificationList, notification):
      if notificationList is None:
         notificationList = []
      notificationList.append(notification)
      return notificationList

   def _getNotificationList(self, baseImageComponents, addOnComponents,
                            addOnRemovedCompNames):
      """Get the info/warning/error messages that can be reported by the task
      """
      # Notifications are mandatory. Hence we initialize it at the begining
      # even if eventually no Notifications are populated. This is similar to
      # the scan operation.
      notifications = Notifications(info=None,
                                    warnings=None,
                                    errors=None)

      # Populating Notification type:
      # 1. [ERROR] Stateless host is not supported for seeding. No need to
      # proceed further if we encounter this.
      if HostInfo.IsPxeBooting():
         notifications.errors = \
            self.addNotification(notifications.errors,
               getNotification(UNSUPPORTED_STATELESS_HOST_ID,
                               UNSUPPORTED_STATELESS_HOST_ID))
         return notifications, None

      compInfo = self._getRemovedDgAndUgComponents(baseImageComponents, \
         addOnComponents, addOnRemovedCompNames)

      # Populating Notification type:
      # 2. [ERROR] Removed or Downgraded BI Components
      removedDgBiComps = set()
      for comp in compInfo[REMOVED_DG_BI_COMP_KEY]:
         removedDgBiComps.add(comp)

      if removedDgBiComps:
         notifications.errors = self.addNotification(notifications.errors,
            _getImageNotification(COMPONENTS_REMOVED_DG, removedDgBiComps))
         logging.error('BaseImg Comps are downgraded or removed: %s',
            str(compInfo[REMOVED_DG_BI_COMP_KEY]))

      # Populating Notification type:
      # 3. [ERROR] Removed or Downgraded Addon Components
      removedDgAddonComps = set()
      for comp in compInfo[REMOVED_DG_ADDON_COMP_KEY]:
         removedDgAddonComps.add(comp)

      if removedDgAddonComps:
         notifications.errors = self.addNotification(notifications.errors,
            _getImageNotification(COMPONENTS_REMOVED_DG, removedDgAddonComps))
         logging.error('Addon Comps are downgraded or removed: %s',
            str(compInfo[REMOVED_DG_ADDON_COMP_KEY]))

      # Populating Notification type:
      # 4. [WARNING] Orphan Vibs
      hostVibs = self.currImageProfile.vibs
      orphanVibList = list(self.currImageProfile.GetOrphanVibs())

      if orphanVibList:
         orphanVibs = [hostVibs[vibId] for vibId in orphanVibList]
         orphanVibsInfo = dict()

         try:
            allVibs = VibCollection()
         except Exception as e:
            logging.error('Error while getting list of all VIBs: %s', str(e))
            raise

         for vib in orphanVibs:
            allVibs.AddVib(vib)

         scanResult = allVibs.Scan()
         for vib in orphanVibs:
            if not scanResult.vibs[vib.id].replacedBy:
               # Missing and will be removed during next remediation.
               orphanVibsInfo[vib.name] = vib.versionstr

         if orphanVibsInfo:
            notifications.warnings = \
               self.addNotification(notifications.warnings,
               _getImageNotification(ORPHAN_VIB, orphanVibsInfo))
            logging.debug('List of Orphan Vibs: %s', str(orphanVibsInfo))

      # Populating Notification type:
      # 5. [Warning] Host is pending reboot.
      if self.currHostImage.imgstate == self.currHostImage.IMGSTATE_BOOTBANK_UPDATED:
         notifications.warnings = \
            self.addNotification(notifications.warnings,
               getNotification(SEEDING_PENDING_REBOOT_ID,
                               SEEDING_PENDING_REBOOT_ID))

      return notifications, compInfo

   def _deleteFile(self, fileName):
      try:
         os.remove(fileName)
         logging.info('File %s deleted', fileName)
      except OSError as e:
         logging.warn('Failed to remove %s: %s', fileName, str(e))

   def _cleanOldOfflineDepots(self):
      """Delete Offline Bundles older than 30 minutes
      """
      oldZips = glob.glob(HOST_SEED_DIR_NAME + \
         '/OfflineBundle-[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9]--' \
         '[0-2][0-9].[0-5][0-9].zip')
      now = datetime.today()
      waittime = timedelta(minutes=30)
      for zipFile in oldZips:
         timePast = now - datetime.strptime(zipFile[-21:-4], '%Y-%m-%d--%H.%M')
         if timePast > waittime:
            self._deleteFile(zipFile)

   def getCurrentInfo(self):
      """Get the attributes that describe the current software
         information on the host
      """
      baseImageInfoObj, addOnInfoObj, hardwareSupportInfoObj, \
         notificationsObj, softwareInfoObj = None, None, None, None, None
      componentInfoDict, solutionInfoDict = dict(), dict()

      # Populate the Base Image
      baseImageComponents = None
      if self.currImageProfile.baseimage:
         baseImageComponents = self.currImageProfile.baseimage.components
         baseImageVersion = \
            self.currImageProfile.baseimage.versionSpec.version.versionstring
         baseImageDisplayName = BASEIMAGE_UI_NAME
         baseImageDisplayVersion = \
            self.currImageProfile.baseimage.versionSpec.uiString
         baseImageReleaseDate = self.currImageProfile.baseimage.releaseDate
         logging.debug('BaseImage details : %s, %s, %s, %s',
                       baseImageVersion,
                       baseImageDisplayName,
                       baseImageDisplayVersion,
                       baseImageReleaseDate)
         baseImageInfoObj = BaseImageInfo(baseImageVersion,
                                          baseImageDisplayName,
                                          baseImageDisplayVersion,
                                          baseImageReleaseDate)
      else:
         logging.error('SoftwareInfo must have a baseImage')

      # Populate the Add-on
      addOnComponents, addOnRemovedCompNames = None, None
      if self.currImageProfile.addon:
         addOnRemovedCompNames = self.currImageProfile.addon.removedComponents
         addOnComponents = self.currImageProfile.addon.components
         addonName = self.currImageProfile.addon.nameSpec.name
         addonVersion = \
            self.currImageProfile.addon.versionSpec.version.versionstring
         addonDisplayName = self.currImageProfile.addon.nameSpec.uiString
         addonVendor = self.currImageProfile.addon.vendor
         addonDisplayVersion = self.currImageProfile.addon.versionSpec.uiString
         logging.debug('AddOn details : %s, %s, %s, %s, %s',
                      addonName,
                      addonVersion,
                      addonDisplayName,
                      addonVendor,
                      addonDisplayVersion)
         addOnInfoObj = AddOnInfo(addonName,
                                  addonVersion,
                                  addonDisplayName,
                                  addonVendor,
                                  addonDisplayVersion)

      # Populate Notifications
      notificationsObj, compInfo = self._getNotificationList(
         baseImageComponents, addOnComponents, addOnRemovedCompNames)

      # If we have any error notifications, we need to Return an empty
      # SoftwareInfo. Ensure to retain in the mandatory elements, even if
      # empty : SoftwareInfo[BaseImageInfo, ComponentInfo, SolutionInfo]
      if notificationsObj.errors is not None:
         softwareInfoObj = SoftwareInfo(BaseImageInfo(None, None, None, None),
                                        None,
                                        dict(),
                                        dict(),
                                        None)
         # TODO: Currently we raise an error when Notification contains any
         # error messages as the UI does not handle this. This needs to change
         # once UI is able to consume Notifications and take action.
         # Tracked via - https://jira.eng.vmware.com/browse/ESXLCM-7097
         msg = list()
         for error in notificationsObj.errors:
            msg.append(error.message.default_message)
         logging.error("Software info extract errors: %s", ', '.join(msg))
         raise Errors.SoftwareInfoExtractError(', '.join(msg))

      else:
         # Populate the Components
         compInfo[USER_ADD_UPGRADE_COMPS_KEY]
         for installedComp in compInfo[USER_ADD_UPGRADE_COMPS_KEY]:
            componentVersion = installedComp['version']
            componentDisplayName = installedComp['display_name']
            componentDisplayVersion = installedComp['display_version']
            componentVendor = installedComp['vendor']
            logging.debug('Component details : %s, %s, %s, %s',
                          componentVersion,
                          componentDisplayName,
                          componentDisplayVersion,
                          componentVendor)
            component = ComponentInfo(componentVersion,
                                      componentDisplayName,
                                      componentDisplayVersion,
                                      componentVendor)
            componentInfoDict[componentDisplayName] = component

         softwareInfoObj = SoftwareInfo(baseImageInfoObj,
                                        addOnInfoObj,
                                        componentInfoDict,
                                        solutionInfoDict,
                                        hardwareSupportInfoObj)

      installedImageInfo = \
                       InstalledImage.Info(notificationsObj, softwareInfoObj)
      return installedImageInfo

   def getSwSpecAndNotifs(self):
      """Get the attributes that describe the current software
         specification for the ESX host
      """
      baseImageSpecObj, addonSpecObj, hardwareSupportSpecObj, softwareSpecObj, \
         notificationsObj = None, None, None, None, None
      componentSpecDict, solutionsSpecDict = dict(), dict()

      # Populate the Base Image
      baseImageComponents = None
      if self.currImageProfile.baseimage:
         baseImageComponents = self.currImageProfile.baseimage.components
         logging.debug('BaseImage details : %s',
            self.currImageProfile.baseimage.versionSpec.version.versionstring)
         baseImageSpecObj = BaseImageSpec(
            self.currImageProfile.baseimage.versionSpec.version.versionstring)
      else:
         logging.error('SoftwareSpec must have a baseImage')

      # Populate the Add-on
      addOnComponents, addOnRemovedCompNames = None, None
      if self.currImageProfile.addon:
         addOnRemovedCompNames = self.currImageProfile.addon.removedComponents
         addOnComponents = self.currImageProfile.addon.components
         addonSpecName = self.currImageProfile.addon.nameSpec.name
         addonSpecVersion = \
            self.currImageProfile.addon.versionSpec.version.versionstring
         logging.debug('AddOn details : %s, %s',
                      addonSpecName,
                      addonSpecVersion)
         addonSpecObj = AddOnSpec(addonSpecName, addonSpecVersion)

      # Populate Notifications
      notificationsObj, compInfo = self._getNotificationList( \
      baseImageComponents, addOnComponents, addOnRemovedCompNames)

      # If we have any error notifications, we need to Return an empty
      # SoftwareSpec. Ensure to retain in the mandatory elements, even if
      # empty : SoftwareSpec[BaseImageSpec]
      if notificationsObj.errors is not None:
         softwareSpecObj = SoftwareSpec(BaseImageSpec(None),
                                        None,
                                        dict(),
                                        dict(),
                                        None)
      else:
         # Populate the Components
         compInfo[USER_ADD_UPGRADE_COMPS_KEY]
         for installedComp in compInfo[USER_ADD_UPGRADE_COMPS_KEY]:
            componentName = installedComp['component']
            componentVersion = installedComp['version']
            logging.debug('Component details : %s, %s',
                      componentName,
                      componentVersion)
            componentSpecDict[componentName] = componentVersion

         softwareSpecObj = SoftwareSpec(baseImageSpecObj,
                                        addonSpecObj,
                                        componentSpecDict,
                                        solutionsSpecDict,
                                        hardwareSupportSpecObj)

      return softwareSpecObj, notificationsObj

   def extractDepot(self, task):
      """1. Acquire transaction lock.
         2. Create hostSeed directory if not already available.
         3. Delete older zips.
         4. Invoke extracting depotDir of live running image.
         5. Create a zip of the depot and store it in the created directory.
         6. Finally delete the depotDir.
         Any failure results in created zip being deleted and acquired lock
         getting freed.
      """
      try:
         # Lock so that multiple scan/apply will not be concurrent.
         # Lock here as during directory creation itself as we don't want two
         # simultaneous extract depot operations to create multiple directories.
         self.currHostImage._getLock()
      except Errors.LockingError as e:
         logging.error("Extract depot failed. "
                       "Failed to get transaction lock: %s", e.msg)
         raise

      try:
         swSpec, notifications = self.getSwSpecAndNotifs()
         task.setProgress(10)
      except Exception as e:
         logging.error("Extract depot failed: %s", str(e))
         self.currHostImage._freeLock()
         raise

      # TODO: Currently we raise an error when Notification contains any
      # error messages as the UI does not handle this. This needs to change
      # once UI is able to consume Notifications and take action.
      # Tracked via - https://jira.eng.vmware.com/browse/ESXLCM-7097
      if notifications.errors is not None:
         msg = list()
         for error in notifications.errors:
            msg.append(error.message.default_message)
         logging.error("Exiting depot extract due to software info errors: %s",
                       ', '.join(msg))
         self.currHostImage._freeLock()
         raise Errors.SoftwareInfoExtractError(', '.join(msg))

      if not os.path.realpath('/var/vmware').startswith('/vmfs/volumes'):
         logging.error("Depot extraction failed: No available persistent storage.")
         raise Errors.DataStorageNotFound("Depot extraction failed: "
            "No OSData storage partition is available to extract depot. "
            "Configure persistent storage for the host and retry.")

      logging.debug("Creating directory %s", HOST_SEED_DIR_NAME)
      try:
         os.makedirs(HOST_SEED_DIR_NAME, exist_ok=True)
         task.setProgress(25)
      except Exception as e:
         logging.error("Extract depot failed. "
                       "Failed to create %s: %s", HOST_SEED_DIR_NAME, str(e))
         self.currHostImage._freeLock()
         raise

      self._cleanOldOfflineDepots()
      depotDir = os.path.join(HOST_SEED_DIR_NAME, 'recreateVibs')
      shutil.rmtree(depotDir, ignore_errors=True)
      timeStamp = datetime.today().strftime("%Y-%m-%d--%H.%M")
      bundleFile = '%s-%s.zip' % (DEPOT_FILE_NAME, timeStamp)
      bundlePath = os.path.join(HOST_SEED_DIR_NAME, bundleFile)

      try:
         resVibCache = ReservedVibCache()
      except NoVibCacheError:
         logging.info('No reserved VIB cache available on the host')
         resVibCache = None

      # Get a modified image profile which doesn't contain orphan vibs,
      # solutions and manifests
      newProfile = _getSeedImageProfile(self.currImageProfile)

      # Objects/data to help VIB generation
      args = dict(hostImage=self.currHostImage,
                  resVibIds=self.currImageProfile.reservedVibIDs,
                  resVibCache=resVibCache)
      try:
         Depot.DepotFromImageProfile(newProfile, depotDir,
                                     vibdownloadfn=Depot.GenerateVib,
                                     vendor='VMware, Inc.', vendorcode='vmw',
                                     generateRollupBulletin=False,
                                     vibDownloadArgs=args)
         task.setProgress(65)
         ob = OfflineBundle.OfflineBundle(depotDir)
         ob.Load()
         # TODO: passing checkacceptance as False here since an unsigned
         # vib such as vib-test-cert would cause failure at this point.
         ob.WriteBundleZip(bundlePath, checkacceptance=False)
         task.setProgress(90)
         logging.debug('Depot created successfully at %s', bundlePath)
      except Exception as e:
         logging.error("Extract depot failed: %s", str(e))
         if os.path.exists(bundlePath):
            self._deleteFile(bundlePath)
         raise
      finally:
         shutil.rmtree(depotDir, ignore_errors=True)
         self.currHostImage._freeLock()

      try:
         depotExtractObj = \
            InstalledImage.DepotExtractInfo(bundlePath, notifications, swSpec)
      except Exception as e:
         logging.error("Extract depot failed: %s", str(e))
         if os.path.exists(bundlePath):
            self._deleteFile(bundlePath)
         raise

      task.completeTask(result=vapiStructToJson(depotExtractObj))

class ReservedVibCache(object):
   """Class that manages cache of reserved VIBs.
   """
   _VMFS_VOLUMES = os.path.join(os.sep, 'vmfs', 'volumes')

   # ESXi 7.0 and later has /var/vmware -> /scratch/vmware.
   _DEFAULT_CACHE = HOST_SEED_DIR_NAME

   # ESXi 6.x has no /var/vmware, we will use /scratch.
   _SCRATCH = os.path.join(os.sep, 'scratch')
   _SCRATCH_HOSTSEED = os.path.join(_SCRATCH, 'lifecycle', _HOST_SEED_POSTFIX)

   def __init__(self):

      # All locations with reservedVibs, the first one is the default one.
      self._cacheLocations = []

      # Current cached VIB ID and location sorted by ID.
      self._currCachedVibs = dict()

      self._loadCachedVibs()

      # New cached VIBs, used to determine which VIBs to keep or revert
      # when a transaction is finalized.
      self._newCachedVibs = dict()

   def _loadCachedVibs(self):
      """Initiate information about cache locations and cached VIBs.
      """
      def dirExistsAndOnDisk(path):
         return (os.path.isdir(path) and
                 os.path.realpath(path).startswith(self._VMFS_VOLUMES))

      def makeAndAddResVibsDir(resVibDir):
         if not os.path.exists(resVibDir):
            os.makedirs(resVibDir)
         self._cacheLocations.append(resVibDir)

      if dirExistsAndOnDisk(os.path.dirname(self._DEFAULT_CACHE)):
         # ESXi 7.0 and later with disk-backed OSdata.
         resVibDir = os.path.join(self._DEFAULT_CACHE, 'reservedVibs')
         makeAndAddResVibsDir(resVibDir)
      elif dirExistsAndOnDisk(self._SCRATCH):
         # ESXi 6.x with disk-backed scratch.
         resVibDir = os.path.join(self._SCRATCH_HOSTSEED, 'reservedVibs')
         makeAndAddResVibsDir(resVibDir)
      else:
         # If there is no OSdata/scratch or the default location is not
         # not disk-backed, then there cannot be any additional locations.
         logging.warn('OSdata/scratch does not exist or is not backed by '
                      'disk storage, reserved VIBs will not be cached.')
         return

      # Find any additional lifecycle/reservedVibs dirs in non-active
      # OSData/VMFS-L partitions. We do not need to look for additional
      # VFAT scratch since one is enough for temporary transfer during
      # an upgrade.
      for fsPath in HostInfo.GetVmfslFileSystems():
         if os.path.realpath(self._cacheLocations[0]).startswith(fsPath):
            # Skip default OSdata.
            continue
         resVibsDir = os.path.join(fsPath, 'lifecycle', _HOST_SEED_POSTFIX,
                                   'reservedVibs')
         if os.path.isdir(resVibsDir):
            self._cacheLocations.append(resVibsDir)

      # Scan and record reserved VIBs.
      for cachePath in self._cacheLocations:
         for fn in os.listdir(cachePath):
            filePath = os.path.join(cachePath, fn)
            if os.path.isfile(filePath):
               try:
                  v = Vib.ArFileVib.FromFile(filePath)
                  self._currCachedVibs[v.id] = filePath
               except Exception as e:
                  logging.debug('File %s in reservedVibs directory is not a '
                                'VIB: %s', filePath, str(e))

   def getVibLocation(self, vibId):
      """Returns cached location of a VIB.
      """
      if not self._cacheLocations:
         raise NoVibCacheError('No VIB cache location is available')
      if not vibId in self._currCachedVibs and not vibId in self._newCachedVibs:
         raise VibNotInCacheError('VIB %s is not available in cached locations'
                                  % vibId)
      return (self._currCachedVibs.get(vibId, None) or
              self._newCachedVibs.get(vibId))

   def cacheVibs(self, hostImage, imageprofile, deployDir):
      """Invokes the appropriate handler to cache the reserved VIB from the
         incoming image profile. In case of a VUM upgrade, all the reserved
         VIBs are extracted from the resvibs.tgz in the deployDir. Otherwise,
         we go through each of the reserved VIB in the profile and add them
         to the cache.
      """
      if deployDir:
         self.extractResVibs(deployDir, hostImage)
      else:
         for vib in imageprofile.reservedVibs.values():
            try:
               self.addVib(vib, hostImage, deployDir)
            except Errors.VibRecreateError as e:
               # There can be legit reason why a reserved VIB cannot be
               # re-created, e.g. the VIB was created by older tools and
               # has timestamp in gzip header.
               logging.exception('Skip storing reserved VIB %s due to '
                                 're-creation failure', vib.id)
            except Exception as e:
               # XXX: ignore errors to not break an otherwise successful
               # transaction; we have to fix all other caveats/bugs before
               # this can be removed.
               logging.exception('Unexpected error when storing reserved '
                                 'VIB %s', vib.id)

   def addVib(self, vib, hostImage=None, isoDir=None):
      """Adds a VIB to the cache, input are VIB metadata object (required),
         HostImage reference for local installed VIB, and isoDir for VUM upgrade
         ISO folder.
      """
      if not self._cacheLocations:
         raise NoVibCacheError('No VIB cache location is available')

      def getCacheLocation(vibId):
         return os.path.join(self._cacheLocations[0], vibId) + '.vib'

      if not vib.payloads:
         # VIB has no payloads, can be an unit test.
         logging.info('Reserved VIB %s does not have any payloads, skip '
                      'caching.')
         return

      if vib.id in self._currCachedVibs:
         # VIB is present in the cache, but we want to move to the default
         # cache location.
         vibLocation = self._currCachedVibs[vib.id]

         if os.path.realpath(vibLocation).startswith(self._cacheLocations[0]):
            # Already in default cache.
            self._newCachedVibs[vib.id] = vibLocation
            return

         newCachePath = getCacheLocation(vib.id)
         logging.info('Moving VIB %s from %s to default cache location %s',
                      vib.id, os.path.dirname(vibLocation),
                      os.path.dirname(newCachePath))
         shutil.move(vibLocation, newCachePath)
         self._currCachedVibs[vib.id] = newCachePath
         self._newCachedVibs[vib.id] = newCachePath
         return

      if vib.id in self._newCachedVibs:
         # VIB just added.
         return

      cachePath = getCacheLocation(vib.id)

      vibDownloadFn = (Depot.VibDownloader if vib.remotelocations
                       else Depot.GenerateVib)
      vibDownloadFn(cachePath, vib, checkdigests=True,
                    extraArgs={'hostImage': hostImage,
                               'isoDir': isoDir,
                               'isReservedVib': True})
      self._newCachedVibs[vib.id] = cachePath

   def extractResVibs(self, isoDir, hostImage):
      """Extract and cache all the reserved VIBs present in the ISO/PXE image.
      """
      if not self._cacheLocations:
         raise NoVibCacheError('No VIB cache location is available')

      cachePath = self._cacheLocations[0]
      resvibsTar = hostImage.TryLowerUpperPath(isoDir, 'RESVIBS.TGZ')
      if resvibsTar:
         try:
            os.makedirs(cachePath, exist_ok=True)
            with tarfile.open(resvibsTar, 'r:gz') as t:
               for member in t.getmembers():
                  # Extract only the VIB file and not the complete directory.
                  if member.isfile():
                     member.name = os.path.basename(member.name)
                     t.extract(member, cachePath)
         except Exception as e:
            logging.exception('Unexpected error when storing reserved VIBs: %s',
                              str(e))

   def _safeRemove(self, filePath):
      """Safely remove a file.
      """
      if os.path.isfile(filePath):
         try:
            os.remove(filePath)
         except OSError as e:
            logging.warn('Failed to remove %s: %s', filePath, str(e))

   def revert(self):
      """Remove any newly added cache VIBs to revert to previous cache contents.
      """
      if not self._cacheLocations:
         # Nothing to do when there is no cache location.
         return

      for vibId, vibPath in self._newCachedVibs.items():
         if vibId not in self._currCachedVibs:
            self._safeRemove(vibPath)
      self._newCachedVibs.clear()

   def finalize(self):
      """Finalize the cache by keeping new cached VIBs and remove the rest.
      """
      if not self._cacheLocations:
         # Nothing to do when there is no cache location.
         return

      for vibId, vibPath in self._currCachedVibs.items():
         if vibId not in self._newCachedVibs:
            # Failure to remove a file would merely leave an unneeded VIB.
            self._safeRemove(vibPath)
      self._currCachedVibs = self._newCachedVibs.copy()
      self._newCachedVibs.clear()
