########################################################################
# Copyright (C) 2010-2021 VMware, Inc.                                 #
# All Rights Reserved                                                  #
########################################################################

"""This module defines a class for managing a collection of VIB depots,
   where each depot consists of index, vendor-index, and metadata.zip files.
"""
import logging
import os
import tempfile
import threading
import sys

if sys.version_info[0] >= 3:
   from urllib.request import url2pathname
   from urllib.parse import urlparse
else:
   from urllib import url2pathname
   from urlparse import urlparse

from . import Bulletin
from . import Depot
from . import Errors
from . import Vib
from . import VibCollection
from . import ImageProfile
from . import Downloader
from . import ReleaseCollection
from .ConfigSchema import ConfigSchemaCollection
from .Utils import PathUtils
from .Utils.Misc import isString
from . import Scan


log = logging.getLogger('DepotCollection')

def _readContent(f):
   """ Read content from the argument: a file descriptor or a Response object.
       The with clause closes the object.
   """
   with f:
      if hasattr(f, 'read'):
         indexxml = f.read()
      else:
         indexxml = b''
         for chunk in f.iter_content(1024):
            indexxml += chunk
   return indexxml

class OfflineBundleNotLocal(Exception):
   pass

class ProfileAlreadyExists(Exception):
   pass

class VibAlreadyExists(Exception):
   pass

class BaseImageAlreadyExists(Exception):
   pass

class AddonAlreadyExists(Exception):
   pass

class SolutionAlreadyExists(Exception):
   pass

class ManifestAlreadyExists(Exception):
   pass

## Not a real URL, this is used to describe the depot specially created
#  for custom channels created via AddChannel().
#  Note: it needs to use a supported scheme or urlparse() behavior is
#        unpredictable
CUSTOM_DEPOT_URL = 'file://custom/depot/'

class DepotCollection(object):
   """This class holds a collection of depots and their channels and
      metadata files.  It helps with constructing the depot hierarchy,
      managing the channels and metadata files, and
      querying and filtering on VIBs and image profiles within metadatas
      managed by this collection.

      Attributes:
         * depots   - A list of DepotIndex instances
         * channels - A dict of available channels, with the unique
                      channel ID as the key, and each value being an instance
                      of DepotChannel.
         * vibs     - A VibCollection of all Vib packages contained in all
                      the available channels.
         * profiles - An ImageProfileCollection for all profiles contained in
                      all the available channels.
         * vibscandata - An instance of VibScanner. Holds scan results for all
                         the VIBs.
         * baseimages  - A collection of base image objects.
         * addons      - A collection of addon objects.
         * solutions   - A collection of solution objects.
         * manifests   - A collection of manifest objects.

      Class Variables:
         * INVENTORY_*      - types of inventory collections.
         * INVENTORY_CLASS  - a map from inventory types to their collection
                              class names.
   """
   INVENTORY_VIBS = 'vibs'
   INVENTORY_PROFILES = 'profiles'
   INVENTORY_BULLETINS = 'bulletins'
   INVENTORY_BASEIMAGES = 'baseimages'
   INVENTORY_ADDONS = 'addons'
   INVENTORY_SOLUTIONS = 'solutions'
   INVENTORY_MANIFESTS = 'manifests'
   INVENTORY_CONFIGSCHEMAS = 'configSchemas'

   RELEASE_OBJECT_TYPE = ('vib', 'bulletin', 'baseimage', 'addon', 'solution',
                          'manifest')

   INVENTORY_CLASS = {
      INVENTORY_VIBS: VibCollection.VibCollection,
      INVENTORY_PROFILES: ImageProfile.ImageProfileCollection,
      INVENTORY_BULLETINS: Bulletin.BulletinCollection,
      INVENTORY_BASEIMAGES: ReleaseCollection.BaseImageCollection,
      INVENTORY_ADDONS: ReleaseCollection.AddonCollection,
      INVENTORY_SOLUTIONS: ReleaseCollection.SolutionCollection,
      INVENTORY_MANIFESTS: ReleaseCollection.ManifestCollection,
      INVENTORY_CONFIGSCHEMAS: ConfigSchemaCollection,
   }

   def __init__(self, depots=None, cachedir=None):
      """Constructs a new DepotCollection.
         Parameters:
            * depots   - An initial list of DepotIndex instances
                         to populate the DepotCollection with.
            * cachedir - Path to a directory for caching depot metadata files
         Returns:
            An instance of DepotCollection.
      """
      self._depots = Depot.DepotTreeNode(children=depots)
      self._urlToChannelMap = dict()
      self.cachedir = cachedir
      self._channels = dict()
      self._vibscandata = Scan.VibScanner()
      self._lock = threading.RLock()

      for inventoryName, cls in self.INVENTORY_CLASS.items():
         self._setInventory(inventoryName, cls())

      # Create a depot for custom channels. Make sure everything has a URL.
      self.customvendor = Depot.VendorIndex(name="Custom Depot", code="XXX",
                                            indexfile="custom-depot.xml",
                                            baseurl=CUSTOM_DEPOT_URL)
      self.customdepot = Depot.DepotIndex(
                                 url=self._TranslateDepotUrl(CUSTOM_DEPOT_URL),
                                 children=[self.customvendor])

   def _Lock(self):
      """ Wait for _lock is assigned: when this object is deeply copied, _lock
          is removed first since threading.RLock is not clonable.
      """
      while not self._lock:
         from time import sleep
         sleep(0.01)
      self._lock.acquire()

   def _Unlock(self):
      """ Wrapper that matches _Lock.
      """
      self._lock.release()

   def _safeget(self, attr):
      """Gets a class attribute with locking.
      """
      self._Lock()
      try:
         return getattr(self, attr)
      finally:
         self._Unlock()

   def _getInventorySafe(self, inventoryName):
      """Safely gets an internal inventory collection.
      """
      return self._safeget('_%s' % inventoryName)

   def _setInventory(self, inventoryName, val):
      """Sets an internal invetory collection without locking.
      """
      setattr(self, '_%s' % inventoryName, val)

   depots = property(lambda self: self._safeget('_depots').children)
   channels = property(lambda self: self._safeget('_channels'))
   vibscandata = property(lambda self: self._safeget('_vibscandata'))
   vibs = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_VIBS))
   profiles = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_PROFILES))
   bulletins = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_BULLETINS))
   baseimages = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_BASEIMAGES))
   addons = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_ADDONS))
   solutions = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_SOLUTIONS))
   manifests = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_MANIFESTS))
   configSchemas = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_CONFIGSCHEMAS))

   @staticmethod
   def _TranslateDepotUrl(durl):
      # Translates offline .zip file paths to a zip:/ URI
      # Adds /index.xml to the end of depot URLs which don't have it
      # NOTE: does not translate local paths to file:/// URI's
      if durl.lower().endswith('.zip'):
         if os.path.splitdrive(durl)[0] or PathUtils.IsDatastorePath(durl):
            # If this is a Windows path starting with a drive letter, we're
            # already done. We specifically check this before trying urlparse,
            # because urlparse will think the drive letter is the URL scheme.
            # Likewise, if this is a [datastore] path, don't mess with it;
            # PathUtils.CreateZipUrl will handle it.
            path = durl
         else:
            scheme, _, path = urlparse(durl)[:3]
            if scheme == '':
               # durl was actually just a relative path. Convert it to an
               # absolute one.
               path = os.path.abspath(durl)
            elif scheme == 'file':
               # durl was a file URL. Use url2pathname to convert the URL path
               # to an OS path.
               path = url2pathname(path)
            else:
               # durl was a URL, but not a file URL. We don't support this.
               raise OfflineBundleNotLocal
         durl = PathUtils.CreateZipUrl(path, 'index.xml')
      elif not durl.lower().endswith('.xml'):
         # Note: forward slashes can be mixed in to Windows pathnames, it
         # still works.
         durl = durl.rstrip('/').rstrip('\\') + '/index.xml'

      return durl


   @staticmethod
   def genFileURL(filePath):
      """ Generate file URL as depot absolute URL:
             zip file -- zip:/...?index.xml
             non zip file -- file://...
      """
      filePath = DepotCollection._TranslateDepotUrl(filePath)
      if os.path.exists(filePath):
         filePath = PathUtils.FilepathToUrl(filePath)
      return filePath

   @staticmethod
   def getDepotFilePaths(depotPaths):
      """ Convert file URLs and zip URLs to file paths.
      """
      for i in range(len(depotPaths)):
         url = depotPaths[i]
         if url.startswith('file:'):
            depotPaths[i] = PathUtils.FileURLToPath(url)
         elif url.startswith('zip:'):
            depotPaths[i] = PathUtils.ZipURLToPath(url)
      return depotPaths

   def ConnectDepots(self, depotUrls, timeout=None,
                     ignoreerror=True, validate=False):
      """Connects to depots and downloads the entire hierarchy of metadata
         files from index.xml down to every metadata.zip.  Parses all metadata
         files including VIB package and image profile metadata.  The
         vibs, profiles, and vibscandata properties will all be updated.
         Parameters:
            * depotUrls - The full remote URLs of the depot index.xml or local
                          offline bundle. If URL ends with '.zip', the URL will
                          be treated as pointing to offline bundle and local
                          file path is required. If the URL does not end
                          in .xml, it will be assumed to be a directory and
                          'index.xml' will be added.
            * timeout   - Network timeout in seconds
            * ignoreerror - If true, exception will be logged and will continue
                            to try to connect to the next depot URL.
            * validate - If True the function will enforce metadata validation
                         while parsing the metadata.zip file.
         Returns:
            A tuple.  First is a list of DepotIndex instances for the depots
            connected to.  The second element is a list of error instances
            for URLs that could not be connected to or parsed.  The error
            instances would be either MetadataDownloadError or
            MetadataFormatError.
            Note: an invalid offline .zip file (not in .zip format) yields
            MetadataDownloadError.
      """

      Downloader.Downloader.setEsxupdateFirewallRule('true')
      try:
         return self._connectDepots(depotUrls, timeout,
                                    ignoreerror, validate)
      finally:
         Downloader.Downloader.setEsxupdateFirewallRule('false')

   def _connectDepots(self, depotUrls, timeout=None,
                      ignoreerror=True, validate=False):
      vendorurls = list()
      connected = list()
      errors = list()
      vendorToDepot = dict()
      for durl in depotUrls:
         try:
            durl = self._TranslateDepotUrl(durl)
         except OfflineBundleNotLocal:
            msg = ('local file path is required for bundle zip file :'
                   '(%s) seems to be a remote URI' % (durl))
            log.info(msg)
            e = ValueError(msg)
            errors.append(e)
            if not ignoreerror:
               raise e
            continue
         try:
            log.debug("Downloading depot index.xml from %s" % (durl))
            d = Downloader.Downloader(durl, timeout=timeout)
            do = d.Open()

            # Use the updated Downloader().url to handle redirecting urls.
            durl = d.url
            depotindex = self.ParseDepotIndex(durl, do)
         except Downloader.DownloaderError as e:
            msg = "Could not download from depot at %s, skipping (%s)" % (
                     durl, str(e))
            log.info(msg)
            err = Errors.MetadataDownloadError(durl, '', msg)
            errors.append(err)
            if not ignoreerror:
               raise err
            continue
         except Exception as e:
            log.info("Error parsing depot index.xml from %s: %s" % (
                     durl, str(e)))
            errors.append(e)
            if not ignoreerror:
               raise
            continue

         connected.append(depotindex)
         vendorurls.extend(v.absurl for v in depotindex.children)
         depotUrl = self.genFileURL(durl)
         self._urlToChannelMap[depotUrl] = list()
         for v in depotindex.children:
            vendorToDepot[v.absurl] = depotUrl

      metaurls = list()
      for vurl in vendorurls:
         try:
            log.debug("Downloading vendor index.xml from %s" % (vurl))
            d = Downloader.Downloader(vurl, timeout=timeout)
            vindex = self.ParseVendorIndex(vurl, d.Open())
            if vindex:
               durl = vendorToDepot[vurl]
               channels = vindex.channels.values()
               self._urlToChannelMap.setdefault(durl, list()).extend(channels)
            else:
               log.debug('Empty depot for %s', vurl)
         except Downloader.DownloaderError as e:
            msg = ("Could not download from vendor index at %s, skipping (%s)"
                   % (vurl, str(e)))
            log.info(msg)
            err = Errors.MetadataDownloadError(vurl, '', msg)
            errors.append(err)
            if not ignoreerror:
               raise err
            continue
         except Exception as e:
            log.info("Error parsing vendor index.xml from %s: %s" % (
                      vurl, str(e)))
            errors.append(e)
            if not ignoreerror:
               raise
            continue

         metaurls.extend(m.absurl for m in vindex.children)

      for metaurl in metaurls:
         with tempfile.NamedTemporaryFile() as f:
            try:
               log.debug("Downloading metadata.zip from %s" % (metaurl))
               d = Downloader.Downloader(metaurl, local=f.name, fileobj=f,
                                         timeout=timeout)
               metafile = d.Get()
               metanode = self.ParseMetadata(metaurl, metafile,
                                             validate=validate)
            except Downloader.DownloaderError as e:
               msg = ("Could not download metadata.zip from %s, skipping (%s)"
                      % (metaurl, str(e)))
               log.info(msg)
               err = Errors.MetadataDownloadError(metaurl, '', msg)
               errors.append(err)
               if not ignoreerror:
                  raise err
            except Exception as e:
               log.info("Error parsing metadata.zip from %s: %s" % (
                  metaurl, str(e)))
               errors.append(e)
               if not ignoreerror:
                  raise

      self._buildCollections(scanvibs=True)

      return connected, errors

   def DisconnectDepots(self, depotUrls, isPman=False):
      """Disconnects from one or more depots.  The DepotIndexes specified
         will be removed from the depots property.
         Also, the vibs, profiles, and vibscandata properties will be updated.
         Parameters:
            * depotUrls   : The URLs of the depots to remove.
            * isPman      : Indicate the new behavior of generating URL; if not
                            set, keep old behavior. XXX: This is temporary; will
                            investigate and remove.
         Exceptions:
            IndexError - if one of the depotUrls is not found.
      """
      if isPman:
         depotUrls = [self.genFileURL(url) for url in depotUrls]
      toremove = set(depotUrls)
      delindex = list()
      self._Lock()
      needrebuild = False
      try:
         for i, depot in enumerate(self._depots.children):
            if depot.absurl in toremove:
               # Remove channels
               for vendoridx in depot.children:
                  for chan in vendoridx.channels.values():
                     try:
                        del self._channels[chan.channelId]
                     except KeyError as e:
                        if isPman:
                           # XXX: workaround of PR 2575639
                           log.warning('vLCM ignores the missing channel %s',
                                       str(e))
                        else:
                           raise
               delindex.append(i)
               toremove.remove(depot.absurl)
               needrebuild = True
         # Delete depots in reverse order
         while delindex:
            del self._depots.children[delindex.pop()]
      finally:
         if isPman:
            for url in depotUrls:
               if url not in toremove:
                  del self._urlToChannelMap[url]
         if needrebuild:
            self._buildCollections(scanvibs=True)
         self._Unlock()
      if len(toremove):
         raise IndexError('URLs %s could not be found in existing depots' %
                          (', '.join(list(toremove))))

   def ParseDepotIndex(self, depotIndexUrl, localpath):
      """Parses the depot index.xml file.  The depot will be added to the
         DepotCollection list or the existing entry will be updated.
         NOTE: Please handle HTTP redirects BEFORE this method is invoked.
               depotIndexUrl should be the redirected URL.  This URL will
               be kept and used for comparison later, so changes in the URL
               may result in duplicate or inconsistent state.
         Parameters:
            * depotIndexUrl - The full remote URL of the depot index.xml file.
                              Used to compute the URLs of vendor-indexes.
            * localpath     - The local path where the index.xml to be parsed
                              resides, or a file object
         Returns:
            The instance of DepotIndex that was created or updated.  To obtain
            the URLs of the vendor indexes, look up the absurl attribute of each
            VendorIndex in the 'children' list attribute of the returned
            instance.
         Exceptions:
            MetadataFormatError - cannot parse metadata
            FileIOError         - cannot access localpath
      """
      try:
         if isString(localpath):
            f = open(localpath, 'r')
         else:
            f = localpath
         xmlData = _readContent(f)
         index = Depot.DepotIndex.FromXml(xmlData, url=depotIndexUrl)
      except EnvironmentError as e:
         raise Errors.FileIOError(localpath, str(e))

      self._Lock()
      try:
         dindex = self._depots.AddChild(index)
      finally:
         self._Unlock()
      return dindex

   def ParseVendorIndex(self, vendorIndexUrl, localpath):
      """Parses a vendor-index.xml file and its channels and metadata file info.
         Inserts the information in the appropriate place in the depot
         hierarchy. Updates the channels attribute with the channels found here.
         NOTE: Please handle HTTP redirects BEFORE this method is invoked.
               depotIndexUrl should be the redirected URL.  This URL will
               be kept and used for comparison later, so changes in the URL
               may result in duplicate or inconsistent state.
         Parameters:
            * vendorIndexUrl - The full remote URL of the vendor-index.xml file.
                               NOTE: This must match the absurl in the child
                               nodes as returned by ParseDepotIndex() above
                               in order for insertion to work correctly.
            * localpath      - The local path where the vendor-index.xml file
                               to be parsed resides, or a file object
         Returns:
            An instance of VendorIndex.  The child nodes contains information
            about metadata URL as well as channels.
         Exceptions:
            MetadataFormatError - cannot parse metadata
            FileIOError         - cannot access localpath
            IndexError - None of the depots contains the vendorIndexUrl passed
                         in.
      """
      index = None
      for depot in self._depots.children:
         #
         # For each depot, does it have the vendor-index at vendorIndexUrl?
         index = depot.GetChildFromUrl(vendorIndexUrl)
         if index:
            try:
               if isString(localpath):
                  f = open(localpath, 'r')
               else:
                  f = localpath
               xml = _readContent(f)
               vidx = Depot.VendorIndex.FromXml(xml, url=vendorIndexUrl)
            except EnvironmentError as e:
               raise Errors.FileIOError(localpath, str(e))

            self._Lock()
            try:
               index = depot.AddChild(vidx)
               self._AddChannels(index)
            finally:
               self._Unlock()
            return index

      # None of the depots contained vendorIndexUrl.  Just give up.
      raise IndexError("Vendor-index URL %s not found in any depot"
                       % vendorIndexUrl)

   def _AddChannels(self, vendorindex):
      # Prune channels in self._channels that don't belong
      vchannels = vendorindex.channels
      rm_list = []
      for channel in self._channels.values():
         if channel.vendorindex.absurl == vendorindex.absurl:
            if channel.name not in vchannels:
               rm_list.append(channel.channelId)
      for channelId in rm_list:
         del self._channels[channelId]

      # Adds the channels from vendorindex to self._channels
      for channel in vchannels.values():
         self._channels[channel.channelId] = channel

   def ParseMetadata(self, metadataUrl, localpath, validate=False):
      """Parses a metadata.zip file.  The VIB packages and image profiles
         found in the metadata.zip file will be subsequently available
         for query and scan operations.  The vibs and profiles properties
         will be updated.
         Parameters:
            * metadataUrl - The full remote URL of the metadata.zip file.
                            Will be used to compute VIB remotelocations.
                            NOTE: This must match the absurl in the child
                            nodes as returned by ParseVendorIndex() above
                            in order for insertion to work correctly.
            * localpath   - The local path where the metadata.zip file
                            to be parsed resides.
            * validate    - If True the function will enforce metadata
                            validation while parsing the metadata.zip file.
         Returns:
            An instance of Depot.MetadataNode
         Exceptions:
            MetadataFormatError - cannot parse metadata.zip
            MetadataIOError     - cannot access or read localpath
            IndexError          - None of the vendor-indexes contain the
                                  metadataUrl passed in.
      """
      # Find the right MetadataNode to replace
      # Note this assumes each MetadataNode has a unique absurl
      for depot in self._depots.children:
         for vidx in depot.children:
            metanode = vidx.GetChildFromUrl(metadataUrl)
            if metanode:
               self._Lock()
               try:
                  metanode.ReadMetadataZip(localpath, validate=validate)
                  metanode.SetVibRemoteLocations(metadataUrl)
               finally:
                  self._Unlock()
               self._buildCollections(scanvibs=False)
               return metanode

      raise IndexError("Metadata URL %s not found in the depot" % (metadataUrl))

   def GetChannelsByName(self, names):
      """Returns a set of channel IDs coresponding to the given channel names.
         Parameters:
            * names - an iterable of strings for the channel names to search.
         Returns:
            A set of channel IDs, or the empty set if none of the names
            correspond to an existing channel.  If two channels contain the same
            channel name, they will both be returned.
      """
      channels = set()
      for chanID, channel in self._channels.items():
         if channel.name in names:
            channels.add(chanID)
      return channels

   def GetChannelsByDepotUrl(self, urls):
      """Returns a set of channel IDs for channels in the given depots.
         Parameters:
            * urls - an iterable of depot URLs whose channels should be
                     returned.
         Returns:
            A set of channel IDs, or the empty set of none of the depot URLs
            are valid or contain channels.
      """
      channels = set()
      for url in urls:
         durl = self._TranslateDepotUrl(url)
         if PathUtils.IsAbsolutePath(durl):
            durl = PathUtils.FilepathToUrl(durl)
         depot = self._depots.GetChildFromUrl(durl)
         if depot:
            for vendor in depot.children:
               channels.update(set(c.channelId for c in
                                   vendor.channels.values()))
      return channels

   def _getByChannelId(self, channelId, inventoryName):
      """Returns all objects of one inventory type from a given channel
         as a collection.
         Parameters:
            * channelId     - The channel GUID, from the keys of self.channels
                              or channelId property of a DepotChannel.
            * inventoryName - One of INVENTORY_* constants.
      """
      collection = self.INVENTORY_CLASS[inventoryName]()
      for meta in self.channels[channelId].metadatas:
         collection += getattr(meta, inventoryName)
      return collection

   def _getByDepots(self, depotUrls, inventoryName):
      """Returns all same type objects of one inventory type from one
        or more depots.
         Parameters:
            * depotUrls     - The URLs for the interested depots.
            * inventoryName - One of INVENTORY_* constants.
      """
      collection = self.INVENTORY_CLASS[inventoryName]()
      depotFilter = set(self._TranslateDepotUrl(depotUrl)
                        for depotUrl in depotUrls)
      for depot in self._depots.children:
         if depot.absurl in depotFilter:
            for vendorindex in depot.children:
               for meta in vendorindex.children:
                  collection += getattr(meta, inventoryName)
      return collection

   def GetVibsByChannelId(self, channelId):
      """Returns all the VIBs from a given channel as a VibCollection.
         Parameters:
            * channelId - the channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of VibCollection.
         Raises:
            KeyError, if channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_VIBS)

   def GetVibsByDepots(self, depotUrls):
      """Returns all the VIBs from one or more depots.
         Parameters:
            * depotUrls - the URLs for the interested depots.
         Returns:
            An instance of VibCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_VIBS)

   def GetProfilesByChannelId(self, channelId):
      """Returns all the image profiles from a given channel.
         Parameters:
            * channelId - the channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of ImageProfileCollection.
         Raises:
            KeyError, if channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_PROFILES)

   def GetProfilesByDepots(self, depotUrls):
      """Returns all the image profiles from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of ImageProfileCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_PROFILES)

   def GetBulletinsByChannelId(self, channelId):
      """Returns all the bulletins from a given channel.
         Parameters:
            * channelId - the channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of BulletinCollection.
         Raises:
            KeyError, if channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_BULLETINS)

   def GetBulletinsByDepots(self, depotUrls):
      """Returns all bulletins from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of BulletinCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_BULLETINS)

   def GetBaseImagesByChannelId(self, channelId):
      """Returns all the base images from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of BaseImageCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_BASEIMAGES)

   def GetBaseImagesByDepots(self, depotUrls):
      """Returns all the base images from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of BaseImageCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_BASEIMAGES)

   def GetAddonsByChannelId(self, channelId):
      """Returns all the addons from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of AddonCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_ADDONS)

   def GetAddonsByDepots(self, depotUrls):
      """Returns all the addons from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of AddonCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_ADDONS)

   def GetSolutionsByChannelId(self, channelId):
      """Returns all the solutions from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of SolutionCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_SOLUTIONS)

   def GetSolutionsByDepots(self, depotUrls):
      """Returns all the solutions from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of SolutionCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_SOLUTIONS)

   def GetManifestsByChannelId(self, channelId):
      """Returns all the manifests from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of ManifestCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_MANIFESTS)

   def GetManifestsByDepots(self, depotUrls):
      """Returns all the manifests from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots
         Returns:
            An instance of ManifestCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_MANIFESTS)

   def ScanVibs(self):
      """Populates the vibscandata property by scanning all the VIB package
         dependencies of the internal vibs collection.
      """
      self._Lock()
      try:
         self._vibscandata = self._vibs.Scan()
      finally:
         self._Unlock()

   def AddChannel(self, name):
      """Adds a new channel to the depotcollection.  This is mostly designed
         for persisting user-created image profiles.  A special URL will be
         created for the channel.
         The channel will be added to a common depot created for the purpose
         of persisting custom channels.  The common depot and all custom
         channels can be removed using DisconnectDepots.
         If the channel already exists, then this is a NOP.
         Parameters:
            * name       - A friendly name for the new channel
         Returns:
            The new DepotChannel instance.
      """
      metaurl = CUSTOM_DEPOT_URL + name
      metanode = Depot.MetadataNode(productId=Depot.DEPOT_PRODUCT,
                                    channels=[name],
                                    url=metaurl)
      log.debug("Adding custom channel %s" % (metanode.absurl))

      with self._lock:
         self._depots.AddChild(self.customdepot)
         self.customvendor.AddChild(metanode)
         self._AddChannels(self.customvendor)
      return self.customvendor.channels[name]

   def AddProfile(self, profile, channelId, replace=True):
      """Adds or replaces an image profile in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * profile   - an ImageProfile instance to add/replace.
            * channelId - the channel ID of the channel to affect
            * replace   - if True, allows an existing image profile
                          to be replaced, where existing means an image profile
                          with the same ID.  If False, no profile with the same
                          ID can exist in this DepotCollection.
         Raises:
            KeyError    - if the channelId does not correspond to a valid
                          channel.
            ProfileAlreadyExists - if the image profile already exists and
                                   replace is False.
            ValueError  - The image profile to be replaced is read-only and
                          cannot be modified.
      """
      if profile.profileID not in self._profiles:
         # A new profile.  Add it to first meta of the channel.
         with self._lock:
            meta = self._channels[channelId].metadatas[0]
            meta.profiles[profile.profileID] = profile
            self._profiles[profile.profileID] = profile
      elif replace:
         # Profile exists already and we can replace it
         if not self._profiles[profile.profileID].readonly:
            with self._lock:
               for meta in self._channels[channelId].metadatas:
                  if profile.profileID in meta.profiles:
                     meta.profiles[profile.profileID] = profile
               self._profiles[profile.profileID] = profile
         else:
            raise ValueError("Image Profile '%s' is read-only and cannot be "
                             "updated"
                             % (self._profiles[profile.profileID].name))
      else:
         raise ProfileAlreadyExists("Image Profile '%s' already exists and "
                                    "replace=False"
                             % (self._profiles[profile.profileID].name))

   def RemoveProfile(self, profileID, channelname=''):
      """Removes an image profile from the collection for a channel. If
         channelname is empty, remove the image profile from all the channels.
         Parameters:
            * profileID - the ID of the profile to remove.
            * channelname - the name of the channel from which to remove the
                            profile.
         Raises:
            KeyError - the profile does not exist in the collection.
            ValueError - the profile is not found in the specified channel
      """
      with self._lock:
         keep = False
         deleted = False
         for chan in self._channels.values():
            for meta in chan.metadatas:
              if profileID in meta.profiles:
                 if len(channelname) == 0 or chan.name == channelname:
                    del meta.profiles[profileID]
                    deleted = True
                 else:
                    keep = True

         # Since we only support removing from customer channel. If the profile
         # is not in the specified channel but in other channels, which means
         # the profile cannot be removed as it is from a real depot.
         if keep and not deleted:
            raise ValueError("The image profile '%s' cannot be removed as it "
               "is from a read-only depot." % (self._profiles[profileID].name))

         # Not in any channels, remove it from top-level
         if not keep:
            del self._profiles[profileID]

   def _AddReleaseUnit(self, releaseUnit, channelId, replace,
                       inventoryType, errorType):
      """Common method for adding base image, addon, solution or manifest into
         depot collection.
      """
      def _setReleaseUnit(meta, releaseUnit):
         getattr(meta, inventoryType)[releaseUnit.releaseID] = releaseUnit

      releaseID = releaseUnit.releaseID
      if releaseID not in getattr(self, inventoryType):
         # Add/replace a release unit to first meta of the channel.
         with self._lock:
            meta = self._channels[channelId].metadatas[0]
            _setReleaseUnit(meta, releaseUnit)
            _setReleaseUnit(self, releaseUnit)
      elif replace:
         # Release unit exists already and we can replace it
         with self._lock:
            for meta in self._channels[channelId].metadatas:
               if releaseID in getattr(meta, inventoryType):
                  _setReleaseUnit(meta, releaseUnit)
            _setReleaseUnit(self, releaseUnit)
      else:
         raise errorType("%s'%s' already exists" %
                         (releaseUnit.releaseType, releaseID))

   def _RemoveReleaseUnit(self, releaseID, channelName, inventoryName):
      """Common method for removing base image, addon, solution or manifest from
         depot collection.
      """
      def _delReleaseUnit(meta, releaseID):
         del getattr(meta, inventoryName)[releaseID]

      with self._lock:
         keep = False
         deleted = False
         for chan in self._channels.values():
            for meta in chan.metadatas:
               if releaseID in getattr(meta, inventoryName):
                  if len(channelName) == 0 or chan.name == channelName:
                     _delReleaseUnit(meta, releaseID)
                     deleted = True
                  else:
                     keep = True

         # Since we only support removing from customer channel. If the release
         # unit is not in the specified channel but in other channels, which
         # means the release unit cannot be removed as it is from a real depot.
         if keep and not deleted:
            raise ValueError("The release unit '%s' cannot be removed as it is "
                             "from a read-only depot." % (releaseID))

         # Not in any channels, remove it from top level.
         if not keep and releaseID in getattr(self, inventoryName):
            _delReleaseUnit(self, releaseID)

   def AddBaseImage(self, baseimage, channelId, replace=True):
      """Adds or replaces a base image in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * baseimage - A BaseImage object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing base image to be replaced,
                          where existing means a base image with the same
                          releaseID.  If False, no base image with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError - If the channelId does not correspond to a valid channel
            BaseImageAlreadyExists -
               If the base image already exists and replace is False.
      """
      self._AddReleaseUnit(baseimage, channelId, replace,
                           self.INVENTORY_BASEIMAGES,
                           BaseImageAlreadyExists)

   def RemoveBaseImage(self, releaseID, channelname=''):
      """Removes a base image from the collection for a channel. If channelname
         is empty, remove the base image from all the channels.
         Parameters:
            * releaseID   - The ID of the base image to remove.
            * channelname - The name of the channel from which to remove the
                 base image.
         Raises:
            KeyError   - The base image does not exist in the collection.
            ValueError - The base image is not found in the specified channel
      """
      self._RemoveReleaseUnit(releaseID, channelname, self.INVENTORY_BASEIMAGES)

   def AddAddon(self, addon, channelId, replace=True):
      """Adds or replaces an addon in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * addon     - An Addon object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing addon to be replaced,
                          where existing means an addon with the same
                          releaseID.  If False, no addon with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError    - If the channelId does not correspond to a valid
                          channel.
            AddonAlreadyExists  - If the addon already exists and replace
                          is False.
      """
      self._AddReleaseUnit(addon, channelId, replace, self.INVENTORY_ADDONS,
                           AddonAlreadyExists)

   def RemoveAddon(self, releaseID, channelname=''):
      """Removes an addon from the collection for a channel. If channelname
         is empty, remove the addon from all the channels.
         Parameters:
            * releaseID   - The ID of the addon to remove.
            * channelname - The name of the channel from which to remove the
                  addon.
         Raises:
            KeyError   - The addon does not exist in the collection.
            ValueError - The addon is not found in the specified channel
      """
      self._RemoveReleaseUnit(releaseID, channelname, self.INVENTORY_ADDONS)

   def AddSolution(self, solution, channelId, replace=True):
      """Adds or replaces a solution in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * solution  - A Solution object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing solution to be replaced,
                          where existing means a solution with the same
                          releaseID.  If False, no solution with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError    - If the channelId does not correspond to a valid
                          channel.
            SolutionAlreadyExists  - If the solution already exists and replace
                                     is False.
      """
      self._AddReleaseUnit(solution, channelId, replace,
                           self.INVENTORY_SOLUTIONS, SolutionAlreadyExists)

   def RemoveSolution(self, solutionID, channelname=''):
      """Removes a solution from the collection for a channel. If channelname
         is empty, remove the solution from all the channels.
         Parameters:
            * solutionID   - The ID of the solution to remove.
            * channelname  - The name of the channel from which to remove the
                             solution.
         Raises:
            KeyError   - The solution does not exist in the collection.
            ValueError - The solution is not found in the specified channel
      """
      self._RemoveReleaseUnit(solutionID, channelname, self.INVENTORY_SOLUTIONS)

   def AddManifest(self, manifest, channelId, replace=True):
      """Adds or replaces a manifest in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * manifest  - A Manifest object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing manifest to be replaced,
                          where existing means a manifest with the same
                          releaseID.  If False, no manifest with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError    - If the channelId does not correspond to a valid
                          channel.
            ManifestAlreadyExists - If the manifest already exists and replace
                                    is False.
      """
      self._AddReleaseUnit(manifest, channelId, replace,
                           self.INVENTORY_MANIFESTS, ManifestAlreadyExists)

   def RemoveManifest(self, manifestID, channelname=''):
      """Removes a manifest from the collection for a channel. If channelname
         is empty, remove the manifest from all the channels.
         Parameters:
            * manifestID   - The ID of the manifest to remove.
            * channelname  - The name of the channel from which to remove the
                             manifest.
         Raises:
            KeyError   - The manifest does not exist in the collection.
            ValueError - The manifest is not found in the specified channel
      """
      self._RemoveReleaseUnit(manifestID, channelname, self.INVENTORY_MANIFESTS)

   def AddVib(self, vib, channelId, replace=True):
      """Adds or replaces a VIB in an existing channel. The channel should have
         have been added using AddChannel above.
         Parameters:
            * vib       - a BaseVib instance to add/replace.
            * channelId - the channel ID of the channel to affect
            * replace   - if True, allows an existing VIB with the same ID to be
                          replaced. If False, no VIB with the same ID can exist
                          in this DepotCollection.
         Raises:
            KeyError    - if the channelId does not correspond to a valid
                          channel.
            VibAlreadyExists - if the VIB already exists and replace is
                               False.
      """
      if vib.id not in self._vibs:
         # A new VIB. Add it to first meta of the channel.
         with self._lock:
            meta = self._channels[channelId].metadatas[0]
            meta.vibs[vib.id] = vib
            self._vibs[vib.id] = vib
      elif replace:
         # VIB exists already and we can replace it
         with self._lock:
            for meta in self._channels[channelId].metadatas:
               if vib.id in meta.vibs:
                  meta.vibs[vib.id] = vib
            self._vibs[vib.id] = vib
      else:
         raise VibAlreadyExists("VIB '%s' already exists and replace=False"
                                % (self._vibs[vib.id].name))
      self._buildCollections(scanvibs=True)

   def AddVibFromUrl(self, url, channelId, replace=True):
      try:
         d = Downloader.Downloader(url)
         vib = Vib.ArFileVib.FromFile(d.Open())
         vib.remotelocations = [url]
         # We don't know how much time is going to elapse before we try to
         # do anything with VIB payloads (like export an image profile), so
         # just close the file. We'll re-open it when it's needed.
         vib.Close()
      except Downloader.DownloaderError as e:
         raise Errors.VibDownloadError(url, None, str(e))
      self.AddVib(vib, channelId, replace)
      return vib

   def RemoveVib(self, vibID):
      """Removes an VIB from the collection.
         Parameters:
            * vibID - the ID of the VIB to remove.
         Raises:
            KeyError - the VIB does not exist in the collection.
      """
      with self._lock:
         del self._vibs[vibID]
         for chan in self._channels.values():
            for meta in chan.metadatas:
               try:
                  del meta.vibs[vibID]
               except KeyError:
                  pass
      self._buildCollections(scanvibs=True)

   def GetBaseImage(self, releaseID):
      ''' Get the base image with provided releaseID. '''
      return self.baseimages[releaseID]

   def GetAddon(self, releaseID):
      ''' Get the addon with provided release ID. '''
      return self.addons[releaseID]

   def _buildCollections(self, scanvibs=False):
      ''' Rebuilds collections of VIBs, profiles, bulletins and release units.
          If scanvibs is True, also rebuild the vibscandata.
      '''
      #
      # It is OK if there are duplicate metadata.zips in different channels.
      # The VibCollection Merge method would simply add the same Vib instances
      # together, which results in a VibCollection with the same contents.
      #
      self._Lock()
      try:
         for inventoryName, cls in self.INVENTORY_CLASS.items():
            collection = cls()
            for channel in self._channels:
               collection += self._getByChannelId(channel, inventoryName)
            self._setInventory(inventoryName, collection)

         if scanvibs:
            self.ScanVibs()
      finally:
         self._Unlock()


   def CalculateMicroDepots(self, imageProfile, getURL=False):
      """ Calculate the micro depots that contain all image related objects in
          the provided image profile.

          Return value can be in url format or file path format, depending on
          the argument getURL.
      """
      depotPaths = set()
      knownCompIDs = set(imageProfile.components.GetComponentNameIds())
      knownCompIDs.update(imageProfile.reservedComponents.GetComponentNameIds())

      # Collect all release units into a set of (type, ID).
      releaseUnits = imageProfile.GetAllReleaseUnits()

      for url in self._urlToChannelMap:
         for channel in self._urlToChannelMap[url]:
            # Find and remove release units.
            if channel.RemoveMatchedReleaseUnits(releaseUnits):
               depotPaths.add(url)

            # Find and remove component IDs
            if channel.RemoveMatchedComponentIDs(knownCompIDs):
               depotPaths.add(url)

      if releaseUnits:
         msg = ('The following data are not in depots: \n\t%s' %
                '\n\t'.join([', '.join(relTupe, relID)
                             for relTupe, relID in releaseUnits]))
         raise Errors.MetadataNotFoundError(releaseUnits, msg)

      if knownCompIDs:
         msg = ('The following componnets are not in depots: \n\t%s' %
                '\n\t'.join(knownCompIDs))
         raise Errors.MetadataNotFoundError(knownCompIDs, msg)

      if getURL:
         return list(depotPaths)
      return self.getDepotFilePaths(list(depotPaths))

   def GetRelatedVibs(self, imageProfile):
      """ Generate a VibCollection that only contains the vibs from the
          micro depots that overlap with the provided image profile.
      """
      relatedVibs = VibCollection.VibCollection()
      urls = self.CalculateMicroDepots(imageProfile, True)
      for url in urls:
         for channel in self._urlToChannelMap[url]:
            for meta in channel.metadatas:
               relatedVibs += meta.vibs
      return relatedVibs

   def GetReleaseObjects(self, depotUrl):
      """ Get all collections of release objects for a depot.
      """
      self._Lock()
      try:
         depotUrl = self.genFileURL(depotUrl)
         if depotUrl not in self._urlToChannelMap:
            raise Errors.DepotNotFoundError(
                     'Depot not found in depot collection', depotUrl)
         releaseObjects = {rot : self.INVENTORY_CLASS[rot + 's']()
                           for rot in self.RELEASE_OBJECT_TYPE}
         for channel in self._urlToChannelMap[depotUrl]:
            for meta in channel.metadatas:
               for rot in self.RELEASE_OBJECT_TYPE:
                  releaseObjects[rot] += meta.GetReleaseObjects(rot)
         return releaseObjects
      finally:
         self._Unlock()

   def GetDepotURLs(self):
      """ Get all depot URLs.
      """
      self._Lock()
      try:
         return self.getDepotFilePaths(list(self._urlToChannelMap.keys()))
      finally:
         self._Unlock()

   def GetVibConfigSchemas(self, vibs):
      """ Retrieve config schemas for the given VIBs.

         Parameters:
            vibs: A VibCollection containing the VIBs for which the config
                  schemas are retrieved for.
         Returns:
            The retrieved config schemas for the given VIBs
      """
      vibConfigSchemas = ConfigSchemaCollection()

      missingVibIds = set()
      for vib in vibs.values():
         csTag = vib.GetConfigSchemaTag()
         if csTag is not None:
            if csTag.schemaId in self.configSchemas:
               vibConfigSchemas.AddConfigSchema(self.configSchemas[csTag.schemaId])
            else:
               missingVibIds.add(vib.id)

      if missingVibIds:
         log.error("Failed to find config schema for VIB(s): %s",
                   ", ".join(missingVibIds))

      return vibConfigSchemas if vibConfigSchemas else None
