vmcaster

#!/usr/bin/env python
import sys

import vmcasterpub.dishpubdb as model
import vmcasterpub.state as dishpubstate
import vmcasterpub.uploader as uploader
from vmcasterpub.versioning import bumpVersion
from vmcasterpub.process import hostUploader, InputError
from vmcasterpub.uglyuri import uglyUriParser,uglyUriBuilder
import os.path
import logging
import optparse
from vmcasterpub.__version__ import version
import urllib2
import urllib
import hashlib
import datetime
import os, statvfs
import shutil
import commands
try:
    import simplejson as json
except:
    import json

import subprocess
import time
import types
# needed for the signing of images.
import M2Crypto.BIO
import M2Crypto.SMIME
import M2Crypto.X509
import tempfile
import urlparse
import magic

import optparse





def checkVoms(requiredExtensions = set([])):
    log = logging.getLogger("vomscheck")
    cmd = "voms-proxy-info  --all"
    process = subprocess.Popen([cmd], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    processRc = None
    handleprocess = True
    counter = 0
    stdout = ''
    stderr = ''
    while handleprocess:
        counter += 1
        time.sleep(1)
        cout,cerr = process.communicate()
        stdout += cout
        stderr += cerr
        process.poll()
        processRc = process.returncode
        if processRc != None:
            break
        if counter == timeout:
            os.kill(process.pid, signal.SIGQUIT)
        if counter > timeout:
            os.kill(process.pid, signal.SIGKILL)
            processRc = -9
            break

    if processRc != 0:
        log.error("Failed to run voms-proxy-info sucessfully")
        log.info("stdout:%s" % (stdout))
        log.info("stderr:%s" % (stderr))
        return None
    vomsInfo = {}
    foundVos = set([])
    issuer = None
    identity = None
    for lineUnclean in stdout.split('\n'):
        foundPos = lineUnclean.find(':')
        if foundPos > 0:
            head = lineUnclean[:foundPos].strip(' \t\n\r')
            tail = lineUnclean[(foundPos +1):].strip(' \t\n\r')
            if head == 'timeleft':
                if tail == '0:00:00':
                    log.error("Proxy expired.")
                    return None
            if head == 'VO':
                foundVos.add(tail)
            if head == 'identity':
                identity = tail
    if len(requiredExtensions.difference(foundVos)) > 0:
        log.error("not all extensions found")
        return None
    return identity

def main():
    """Runs program and handles command line options"""
    p = optparse.OptionParser(version = "%prog " + version)
    p.add_option('-d', '--database', action ='store', help='Database conection string')
    p.add_option('-L', '--logcfg', action ='store',help='Logfile configuration file.', metavar='CFG_LOGFILE')
    p.add_option('-v', '--verbose', action ='count',help='Change global log level, increasing log output.', metavar='LOGFILE')
    p.add_option('-q', '--quiet', action ='count',help='Change global log level, decreasing log output.', metavar='LOGFILE')
    p.add_option('-C', '--config-file', action ='store',help='Configuration file.', metavar='CFG_FILE')
    p.add_option('--x509-dir', action ='store',help='Directory for x.509 certificates and keys.', metavar='VMCASTER_X509_DIR')
    p.add_option('--x509-cert', action ='store',help='Path of x.509 Certificate.', metavar='VMCASTER_X509_CERT')
    p.add_option('--x509-key', action ='store',help='Path of x.509 Key.', metavar='VMCASTER_X509_KEY')
    p.add_option('--flag', action ='append',help='Upload flags', metavar='UPLOAD_FLAG')
    p.add_option('--list-imagelist', action ='store_true',help='list image lists by "dc:identifier".')
    p.add_option('--list-image', action ='store_true',help='list Images by "dc:identifier".')
    p.add_option('--list-endorser', action ='store_true',help='list Endorsers by "hv:dn".')

    p.add_option('--import-imagelist-smime', action ='store',help='Import a signed imagelist from path.', metavar='IMAGE_PATH')
    p.add_option('--import-imagelist-json', action ='store',help='Import an image list as json.', metavar='IMAGE_PATH')

    p.add_option('--upload-image', action ='store',help='Path for image to be uploaded. The image format should be detected automatically and compressed if needed.')
    p.add_option('--upload-imagelist', action ='store_true',help="""Update the image list with a newly signed image list.
Expect to type in your grid certificate password to sign a new image list, and then transfer the new image list and replace the master copy.""")

    p.add_option('--download-imagelist', action ='store_true',help="Download the imagelist to allow updating from multiple image generators.")
    p.add_option('--download-imagelist-flags', action ='store',help="""Imagelist download merging flags.
Use with --download-imagelist""")

    p.add_option('--lifetime-imagelist', action ='store',help="""Set the length of the imagelist lifetime in days. To be used in conjuction with '--upload-imagelist'. Defaults to 28 days.""", metavar='VMCASTER_IMAGELIST_LIFETIME')
    p.add_option('--select-imagelist', action ='store',help='Select "imagelist" by "dc:identifier".', metavar='IMAGELIST_UUID')
    p.add_option('--select-image', action ='store',help='Select "image" by "dc:identifier"', metavar='IMAGELIST_UUID')
    p.add_option('--select-endorser', action ='store',help='Select "endorser" by "hv:dn".', metavar='ENDORSER_UUID')

    p.add_option('--show-imagelist', action ='store_true',help='Write json image list. This option requires an "image list" selection.')
    p.add_option('--show-image', action ='store_true',help='Write json image metadata. This option requires an "image" selection.')
    p.add_option('--show-endorser', action ='store_true',help='Write json endorser metadata by "hv:dn". This option requires an "endorser" selection.')

    p.add_option('--add-imagelist', action ='store_true',help='Image list create. This option requires an "image list" selection.')
    p.add_option('--add-image', action ='store_true',help='Image create.')
    p.add_option('--add-endorser', action ='store_true',help='Endorser create.')

    p.add_option('--del-imagelist', action ='store_true',help='Image list delete. This option requires an "image list" selection.')
    p.add_option('--del-image', action ='store_true',help='Image delete. This option requires an "image" selection.')
    p.add_option('--del-endorser', action ='store',help='Endorser delete.')

    p.add_option('--imagelist-add-endorser', action ='store_true',help='Bind endorser to image list. This option requires an "image list" and an "endorser" selection.')
    p.add_option('--imagelist-del-endorser', action ='store_true',help='Unbind endorser from image list. This option requires an "image list" and an "endorser" selection.')
    p.add_option('--imagelist-list-endorser',action ='store_true',help='List bound image from image list. This option requires an "image list" selection.')

    p.add_option('--imagelist-add-image', action ='store_true',help='Bind image to image list. This option requires an "image list" and an "image" selection.')
    p.add_option('--imagelist-del-image', action ='store_true',help='Unbind image from image list. This option requires an "image list" and an "image" selection.')
    p.add_option('--imagelist-list-image',action ='store_true',help='List bound image from image list. This option requires an "image list" selection.')


    p.add_option('--keys-imagelist', action ='store_true',help='List image list metadata keys. . This option requires an "image" selection.')
    p.add_option('--keys-image', action ='store_true',help='List image metadata keys. This option requires an "image" selection.')
    p.add_option('--keys-endorser', action ='store',help='List endorser metadata keys. This option requires an "endorser" selection.')
    p.add_option('--key-set-endorser', action ='store',help="""Endorser metadata key to create/overwrite.
As guidance keys should follow a pattern of "namespace:type".
This option requires an "endorser" selection and a "--key-value-endorser" """)
    p.add_option('--key-set-imagelist', action ='store',help="""Image list metadata key to create/overwrite.
As guidance keys should follow a pattern of "namespace:type".
This option requires an "image list" selection and a "--key-value-imagelist" parameter.""")
    p.add_option('--key-set-image', action ='store',help="""Image metadata key to create/overwrite.
As guidance keys should follow a pattern of "namespace:type".
This option requires an "image" selection and a "--key-value-image" parameter.""")

    p.add_option('--key-del-imagelist', action ='store',help='image list metadata key to delete. This option requires an "image" selection.')
    p.add_option('--key-del-image', action ='store',help='Image metadata key to delete. This option requires an "image" selection.')
    p.add_option('--key-del-endorser', action ='store_true',help='Endorser metadata key to delete. This option requires an "endorser" selection.')

    p.add_option('--key-value-imagelist', action ='store',help='Image list metadata value to set. This option requires an "imagelist" selection and a "--key-set-imagelist" parameter.')
    p.add_option('--key-value-imagelist-file', action ='store',help='Image list metadata value to set from a file. This option requires an "imagelist" selection and a "--key-set-imagelist" parameter.')
    p.add_option('--key-value-image', action ='store',help='Image metadata value to set. This option requires an "image" selection and a "--key-set-image" parameter.')
    p.add_option('--key-value-image-file', action ='store',help='Image metadata value to set from a file. This option requires an "image" selection and a "--key-set-image" parameter.')
    p.add_option('--key-value-endorser', action ='store',help='Endorser metadata value to set. This option requires an "endorser" selection and a "--key-set-endorser" parameter.')
    p.add_option('--key-value-endorser-file', action ='store',help='Endorser metadata value to set from a file. This option requires an "endorser" selection and a "--key-set-endorser" parameter.')

    options, arguments = p.parse_args()

    # Set up basic variables
    logFile = None
    databaseConnectionString = None
    imagelistUUID = None
    imagelist_req = False
    imagelist_key = None
    imagelist_key_set_req = False
    imagelist_key_value = None
    imagelist_key_value_add_req = False
    imagelist_key_value_file = None
    imagelist_import_json = None
    imageUuid = None
    image_key = None
    image_key_req = False
    image_req = None
    image_key_value = None
    image_key_value_add_req = False
    image_key_value_file = None
    endorserSub = None
    endorser_req  = False

    endorserKey = None
    endorserKeyReq = False

    endorserValue = None
    endorserValueReq = False
    endorserValue_file = None
    endorser_key_value_add_req = False
    imageFileLocal = None
    applicationHome = None
    pathX509Key = None
    pathX509Cert = None
    pathX509Dir = None

    signingPathDefaultKey = None
    signingPathDefaultCert = None
    signingPathHome = None
    imagelistLifeTimeStr = "28"
    uploadFlags = []
    dishCfg = None

    # download imagelist variables
    download_imagelist_flags = set([])

    # Read enviroment variables
    if 'VMCASTER_LOG_CONF' in os.environ:
        logFile = os.environ['VMCASTER_LOG_CONF']
    if 'VMCASTER_RDBMS' in os.environ:
        databaseConnectionString = os.environ['VMCASTER_RDBMS']
    if 'VMCASTER_CFG' in os.environ:
        dishCfg = os.environ['VMCASTER_CFG']
    if 'HOME' in os.environ:
        applicationHome = os.environ['HOME']

    if 'VMCASTER_X509_DIR' in os.environ:
        pathX509Dir = os.environ['VMCASTER_X509_DIR']
    if 'VMCASTER_X509_KEY' in os.environ:
        pathX509Key = os.environ['VMCASTER_X509_KEY']
    if 'VMCASTER_X509_CERT' in os.environ:
        pathX509Cert = os.environ['VMCASTER_X509_CERT']

    if 'VMCASTER_IMAGELIST_LIFETIME' in os.environ:
        imagelistLifeTimeStr = os.environ['VMCASTER_IMAGELIST_LIFETIME']


    # Set up log file

    LoggingLevel = logging.INFO
    LoggingLevelCounter = 1
    if options.verbose:
        LoggingLevelCounter = LoggingLevelCounter - options.verbose
        if options.verbose == 1:
            LoggingLevel = logging.INFO
        if options.verbose == 2:
            LoggingLevel = logging.DEBUG
    if options.quiet:
        LoggingLevelCounter = LoggingLevelCounter + options.quiet
    if LoggingLevelCounter <= 0:
        LoggingLevel = logging.DEBUG
    if LoggingLevelCounter == 1:
        LoggingLevel = logging.INFO
    if LoggingLevelCounter == 2:
        LoggingLevel = logging.WARNING
    if LoggingLevelCounter == 3:
        LoggingLevel = logging.ERROR
    if LoggingLevelCounter == 4:
        LoggingLevel = logging.FATAL
    if LoggingLevelCounter >= 5:
        LoggingLevel = logging.CRITICAL

    if options.logcfg:
        logFile = options.logcfg
    if logFile != None:
        if os.path.isfile(str(options.log_config)):
            logging.config.fileConfig(options.log_config)
        else:
            logging.basicConfig(level=LoggingLevel)
            log = logging.getLogger("main")
            log.error("Logfile configuration file '%s' was not found." % (options.log_config))
            sys.exit(1)
    else:

        logging.basicConfig(level=LoggingLevel)
    log = logging.getLogger("main")
    # Now process command line
    actions = set([])

    if options.select_endorser:
        endorserSub = options.select_endorser

    if options.list_endorser:
        actions.add('endorser_list')

    if options.show_endorser:
        actions.add('endorser_show')
        endorser_req = True

    if options.add_endorser:
        actions.add('endorser_add')
        endorser_req = True

    if options.del_endorser:
        actions.add('endorser_del')
        endorserSub = options.del_endorser

    if options.key_set_endorser:
        actions.add('endorser_key_set')
        endorser_req = True
        endorser_key_value_add_req = True
        endorserKey = options.key_set_endorser

    if options.key_del_endorser:
        actions.add('endorser_key_del')
        endorser_req = True

    if options.key_value_endorser:
        endorserValue = options.key_value_endorser
        endorser_req = True

    if options.key_value_endorser_file:
        endorserValue_file = options.key_value_endorser_file
        endorser_req = True


    if options.imagelist_add_endorser:
        endorser_req = True
        imagelist_req = True
        actions.add('connect')
    if options.imagelist_del_endorser:
        endorser_req = True
        imagelist_req = True
        actions.add('disconnect')

    if options.imagelist_add_image:
        image_req = True
        imagelist_req = True
        actions.add('imagelist_image_bind')
    if options.imagelist_del_image:
        image_req = True
        imagelist_req = True
        actions.add('imagelist_image_unbind')

    if options.select_imagelist:
        imagelistUUID = options.select_imagelist
    if options.list_imagelist:
        actions.add('imagelist_list')
    if options.upload_imagelist:
        actions.add('imagelist_upload')
        imagelist_req = True
    if options.lifetime_imagelist:
        imagelistLifeTimeStr = options.lifetime_imagelist

    if options.add_imagelist:
        actions.add('imagelist_add')
        imagelist_req = True

    if options.del_imagelist:
        actions.add('imagelist_del')
        imagelist_req = True

    if options.show_imagelist:
        actions.add('imagelist_show')
        imagelist_req = True

    if options.keys_imagelist:
        actions.add('imagelist_keys')
        imagelist_req = True

    if options.key_set_imagelist:
        actions.add('imagelist_key_update')
        imagelist_req = True
        imagelist_key_value_add_req = True
        imagelist_key = options.key_set_imagelist
    if options.key_del_imagelist:
        actions.add('imagelist_key_del')
        imagelist_req = True
        imagelist_key = options.key_del_imagelist

    if options.key_value_imagelist:
        actions.add('imagelist_key_update')
        imagelist_req = True
        imagelist_key_set_req = True
        imagelist_key_value = options.key_value_imagelist

    if options.key_value_imagelist_file:
        actions.add('imagelist_key_update')
        imagelist_req = True
        imagelist_key_set_req = True
        imagelist_key_value_file = options.key_value_imagelist_file

    if options.download_imagelist:
        actions.add('download_imagelist')
        imagelist_req = True

    if options.download_imagelist_flags:
        download_imagelist_flags = set(options.download_imagelist_flags.split(","))

    if options.import_imagelist_smime:
        actions.add('imagelist_import_smime')

        imagelist_import_smime = options.import_imagelist_smime
    if options.import_imagelist_json:
        actions.add('imagelist_import_json')

        imagelist_import_json = options.import_imagelist_json
    if options.list_image:
        actions.add('image_list')
    if options.select_image:
        imageUuid = options.select_image

    if options.add_image:
        actions.add('image_add')
        image_req = True


    if options.del_image:
        actions.add('image_del')
        image_req = True

    if options.show_image:
        actions.add('image_show')
        image_req = True

    if options.keys_image:
        actions.add('image_keys')
        imagelist_req = True
        image_req = True

    if options.key_set_image:
        actions.add('image_key_update')
        image_req = True
        image_key_value_add_req = True
        image_key = options.key_set_image

    if options.key_value_image:
        actions.add('image_key_update')
        image_req = True
        image_key_req = True
        image_key_value = options.key_value_image

    if options.key_value_image_file:
        actions.add('image_key_update')
        image_req = True
        image_key_req = True
        image_key_value_file = options.key_value_image_file

    if options.upload_image:
        actions.add('image_upload')
        imageFileLocal = options.upload_image
    if options.key_del_image:
        actions.add('image_key_del')
        image_req = True
        image_key_del = options.key_del_image
    if options.database:
        databaseConnectionString = options.database
    if options.config_file:
        dishCfg = options.config_file

    if options.flag:
        uploadFlags = options.flag

    if options.imagelist_list_image:
        imagelist_req = True
        actions.add('imagelist_list_image')
    if options.imagelist_list_endorser:
        imagelist_req = True
        actions.add('imagelist_list_endorser')

    actionsLen = len(actions)
    if actionsLen == 0:
        log.error('No actions added')
        sys.exit(1)
    if actionsLen > 1:
        log.error('To many actions added')
        sys.exit(1)

    # Now default unset values

    if databaseConnectionString == None:
        databaseConnectionString = 'sqlite:///vmcaster.db'
        log.info("Defaulting DB connection to '%s'" % (databaseConnectionString))

    # Now over ride the values when setting a file

    if endorserValue_file != None:
        if not os.path.isfile(endorserValue_file):
            log.error("Invalid --key-value-endorser-file is not a file :'%s'" % (endorserValue_file))
            sys.exit(1)
        if endorserValue != None:
            log.warning("Parameter --key-value-endorser will be over writern with the content of file :'%s'" % (endorserValue_file))
        endorserValue = ''
        fp = open(endorserValue_file)
        with fp:
            endorserValue += fp.read()
    if imagelist_key_value_file != None:
        if not os.path.isfile(imagelist_key_value_file):
            log.error("Invalid --key-value-imagelist-file is not a file :'%s'" % (imagelist_key_value_file))
            sys.exit(1)
        if imagelist_key_value != None:
            log.warning("Parameter --key-value-imagelist will be over writern with the content of file :'%s'" % (imagelist_key_value_file))
        imagelist_key_value = ''
        fp = open(imagelist_key_value_file)
        with fp:
            imagelist_key_value += fp.read()
    if image_key_value_file != None:
        if not os.path.isfile(image_key_value_file):
            log.error("Invalid --key-value-image-file is not a file :'%s'" % (image_key_value_file))
            sys.exit(1)
        if image_key_value != None:
            log.warning("Parameter --key-value-image will be over writern with the content of file :'%s'" % (image_key_value_file))
        image_key_value = ''
        fp = open(image_key_value_file)
        with fp:
            image_key_value += fp.read()


    # Now check for required fields

    if imagelist_req:
        if imagelistUUID == None:
            log.error('Image list UUID is needed ("dc:identifier")')
            sys.exit(1)
    if image_req:
        if imageUuid == None:
            log.error('Image UUID is needed ("dc:identifier")')
            sys.exit(1)
    if endorser_req:
        if endorserSub == None:
            log.error('Endorser subject is needed ("hv:dn")')
            sys.exit(1)
    if dishCfg == None:
        if applicationHome is not None:
            dishCfg = "{home}/{cfg}".format(home=applicationHome,cfg=".vmcaster.cfg")
    if dishCfg != None:
        if not os.path.isfile(str(dishCfg)):
            dishCfgNew = "/etc/vmcaster/vmcaster.cfg"
            log.info("Config file '%s' not found, using system defaults '%s'" % (dishCfg,dishCfgNew))
            dishCfg = dishCfgNew
        else:
            log.info("Found config at '%s'" % (dishCfg))

    try:
        imagelistLifeTime = int(imagelistLifeTimeStr)
    except ValueError,E:
        log.error('Imagelist Lifetime is invalid : %s' % (E))
        sys.exit(1)
    if imagelistLifeTime < 1:
        log.error("Imagelist Lifetime is too short at '%s'" % (imagelistLifeTime))
        sys.exit(1)
    if not os.path.isfile(str(dishCfg)):
        log.error("Configuration file '%s' was not found." % (dishCfg))
        sys.exit(1)


    # Now check for required fields

    if endorserValue == None and endorser_key_value_add_req == True:
        log.error("Missing '--key-value-endorser'")
        sys.exit(1)

    if image_key_value == None and image_key_value_add_req == True:
        log.error("Missing '--key-value-image'")
        sys.exit(1)

    if imagelist_key_value == None and imagelist_key_value_add_req == True:
        log.error("Missing '--key-value-imagelist'")
        sys.exit(1)


    # now do the work.

    imagepub = dishpubstate.imagelistpub(databaseConnectionString)

    if 'endorser_list' in actions:
        imagepub.endorserList()
    if 'endorser_show' in actions:
        output = json.dumps(imagepub.endorserDump(endorserSub),sort_keys=True, indent=4)
        if output != None:
            print output

    if 'endorser_add' in actions:
        imagepub.endorserAdd(endorserSub)
    if 'endorser_del' in actions:
        imagepub.endorserDel(endorserSub)
    if 'endorser_key_set' in actions:
        imagepub.endorserMetadataUpdate(endorserSub,endorserKey,endorserValue)
    if 'endorser_key_del' in actions:
        imagepub.endorserMetadataDel(endorserSub,endorserKey)

    if 'connect' in actions:
        imagepub.imageListEndorserConnect(imagelistUUID,endorserSub)

    if 'disconnect' in actions:
        imagepub.imageListEndorserDisconnect(imagelistUUID,endorserSub)

    if 'imagelist_image_bind' in actions:
        imagepub.imageListImageConnect(imagelistUUID,imageUuid)

    if 'imagelist_image_unbind' in actions:
        imagepub.imageListImageDisconnect(imagelistUUID,imageUuid)


    if 'imagelist_list_image' in actions:
        output = imagepub.ImageList_List_Image(imagelistUUID)
        if output == None:
            return
        for item in output:
            print item

    if 'imagelist_list_endorser' in actions:
        output = imagepub.ImageList_List_Endorser(imagelistUUID)
        if output == None:
            return
        for item in output:
            print item




    if 'imagelist_list' in actions:
        imagepub.imageListList()
    if 'imagelist_show' in actions:
        output = json.dumps(imagepub.imageListShow(imagelistUUID),sort_keys=True, indent=4)
        if output != None:
            print output

    if 'imagelist_add' in actions:
        imagepub.imageListAdd(imagelistUUID)

    if 'imagelist_del' in actions:
        imagepub.imagesDel(imagelistUUID)
    if 'imagelist_key_update' in actions:
        imagepub.imagelist_key_update(imagelistUUID, imagelist_key, imagelist_key_value)
    if 'imagelist_key_del' in actions:
        imagepub.imagelist_key_del(imagelistUUID, imagelist_key)

    if 'imagelist_import_smime' in actions:


        fp = open (imagelist_import_smime)
        inportText = fp.read()
        buf = M2Crypto.BIO.MemoryBuffer(str(inportText))
        try:
            p7, data = M2Crypto.SMIME.smime_load_pkcs7_bio(buf)
        except AttributeError, e:
            log.error("Failed to load SMIME")
            raise e
        readData = data.read()
        try:
            candidate = json.loads(str(readData))
        except ValueError:
            log.error("Failed to parse JSON.")
            sys.exit(20)

        if candidate == None:
            log.error("No JSON content.")
            sys.exit(21)

        imagepub.importer(candidate)



    if 'image_list' in actions:
        images = imagepub.imageList()
        for item in images:
            print item
    if 'image_add' in actions:
        imagepub.imageAdd(imageUuid)
        return
    if 'image_show' in actions:
        output = json.dumps(imagepub.imageShow(imageUuid),sort_keys=True, indent=4)
        if output != None:
            print output

    if 'image_del' in actions:
        imagepub.imageDelete(imageUuid)
        return



    if 'image_key_update' in actions:
        imagepub.image_key_update(imageUuid ,image_key, image_key_value)
    if 'image_key_del' in actions:
        imagepub.image_key_delete(imageUuid, image_key_del)
    if 'image_keys' in actions:
        imagepub.image_keys(imagelistUUID, imageUuid)



    if 'image_upload' in actions:
        listOfImagelists = imagepub.image_get_imagelist(imageUuid)
        if len(listOfImagelists) == 0:
            log.error("No matching image list found")
            sys.exit(45)
        ThisImageListUuid = str(listOfImagelists[0])
        reuiredImageType = imagepub.image_key_get(imageUuid,"hv:format")
        localPath = imageFileLocal
        if reuiredImageType == None:
            log.debug("Using raw image in provided format")
        else:
            ms = magic.open(magic.MAGIC_NONE)
            ms.load()
            sourceImageType =  ms.file(imageFileLocal)
            sFieldList =  sourceImageType.split(',')
            rFeildList = reuiredImageType.split(".")
            mappings = { 'bz2' : "bzip2 compressed data",
                'gz' : "gzip compressed data" }

            shouldCompress = False
            if rFeildList[-1] in mappings.keys():
                if sFieldList[0] != mappings[rFeildList[-1]]:
                    log.info("The image you are uploading should be compresed.")
                    shouldCompress = True
                else:
                    log.debug("File type is as expected")
            localPath = imageFileLocal
            if shouldCompress:
                mytempdir = tempfile.mkdtemp()
                tmpfilePath = os.path.join(mytempdir,"compressed")
                #shutil.copyfile(imageFileLocal,tmpfilePath )
                cmd = 'cat %s | gzip -c > %s.gz' % (imageFileLocal,tmpfilePath)
                if rFeildList[-1] == "bz2":
                    cmd = 'cat %s | bzip2 -c > %s.bz2' % (imageFileLocal,tmpfilePath)
                log.info("Compresing source image '%s'." % (cmd))
                rc,output = commands.getstatusoutput(cmd)
                if rc != 0 :
                    log.error(output)
                    sys.exit(1)
                combinedNamesList = []
                for filename in os.listdir(mytempdir):
                    combainedName = os.path.join(mytempdir,filename)
                    if os.path.isfile(combainedName):
                        combinedNamesList.append(filename)
                uploadablefileName = ""
                if len(combinedNamesList) > 1:
                    print "unknown file found"
                    sys.exit(1)
                if len(combinedNamesList) == 0:
                    print "compresed file not found"
                    sys.exit(1)
                imageName = combinedNamesList[0]
                localPath = os.path.join(mytempdir,imageName)
        m = hashlib.sha512()
        filelength = 0
        for line in open(localPath,'r'):
            filelength += len(line)
            m.update(line)

        sha512 = imagepub.image_key_get(imageUuid,"sl:checksum:sha512")

        if sha512 == m.hexdigest():
            log.warning("Image unchanged, aborting uppdate.")
            sys.exit(0)

        # Generate a new URI based on the old one.
        uri = imagepub.image_key_get(imageUuid,"hv:uri")
        if uri == None:
            log.error("Image has no external URI so cant generate a new URI please add key 'hv:uri' to '%s' image." % (imageUuid))
            sys.exit(13)
        parsedUri = uglyUriParser(uri)
        oldpath = parsedUri['path']
        log.debug("old image path=%s" % (oldpath))
        directory = os.path.dirname(oldpath)
        log.debug("old base directory=%s" % (directory))

        curtime = datetime.datetime.utcnow()
        extension = "img"
        if reuiredImageType != None:
            extension = reuiredImageType
        imageName = "%s_%s.%s" % (imageUuid,curtime.strftime("%Y-%m-%d_%H-%M-%S"),extension)
        newPath = os.path.join(directory, imageName)

        parsedUri['path'] = newPath
        uri = uglyUriBuilder(parsedUri)



        log.debug("new image uri=%s" % (uri))

        timeout = 10000
        uploader = hostUploader(dishCfg)
        try:
            uploaderRc = uploader.replaceFile(localPath,parsedUri['hostname'],uri)
        except InputError, e:
            log.error(e.msg)
            sys.exit(99)
        if uploaderRc == None:
            log.error("Error precessing result of upload.")
            sys.exit(99)
        rc,stdout,stderr = uploaderRc
        if rc != 0:
            log.error("failed to upload file")
            log.error("Upload produced the following output:\n%s" % (stdout))
            log.error("Upload produced the following srderr:\n%s" % (stderr))
            sys.exit(77)
        imagepub.image_key_update(imageUuid,u'hv:size',filelength)
        imagepub.image_key_update(imageUuid,u'sl:checksum:sha512',m.hexdigest())
        versionOld = imagepub.image_key_get(imageUuid, "hv:version")
        if versionOld == None:
            versionOld = "0.0.0"
            log.warning("Adding a default version.")
        versionNew = bumpVersion(versionOld)
        imagepub.image_key_update(imageUuid, "hv:version",versionNew)
        uri = uglyUriBuilder(parsedUri)
        imagepub.image_key_update(imageUuid,  "hv:uri", uri)

    if 'imagelist_import_json' in actions:
        f = open(imagelist_import_json)
        try:
            candidate = json.loads(str(f.read()))
        except ValueError:
            log.error("Failed to parse JSON.")
            sys.exit(20)

        if candidate == None:
            log.error("No JSON content.")
            sys.exit(21)

        imagepub.importer(candidate)

    if 'imagelist_upload' in actions:
        if pathX509Key == None or pathX509Cert == None:
            if pathX509Dir == None:
                if applicationHome == None:
                    log.error("No HOME enviroment variable.")
                    sys.exit(20)
                pathX509Dir = "%s/%s" % (applicationHome,".globus")
                log.debug("Setting pathX509Dir=%s" % (pathX509Dir))
            if not os.path.isdir(pathX509Dir):
                log.warning("Directory '%s' does not exist, unable to guess certificates path." % (pathX509Dir))
            else:
                if pathX509Key == None:
                    pathX509Key = "%s/%s" % (pathX509Dir,"userkey.pem")
                if pathX509Cert == None:
                    pathX509Cert = "%s/%s" % (pathX509Dir,"usercert.pem")
        if pathX509Key == None:
            log.critical("Invalid Key path")
            sys.exit(21)
        if pathX509Cert == None:
            log.critical("Invalid certificate path")
            sys.exit(22)
        if not os.path.isfile(pathX509Key):
            log.critical("Key not found at path '%s'" % (pathX509Key))
            sys.exit(23)
        if not os.path.isfile(pathX509Cert):
            log.critical("Cert not found at path '%s'" % (pathX509Cert))
            sys.exit(23)

        if imagepub.imageListShow(imagelistUUID) == None:
            log.critical("Image list '%s' not found:" % (imagelistUUID))
            sys.exit(23)


        versionOld = imagepub.imagelist_key_get(imagelistUUID, "hv:version")
        if versionOld == None:
            versionOld = "0.0.0"
        versionNew = bumpVersion(versionOld)
        if versionNew == None:
            versionNew = '0.0.1'
        now = datetime.datetime.utcnow()
        servicelength = datetime.timedelta(days=imagelistLifeTime)
        expiry = now + servicelength
        time_format_definition = "%Y-%m-%dT%H:%M:%SZ"
        dateNow = now.strftime(time_format_definition)
        dateExpires = expiry.strftime(time_format_definition)
        imageListAsDict = imagepub.imageListShow(imagelistUUID)
        imageListAsDict[u'hv:imagelist'][u'hv:version'] = unicode(versionNew)
        imageListAsDict[u'hv:imagelist'][u'dc:date:created'] = dateNow
        imageListAsDict[u'hv:imagelist'][u'dc:date:expires'] = dateExpires
        uri = imagepub.imagelist_key_get(imagelistUUID,"hv:uri")
        if uri == None:
            log.error("Image list has no attribute 'hv:uri', image upload aborted.")
            sys.exit(20)
        parsedUri = uglyUriParser(uri)
        mytempdir = tempfile.mkdtemp()
        tmpfilePath = os.path.join(mytempdir,"signed_file")

        smime = M2Crypto.SMIME.SMIME()
        x509 = M2Crypto.X509.load_cert(pathX509Cert)
        get_ext_count = int(x509.get_ext_count())
        extensionDict = {}
        for i in range(get_ext_count):
             extension =  x509.get_ext_at(i)
             extensionDict[unicode(extension.get_name())] = unicode(extension.get_value())
        if not "keyUsage" in extensionDict.keys():
            log.critical("Certificate does not caontain a keyUsage extension %s" % (pathX509Cert))
            sys.exit(31)
        foundKeyUsage = extensionDict["keyUsage"]
        foundKeyUsageSplit = foundKeyUsage.split(', ')
        if not u'Digital Signature' in foundKeyUsageSplit:
            log.critical("Certificate doe not support Digital Signature: %s" % (pathX509Cert))
            sys.exit(32)
        issuerSubject = str(x509.get_issuer())
        subject = str(x509.get_subject())

        if not imagepub.checkMissingFields(imagelistUUID,subject,issuerSubject):
            log.error("Image list has 1 or more required attributes missing.")
            sys.exit(20)
        try:
            smime.load_key(pathX509Key,pathX509Cert)
        except M2Crypto.EVP.EVPError, e:
            log.error("Failed to load key : '%s', cert : '%s'." % (pathX509Key,pathX509Cert))
            sys.exit(29)
        fp = open(str(tmpfilePath),'w')
        content = json.dumps(imageListAsDict,sort_keys=True, indent=4)
        buf = M2Crypto.BIO.MemoryBuffer(content)
        p7 = smime.sign(buf,M2Crypto.SMIME.PKCS7_DETACHED)
        buf = M2Crypto.BIO.MemoryBuffer(content)
        out = M2Crypto.BIO.MemoryBuffer()
        smime.write(out, p7, buf)
        message_signed = str(out.read())
        fp.write(message_signed)
        fp.close()
        uploader = hostUploader(dishCfg)
        uploader.flags = uploadFlags
        try:
            uploader.deleteFile(parsedUri['hostname'],uri)
        except InputError, E:
            log.error(E.msg)
            sys.exit(21)
        output = uploader.replaceFile(tmpfilePath,parsedUri['hostname'],uri)
        if output == None:
            log.error("Could not replaceFile")
            sys.exit(21)
        rc,stdout,stderr = output
        if rc != 0:
            log.error("failed to upload file")
            if len(stdout) > 0:
                log.info("Upload produced the following output:\n%s" % (stdout))
            if len(stderr) > 0:
                log.info("Upload produced the following srderr:\n%s" % (stderr))
            sys.exit(77)
        imagepub.imagelist_key_update(imagelistUUID, "dc:date:created",dateNow)
        imagepub.imagelist_key_update(imagelistUUID, "dc:date:expires",dateExpires)
        imagepub.imagelist_key_update(imagelistUUID, "hv:version",versionNew)


    if 'download_imagelist' in actions:
        imagepub.download_imagelist(imagelistUUID, download_imagelist_flags)


if __name__ == "__main__":
    main()