cmstools/CreateFileList.py at master · apana/cmstools · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
#!/usr/bin/python
#

import sys,string,time,os

MAXFILES=-1
FilesPerCfg=12
# CFGBASE="MinBias_StartupV5_GENSIMRAW_212_"
# CFGBASE="Monitor_Commissioning09-v1_RAW_Run82548_"
CFGBASE="ZeroBiasSkim_beginLS"

# fix for dealing with dCache files
stringtoReplace   = "/pnfs/cms/WAX/"
replacementString = "dcap://cmsdca.fnal.gov:24136/pnfs/fnal.gov/usr/cms/WAX/"


def usage():
    """ Usage: CreateFileList <FileList> <OutputDir>
    """
    pass

def OpenFile(file_in,iodir):
    """  file_in -- Input file name
         iodir   -- 'r' readonly  'r+' read+write """
    try:
        ifile=open(file_in, iodir)
        # print "Opened file: ",file_in," iodir ",iodir
    except:
        print "Could not open file: ",file_in
        sys.exit(1)
    return ifile

def CloseFile(ifile):
    ifile.close()

def ReadFile(file):


    infile=OpenFile(file,'r')
    iline=0

    x = infile.readline()

    files=[]
    while x != "":
        iline+=1
        xx=string.strip(x)
        if (len(xx)>0):
            files.append(xx)

        x = infile.readline()
    CloseFile(infile)

    return files

def createCFG(i,filelist, CFGBASE):

    # add underscore if missing
    if CFGBASE[len(CFGBASE)-1] != "_":
        CFGBASE = CFGBASE + "_"

    CFGFILE=CFGBASE + str(i) +".py"
    print i, CFGFILE
    file = open(CFGFILE,'w')
    file.write("import FWCore.ParameterSet.Config as cms \n")
    file.write("\n")
    file.write("maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) )\n")
    file.write("readFiles = cms.untracked.vstring()\n")
    file.write("secFiles = cms.untracked.vstring()\n")
    file.write("source = cms.Source (\"PoolSource\",fileNames = readFiles, secondaryFileNames = secFiles)\n")
    file.write("readFiles.extend( [\n")

    i=0
    for infile in filelist:
       i=i+1

       if infile.find(stringtoReplace) >-1:
           infile=infile.replace(stringtoReplace,replacementString)

       if i<len(filelist):
           endstr="',"
       else:
           endstr="' ]);"
       outstring="    " + "'" + infile + endstr
       file.write(outstring + "\n")

    file.write("\n")
    file.write("secFiles.extend( [ ])\n")
    CloseFile(file)

if __name__ == '__main__':


    narg=len(sys.argv)
    if narg < 3 :
        print usage.__doc__
        sys.exit(1)


    InputFile=sys.argv[1]
    OutputDir=sys.argv[2]
    if not os.path.exists(OutputDir):
        print "Output directory does not exist. Creating: ", OutputDir
        os.mkdir(OutputDir)


    filelist = ReadFile(InputFile)
    print "\nNumber of input files: ", len(filelist)
    if ( MAXFILES>0 ): print "\n processing first ", MAXFILES, " in list"
    print "Creating filelists with ", FilesPerCfg, "files per filelist\n"

    os.chdir(OutputDir)

    i=0
    ibatch=0
    cfglist=[]
    if (MAXFILES < 0): MAXFILES=100000
    for ifile in filelist:
        i=i+1
        if (i>MAXFILES): break
        filename=ifile
        cfglist.append(filename)
        if i%FilesPerCfg == 0:
            createCFG(ibatch,cfglist,CFGBASE)
            ibatch=ibatch+1
            cfglist=[]
        if (i == len(filelist) and len(cfglist)>0): # get the last batch
            createCFG(ibatch,cfglist)
        #print i%FilesPerCfg, i, filename

    print "\n"