1*cda5da8dSAndroid Build Coastguard Worker"""distutils.archive_util 2*cda5da8dSAndroid Build Coastguard Worker 3*cda5da8dSAndroid Build Coastguard WorkerUtility functions for creating archive files (tarballs, zip files, 4*cda5da8dSAndroid Build Coastguard Workerthat sort of thing).""" 5*cda5da8dSAndroid Build Coastguard Worker 6*cda5da8dSAndroid Build Coastguard Workerimport os 7*cda5da8dSAndroid Build Coastguard Workerfrom warnings import warn 8*cda5da8dSAndroid Build Coastguard Workerimport sys 9*cda5da8dSAndroid Build Coastguard Worker 10*cda5da8dSAndroid Build Coastguard Workertry: 11*cda5da8dSAndroid Build Coastguard Worker import zipfile 12*cda5da8dSAndroid Build Coastguard Workerexcept ImportError: 13*cda5da8dSAndroid Build Coastguard Worker zipfile = None 14*cda5da8dSAndroid Build Coastguard Worker 15*cda5da8dSAndroid Build Coastguard Worker 16*cda5da8dSAndroid Build Coastguard Workerfrom distutils.errors import DistutilsExecError 17*cda5da8dSAndroid Build Coastguard Workerfrom distutils.spawn import spawn 18*cda5da8dSAndroid Build Coastguard Workerfrom distutils.dir_util import mkpath 19*cda5da8dSAndroid Build Coastguard Workerfrom distutils import log 20*cda5da8dSAndroid Build Coastguard Worker 21*cda5da8dSAndroid Build Coastguard Workertry: 22*cda5da8dSAndroid Build Coastguard Worker from pwd import getpwnam 23*cda5da8dSAndroid Build Coastguard Workerexcept ImportError: 24*cda5da8dSAndroid Build Coastguard Worker getpwnam = None 25*cda5da8dSAndroid Build Coastguard Worker 26*cda5da8dSAndroid Build Coastguard Workertry: 27*cda5da8dSAndroid Build Coastguard Worker from grp import getgrnam 28*cda5da8dSAndroid Build Coastguard Workerexcept ImportError: 29*cda5da8dSAndroid Build Coastguard Worker getgrnam = None 30*cda5da8dSAndroid Build Coastguard Worker 31*cda5da8dSAndroid Build Coastguard Workerdef _get_gid(name): 32*cda5da8dSAndroid Build Coastguard Worker """Returns a gid, given a group name.""" 33*cda5da8dSAndroid Build Coastguard Worker if getgrnam is None or name is None: 34*cda5da8dSAndroid Build Coastguard Worker return None 35*cda5da8dSAndroid Build Coastguard Worker try: 36*cda5da8dSAndroid Build Coastguard Worker result = getgrnam(name) 37*cda5da8dSAndroid Build Coastguard Worker except KeyError: 38*cda5da8dSAndroid Build Coastguard Worker result = None 39*cda5da8dSAndroid Build Coastguard Worker if result is not None: 40*cda5da8dSAndroid Build Coastguard Worker return result[2] 41*cda5da8dSAndroid Build Coastguard Worker return None 42*cda5da8dSAndroid Build Coastguard Worker 43*cda5da8dSAndroid Build Coastguard Workerdef _get_uid(name): 44*cda5da8dSAndroid Build Coastguard Worker """Returns an uid, given a user name.""" 45*cda5da8dSAndroid Build Coastguard Worker if getpwnam is None or name is None: 46*cda5da8dSAndroid Build Coastguard Worker return None 47*cda5da8dSAndroid Build Coastguard Worker try: 48*cda5da8dSAndroid Build Coastguard Worker result = getpwnam(name) 49*cda5da8dSAndroid Build Coastguard Worker except KeyError: 50*cda5da8dSAndroid Build Coastguard Worker result = None 51*cda5da8dSAndroid Build Coastguard Worker if result is not None: 52*cda5da8dSAndroid Build Coastguard Worker return result[2] 53*cda5da8dSAndroid Build Coastguard Worker return None 54*cda5da8dSAndroid Build Coastguard Worker 55*cda5da8dSAndroid Build Coastguard Workerdef make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, 56*cda5da8dSAndroid Build Coastguard Worker owner=None, group=None): 57*cda5da8dSAndroid Build Coastguard Worker """Create a (possibly compressed) tar file from all the files under 58*cda5da8dSAndroid Build Coastguard Worker 'base_dir'. 59*cda5da8dSAndroid Build Coastguard Worker 60*cda5da8dSAndroid Build Coastguard Worker 'compress' must be "gzip" (the default), "bzip2", "xz", "compress", or 61*cda5da8dSAndroid Build Coastguard Worker None. ("compress" will be deprecated in Python 3.2) 62*cda5da8dSAndroid Build Coastguard Worker 63*cda5da8dSAndroid Build Coastguard Worker 'owner' and 'group' can be used to define an owner and a group for the 64*cda5da8dSAndroid Build Coastguard Worker archive that is being built. If not provided, the current owner and group 65*cda5da8dSAndroid Build Coastguard Worker will be used. 66*cda5da8dSAndroid Build Coastguard Worker 67*cda5da8dSAndroid Build Coastguard Worker The output tar file will be named 'base_dir' + ".tar", possibly plus 68*cda5da8dSAndroid Build Coastguard Worker the appropriate compression extension (".gz", ".bz2", ".xz" or ".Z"). 69*cda5da8dSAndroid Build Coastguard Worker 70*cda5da8dSAndroid Build Coastguard Worker Returns the output filename. 71*cda5da8dSAndroid Build Coastguard Worker """ 72*cda5da8dSAndroid Build Coastguard Worker tar_compression = {'gzip': 'gz', 'bzip2': 'bz2', 'xz': 'xz', None: '', 73*cda5da8dSAndroid Build Coastguard Worker 'compress': ''} 74*cda5da8dSAndroid Build Coastguard Worker compress_ext = {'gzip': '.gz', 'bzip2': '.bz2', 'xz': '.xz', 75*cda5da8dSAndroid Build Coastguard Worker 'compress': '.Z'} 76*cda5da8dSAndroid Build Coastguard Worker 77*cda5da8dSAndroid Build Coastguard Worker # flags for compression program, each element of list will be an argument 78*cda5da8dSAndroid Build Coastguard Worker if compress is not None and compress not in compress_ext.keys(): 79*cda5da8dSAndroid Build Coastguard Worker raise ValueError( 80*cda5da8dSAndroid Build Coastguard Worker "bad value for 'compress': must be None, 'gzip', 'bzip2', " 81*cda5da8dSAndroid Build Coastguard Worker "'xz' or 'compress'") 82*cda5da8dSAndroid Build Coastguard Worker 83*cda5da8dSAndroid Build Coastguard Worker archive_name = base_name + '.tar' 84*cda5da8dSAndroid Build Coastguard Worker if compress != 'compress': 85*cda5da8dSAndroid Build Coastguard Worker archive_name += compress_ext.get(compress, '') 86*cda5da8dSAndroid Build Coastguard Worker 87*cda5da8dSAndroid Build Coastguard Worker mkpath(os.path.dirname(archive_name), dry_run=dry_run) 88*cda5da8dSAndroid Build Coastguard Worker 89*cda5da8dSAndroid Build Coastguard Worker # creating the tarball 90*cda5da8dSAndroid Build Coastguard Worker import tarfile # late import so Python build itself doesn't break 91*cda5da8dSAndroid Build Coastguard Worker 92*cda5da8dSAndroid Build Coastguard Worker log.info('Creating tar archive') 93*cda5da8dSAndroid Build Coastguard Worker 94*cda5da8dSAndroid Build Coastguard Worker uid = _get_uid(owner) 95*cda5da8dSAndroid Build Coastguard Worker gid = _get_gid(group) 96*cda5da8dSAndroid Build Coastguard Worker 97*cda5da8dSAndroid Build Coastguard Worker def _set_uid_gid(tarinfo): 98*cda5da8dSAndroid Build Coastguard Worker if gid is not None: 99*cda5da8dSAndroid Build Coastguard Worker tarinfo.gid = gid 100*cda5da8dSAndroid Build Coastguard Worker tarinfo.gname = group 101*cda5da8dSAndroid Build Coastguard Worker if uid is not None: 102*cda5da8dSAndroid Build Coastguard Worker tarinfo.uid = uid 103*cda5da8dSAndroid Build Coastguard Worker tarinfo.uname = owner 104*cda5da8dSAndroid Build Coastguard Worker return tarinfo 105*cda5da8dSAndroid Build Coastguard Worker 106*cda5da8dSAndroid Build Coastguard Worker if not dry_run: 107*cda5da8dSAndroid Build Coastguard Worker tar = tarfile.open(archive_name, 'w|%s' % tar_compression[compress]) 108*cda5da8dSAndroid Build Coastguard Worker try: 109*cda5da8dSAndroid Build Coastguard Worker tar.add(base_dir, filter=_set_uid_gid) 110*cda5da8dSAndroid Build Coastguard Worker finally: 111*cda5da8dSAndroid Build Coastguard Worker tar.close() 112*cda5da8dSAndroid Build Coastguard Worker 113*cda5da8dSAndroid Build Coastguard Worker # compression using `compress` 114*cda5da8dSAndroid Build Coastguard Worker if compress == 'compress': 115*cda5da8dSAndroid Build Coastguard Worker warn("'compress' will be deprecated.", PendingDeprecationWarning) 116*cda5da8dSAndroid Build Coastguard Worker # the option varies depending on the platform 117*cda5da8dSAndroid Build Coastguard Worker compressed_name = archive_name + compress_ext[compress] 118*cda5da8dSAndroid Build Coastguard Worker if sys.platform == 'win32': 119*cda5da8dSAndroid Build Coastguard Worker cmd = [compress, archive_name, compressed_name] 120*cda5da8dSAndroid Build Coastguard Worker else: 121*cda5da8dSAndroid Build Coastguard Worker cmd = [compress, '-f', archive_name] 122*cda5da8dSAndroid Build Coastguard Worker spawn(cmd, dry_run=dry_run) 123*cda5da8dSAndroid Build Coastguard Worker return compressed_name 124*cda5da8dSAndroid Build Coastguard Worker 125*cda5da8dSAndroid Build Coastguard Worker return archive_name 126*cda5da8dSAndroid Build Coastguard Worker 127*cda5da8dSAndroid Build Coastguard Workerdef make_zipfile(base_name, base_dir, verbose=0, dry_run=0): 128*cda5da8dSAndroid Build Coastguard Worker """Create a zip file from all the files under 'base_dir'. 129*cda5da8dSAndroid Build Coastguard Worker 130*cda5da8dSAndroid Build Coastguard Worker The output zip file will be named 'base_name' + ".zip". Uses either the 131*cda5da8dSAndroid Build Coastguard Worker "zipfile" Python module (if available) or the InfoZIP "zip" utility 132*cda5da8dSAndroid Build Coastguard Worker (if installed and found on the default search path). If neither tool is 133*cda5da8dSAndroid Build Coastguard Worker available, raises DistutilsExecError. Returns the name of the output zip 134*cda5da8dSAndroid Build Coastguard Worker file. 135*cda5da8dSAndroid Build Coastguard Worker """ 136*cda5da8dSAndroid Build Coastguard Worker zip_filename = base_name + ".zip" 137*cda5da8dSAndroid Build Coastguard Worker mkpath(os.path.dirname(zip_filename), dry_run=dry_run) 138*cda5da8dSAndroid Build Coastguard Worker 139*cda5da8dSAndroid Build Coastguard Worker # If zipfile module is not available, try spawning an external 140*cda5da8dSAndroid Build Coastguard Worker # 'zip' command. 141*cda5da8dSAndroid Build Coastguard Worker if zipfile is None: 142*cda5da8dSAndroid Build Coastguard Worker if verbose: 143*cda5da8dSAndroid Build Coastguard Worker zipoptions = "-r" 144*cda5da8dSAndroid Build Coastguard Worker else: 145*cda5da8dSAndroid Build Coastguard Worker zipoptions = "-rq" 146*cda5da8dSAndroid Build Coastguard Worker 147*cda5da8dSAndroid Build Coastguard Worker try: 148*cda5da8dSAndroid Build Coastguard Worker spawn(["zip", zipoptions, zip_filename, base_dir], 149*cda5da8dSAndroid Build Coastguard Worker dry_run=dry_run) 150*cda5da8dSAndroid Build Coastguard Worker except DistutilsExecError: 151*cda5da8dSAndroid Build Coastguard Worker # XXX really should distinguish between "couldn't find 152*cda5da8dSAndroid Build Coastguard Worker # external 'zip' command" and "zip failed". 153*cda5da8dSAndroid Build Coastguard Worker raise DistutilsExecError(("unable to create zip file '%s': " 154*cda5da8dSAndroid Build Coastguard Worker "could neither import the 'zipfile' module nor " 155*cda5da8dSAndroid Build Coastguard Worker "find a standalone zip utility") % zip_filename) 156*cda5da8dSAndroid Build Coastguard Worker 157*cda5da8dSAndroid Build Coastguard Worker else: 158*cda5da8dSAndroid Build Coastguard Worker log.info("creating '%s' and adding '%s' to it", 159*cda5da8dSAndroid Build Coastguard Worker zip_filename, base_dir) 160*cda5da8dSAndroid Build Coastguard Worker 161*cda5da8dSAndroid Build Coastguard Worker if not dry_run: 162*cda5da8dSAndroid Build Coastguard Worker try: 163*cda5da8dSAndroid Build Coastguard Worker zip = zipfile.ZipFile(zip_filename, "w", 164*cda5da8dSAndroid Build Coastguard Worker compression=zipfile.ZIP_DEFLATED) 165*cda5da8dSAndroid Build Coastguard Worker except RuntimeError: 166*cda5da8dSAndroid Build Coastguard Worker zip = zipfile.ZipFile(zip_filename, "w", 167*cda5da8dSAndroid Build Coastguard Worker compression=zipfile.ZIP_STORED) 168*cda5da8dSAndroid Build Coastguard Worker 169*cda5da8dSAndroid Build Coastguard Worker with zip: 170*cda5da8dSAndroid Build Coastguard Worker if base_dir != os.curdir: 171*cda5da8dSAndroid Build Coastguard Worker path = os.path.normpath(os.path.join(base_dir, '')) 172*cda5da8dSAndroid Build Coastguard Worker zip.write(path, path) 173*cda5da8dSAndroid Build Coastguard Worker log.info("adding '%s'", path) 174*cda5da8dSAndroid Build Coastguard Worker for dirpath, dirnames, filenames in os.walk(base_dir): 175*cda5da8dSAndroid Build Coastguard Worker for name in dirnames: 176*cda5da8dSAndroid Build Coastguard Worker path = os.path.normpath(os.path.join(dirpath, name, '')) 177*cda5da8dSAndroid Build Coastguard Worker zip.write(path, path) 178*cda5da8dSAndroid Build Coastguard Worker log.info("adding '%s'", path) 179*cda5da8dSAndroid Build Coastguard Worker for name in filenames: 180*cda5da8dSAndroid Build Coastguard Worker path = os.path.normpath(os.path.join(dirpath, name)) 181*cda5da8dSAndroid Build Coastguard Worker if os.path.isfile(path): 182*cda5da8dSAndroid Build Coastguard Worker zip.write(path, path) 183*cda5da8dSAndroid Build Coastguard Worker log.info("adding '%s'", path) 184*cda5da8dSAndroid Build Coastguard Worker 185*cda5da8dSAndroid Build Coastguard Worker return zip_filename 186*cda5da8dSAndroid Build Coastguard Worker 187*cda5da8dSAndroid Build Coastguard WorkerARCHIVE_FORMATS = { 188*cda5da8dSAndroid Build Coastguard Worker 'gztar': (make_tarball, [('compress', 'gzip')], "gzip'ed tar-file"), 189*cda5da8dSAndroid Build Coastguard Worker 'bztar': (make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file"), 190*cda5da8dSAndroid Build Coastguard Worker 'xztar': (make_tarball, [('compress', 'xz')], "xz'ed tar-file"), 191*cda5da8dSAndroid Build Coastguard Worker 'ztar': (make_tarball, [('compress', 'compress')], "compressed tar file"), 192*cda5da8dSAndroid Build Coastguard Worker 'tar': (make_tarball, [('compress', None)], "uncompressed tar file"), 193*cda5da8dSAndroid Build Coastguard Worker 'zip': (make_zipfile, [],"ZIP file") 194*cda5da8dSAndroid Build Coastguard Worker } 195*cda5da8dSAndroid Build Coastguard Worker 196*cda5da8dSAndroid Build Coastguard Workerdef check_archive_formats(formats): 197*cda5da8dSAndroid Build Coastguard Worker """Returns the first format from the 'format' list that is unknown. 198*cda5da8dSAndroid Build Coastguard Worker 199*cda5da8dSAndroid Build Coastguard Worker If all formats are known, returns None 200*cda5da8dSAndroid Build Coastguard Worker """ 201*cda5da8dSAndroid Build Coastguard Worker for format in formats: 202*cda5da8dSAndroid Build Coastguard Worker if format not in ARCHIVE_FORMATS: 203*cda5da8dSAndroid Build Coastguard Worker return format 204*cda5da8dSAndroid Build Coastguard Worker return None 205*cda5da8dSAndroid Build Coastguard Worker 206*cda5da8dSAndroid Build Coastguard Workerdef make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0, 207*cda5da8dSAndroid Build Coastguard Worker dry_run=0, owner=None, group=None): 208*cda5da8dSAndroid Build Coastguard Worker """Create an archive file (eg. zip or tar). 209*cda5da8dSAndroid Build Coastguard Worker 210*cda5da8dSAndroid Build Coastguard Worker 'base_name' is the name of the file to create, minus any format-specific 211*cda5da8dSAndroid Build Coastguard Worker extension; 'format' is the archive format: one of "zip", "tar", "gztar", 212*cda5da8dSAndroid Build Coastguard Worker "bztar", "xztar", or "ztar". 213*cda5da8dSAndroid Build Coastguard Worker 214*cda5da8dSAndroid Build Coastguard Worker 'root_dir' is a directory that will be the root directory of the 215*cda5da8dSAndroid Build Coastguard Worker archive; ie. we typically chdir into 'root_dir' before creating the 216*cda5da8dSAndroid Build Coastguard Worker archive. 'base_dir' is the directory where we start archiving from; 217*cda5da8dSAndroid Build Coastguard Worker ie. 'base_dir' will be the common prefix of all files and 218*cda5da8dSAndroid Build Coastguard Worker directories in the archive. 'root_dir' and 'base_dir' both default 219*cda5da8dSAndroid Build Coastguard Worker to the current directory. Returns the name of the archive file. 220*cda5da8dSAndroid Build Coastguard Worker 221*cda5da8dSAndroid Build Coastguard Worker 'owner' and 'group' are used when creating a tar archive. By default, 222*cda5da8dSAndroid Build Coastguard Worker uses the current owner and group. 223*cda5da8dSAndroid Build Coastguard Worker """ 224*cda5da8dSAndroid Build Coastguard Worker save_cwd = os.getcwd() 225*cda5da8dSAndroid Build Coastguard Worker if root_dir is not None: 226*cda5da8dSAndroid Build Coastguard Worker log.debug("changing into '%s'", root_dir) 227*cda5da8dSAndroid Build Coastguard Worker base_name = os.path.abspath(base_name) 228*cda5da8dSAndroid Build Coastguard Worker if not dry_run: 229*cda5da8dSAndroid Build Coastguard Worker os.chdir(root_dir) 230*cda5da8dSAndroid Build Coastguard Worker 231*cda5da8dSAndroid Build Coastguard Worker if base_dir is None: 232*cda5da8dSAndroid Build Coastguard Worker base_dir = os.curdir 233*cda5da8dSAndroid Build Coastguard Worker 234*cda5da8dSAndroid Build Coastguard Worker kwargs = {'dry_run': dry_run} 235*cda5da8dSAndroid Build Coastguard Worker 236*cda5da8dSAndroid Build Coastguard Worker try: 237*cda5da8dSAndroid Build Coastguard Worker format_info = ARCHIVE_FORMATS[format] 238*cda5da8dSAndroid Build Coastguard Worker except KeyError: 239*cda5da8dSAndroid Build Coastguard Worker raise ValueError("unknown archive format '%s'" % format) 240*cda5da8dSAndroid Build Coastguard Worker 241*cda5da8dSAndroid Build Coastguard Worker func = format_info[0] 242*cda5da8dSAndroid Build Coastguard Worker for arg, val in format_info[1]: 243*cda5da8dSAndroid Build Coastguard Worker kwargs[arg] = val 244*cda5da8dSAndroid Build Coastguard Worker 245*cda5da8dSAndroid Build Coastguard Worker if format != 'zip': 246*cda5da8dSAndroid Build Coastguard Worker kwargs['owner'] = owner 247*cda5da8dSAndroid Build Coastguard Worker kwargs['group'] = group 248*cda5da8dSAndroid Build Coastguard Worker 249*cda5da8dSAndroid Build Coastguard Worker try: 250*cda5da8dSAndroid Build Coastguard Worker filename = func(base_name, base_dir, **kwargs) 251*cda5da8dSAndroid Build Coastguard Worker finally: 252*cda5da8dSAndroid Build Coastguard Worker if root_dir is not None: 253*cda5da8dSAndroid Build Coastguard Worker log.debug("changing back to '%s'", save_cwd) 254*cda5da8dSAndroid Build Coastguard Worker os.chdir(save_cwd) 255*cda5da8dSAndroid Build Coastguard Worker 256*cda5da8dSAndroid Build Coastguard Worker return filename 257