545 lines
13 KiB
Python
545 lines
13 KiB
Python
#!/usr/bin/env python2.7 -B
|
|
|
|
from fnmatch import fnmatch
|
|
from glob import glob
|
|
from logging import debug, info, error
|
|
from os import path
|
|
import contextlib
|
|
from distutils import spawn, sysconfig
|
|
import os
|
|
import datetime
|
|
import shutil
|
|
import site
|
|
import subprocess
|
|
import sys
|
|
import tarfile
|
|
import tempfile
|
|
import urllib2
|
|
import zipfile
|
|
|
|
VARS = {}
|
|
|
|
|
|
def setvar(**kwargs):
|
|
for key, item in kwargs.items():
|
|
VARS[key] = item.format(**VARS)
|
|
|
|
|
|
def fill_in(value):
|
|
if type(value) == str:
|
|
return value.format(**VARS)
|
|
return value
|
|
|
|
|
|
def fill_in_args(fn):
|
|
def wrapper(*args, **kwargs):
|
|
args = list(fill_in(arg) for arg in args)
|
|
kwargs = dict((key, fill_in(value)) for key, value in kwargs.items())
|
|
return fn(*args, **kwargs)
|
|
return wrapper
|
|
|
|
|
|
def flatten(*args):
|
|
queue = list(args)
|
|
|
|
while queue:
|
|
item = queue.pop(0)
|
|
if type(item) == list:
|
|
queue = item + queue
|
|
elif type(item) == tuple:
|
|
queue = list(item) + queue
|
|
else:
|
|
yield item
|
|
|
|
|
|
chdir = fill_in_args(os.chdir)
|
|
path.exists = fill_in_args(path.exists)
|
|
path.join = fill_in_args(path.join)
|
|
path.relpath = fill_in_args(path.relpath)
|
|
|
|
|
|
@fill_in_args
|
|
def panic(*args):
|
|
error(*args)
|
|
sys.exit(1)
|
|
|
|
|
|
@fill_in_args
|
|
def cmpver(op, v1, v2):
|
|
assert op in ['eq', 'lt', 'gt']
|
|
|
|
v1 = [int(x) for x in v1.split('.')]
|
|
v2 = [int(x) for x in v2.split('.')]
|
|
|
|
def _cmp(l1, l2):
|
|
if not len(l1) and not len(l2):
|
|
return 0
|
|
if not len(l1):
|
|
return -1
|
|
if not len(l2):
|
|
return 1
|
|
|
|
if l1[0] < l2[0]:
|
|
return -1
|
|
if l1[0] > l2[0]:
|
|
return 1
|
|
if l1[0] == l2[0]:
|
|
return _cmp(l1[1:], l2[1:])
|
|
|
|
res = _cmp(v1, v2)
|
|
|
|
return ((op == 'eq' and res == 0) or
|
|
(op == 'lt' and res < 0) or
|
|
(op == 'gt' and res > 0))
|
|
|
|
|
|
@fill_in_args
|
|
def topdir(name):
|
|
if not path.isabs(name):
|
|
name = path.abspath(name)
|
|
return path.relpath(name, '{top}')
|
|
|
|
|
|
@fill_in_args
|
|
def find_executable(name):
|
|
return (spawn.find_executable(name) or
|
|
panic('Executable "%s" not found!', name))
|
|
|
|
|
|
@fill_in_args
|
|
def find(root, **kwargs):
|
|
only_files = kwargs.get('only_files', False)
|
|
include = kwargs.get('include', ['*'])
|
|
exclude = kwargs.get('exclude', [''])
|
|
lst = []
|
|
for name in sorted(os.listdir(root)):
|
|
fullname = path.join(root, name)
|
|
is_dir = path.isdir(fullname)
|
|
excluded = any(fnmatch(name, pat) for pat in exclude)
|
|
included = any(fnmatch(name, pat) for pat in include)
|
|
if included and not excluded:
|
|
if not (is_dir and only_files):
|
|
lst.append(fullname)
|
|
if is_dir and not excluded:
|
|
lst.extend(find(fullname, **kwargs))
|
|
return lst
|
|
|
|
|
|
@fill_in_args
|
|
def touch(name):
|
|
try:
|
|
os.utime(name, None)
|
|
except:
|
|
open(name, 'a').close()
|
|
|
|
|
|
@fill_in_args
|
|
def mkdtemp(**kwargs):
|
|
if 'dir' in kwargs and not path.isdir(kwargs['dir']):
|
|
mkdir(kwargs['dir'])
|
|
return tempfile.mkdtemp(**kwargs)
|
|
|
|
|
|
@fill_in_args
|
|
def mkstemp(**kwargs):
|
|
if 'dir' in kwargs and not path.isdir(kwargs['dir']):
|
|
mkdir(kwargs['dir'])
|
|
return tempfile.mkstemp(**kwargs)
|
|
|
|
|
|
@fill_in_args
|
|
def rmtree(*names):
|
|
for name in flatten(names):
|
|
if path.isdir(name):
|
|
debug('rmtree "%s"', topdir(name))
|
|
shutil.rmtree(name)
|
|
|
|
|
|
@fill_in_args
|
|
def remove(*names):
|
|
for name in flatten(names):
|
|
if path.isfile(name):
|
|
debug('remove "%s"', topdir(name))
|
|
os.remove(name)
|
|
|
|
|
|
@fill_in_args
|
|
def mkdir(*names):
|
|
for name in flatten(names):
|
|
if name and not path.isdir(name):
|
|
debug('makedir "%s"', topdir(name))
|
|
os.makedirs(name)
|
|
|
|
|
|
@fill_in_args
|
|
def copy(src, dst):
|
|
debug('copy "%s" to "%s"', topdir(src), topdir(dst))
|
|
shutil.copy2(src, dst)
|
|
|
|
|
|
@fill_in_args
|
|
def copytree(src, dst, **kwargs):
|
|
debug('copytree "%s" to "%s"', topdir(src), topdir(dst))
|
|
|
|
mkdir(dst)
|
|
|
|
for name in find(src, **kwargs):
|
|
target = path.join(dst, path.relpath(name, src))
|
|
if path.isdir(name):
|
|
mkdir(target)
|
|
else:
|
|
copy(name, target)
|
|
|
|
|
|
@fill_in_args
|
|
def move(src, dst):
|
|
debug('move "%s" to "%s"', topdir(src), topdir(dst))
|
|
shutil.move(src, dst)
|
|
|
|
|
|
@fill_in_args
|
|
def symlink(src, name):
|
|
if not path.islink(name):
|
|
debug('symlink "%s" points at "%s"', topdir(name), src)
|
|
os.symlink(src, name)
|
|
|
|
|
|
@fill_in_args
|
|
def chmod(name, mode):
|
|
debug('change permissions on "%s" to "%o"', topdir(name), mode)
|
|
os.chmod(name, mode)
|
|
|
|
|
|
@fill_in_args
|
|
def execute(*cmd):
|
|
debug('execute "%s"', " ".join(cmd))
|
|
try:
|
|
subprocess.check_call(cmd)
|
|
except subprocess.CalledProcessError as ex:
|
|
panic('command "%s" failed with %d', " ".join(list(ex.cmd)), ex.returncode)
|
|
|
|
|
|
@fill_in_args
|
|
def textfile(*lines):
|
|
f, name = mkstemp(dir='{tmpdir}')
|
|
debug('creating text file script "%s"', topdir(name))
|
|
os.write(f, '\n'.join(lines) + '\n')
|
|
os.close(f)
|
|
return name
|
|
|
|
|
|
@fill_in_args
|
|
def download(url, name):
|
|
info('download "%s" to "%s"', url, topdir(name))
|
|
|
|
u = urllib2.urlopen(url)
|
|
meta = u.info()
|
|
try:
|
|
size = int(meta.getheaders('Content-Length')[0])
|
|
except IndexError:
|
|
size = None
|
|
|
|
if size:
|
|
info('download: %s (size: %d)', name, size)
|
|
else:
|
|
info('download: %s', name)
|
|
|
|
with open(name, 'wb') as f:
|
|
done = 0
|
|
block = 8192
|
|
while True:
|
|
buf = u.read(block)
|
|
if not buf:
|
|
break
|
|
done += len(buf)
|
|
f.write(buf)
|
|
if size:
|
|
status = r"%d [%3.2f%%]" % (done, done * 100. / size)
|
|
else:
|
|
status = r"%d" % done
|
|
status = status + chr(8) * (len(status) + 1)
|
|
sys.stdout.write(status)
|
|
sys.stdout.flush()
|
|
|
|
print ""
|
|
|
|
|
|
@fill_in_args
|
|
def unarc(name):
|
|
info('extract files from "%s"', topdir(name))
|
|
|
|
if name.endswith('.lha'):
|
|
import lhafile
|
|
arc = lhafile.LhaFile(name)
|
|
for item in arc.infolist():
|
|
filename = os.sep.join(item.filename.split('\\'))
|
|
mkdir(path.dirname(filename))
|
|
debug('extract "%s"', filename)
|
|
if path.isdir(filename):
|
|
continue
|
|
with open(filename, 'w') as f:
|
|
f.write(arc.read(item.filename))
|
|
elif name.endswith('.tar.gz') or name.endswith('.tar.bz2'):
|
|
with tarfile.open(name) as arc:
|
|
for item in arc.getmembers():
|
|
debug('extract "%s"' % item.name)
|
|
arc.extract(item)
|
|
elif name.endswith('.zip'):
|
|
with zipfile.ZipFile(name) as arc:
|
|
for item in arc.infolist():
|
|
debug('extract "%s"' % item.filename)
|
|
arc.extract(item)
|
|
else:
|
|
raise RuntimeError('Unrecognized archive: "%s"', name)
|
|
|
|
|
|
@fill_in_args
|
|
def find_site_dir(dirname):
|
|
prefix = sysconfig.EXEC_PREFIX
|
|
destlib = sysconfig.get_config_var('DESTLIB')
|
|
return path.join(dirname, destlib[len(prefix) + 1:], 'site-packages')
|
|
|
|
|
|
@fill_in_args
|
|
def add_site_dir(dirname):
|
|
dirname = find_site_dir(dirname)
|
|
info('adding "%s" to python site dirs', topdir(dirname))
|
|
site.addsitedir(dirname)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def cwd(name):
|
|
old = os.getcwd()
|
|
if not path.exists(name):
|
|
mkdir(name)
|
|
try:
|
|
debug('enter directory "%s"', topdir(name))
|
|
chdir(name)
|
|
yield
|
|
finally:
|
|
chdir(old)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def env(**kwargs):
|
|
backup = {}
|
|
try:
|
|
for key, value in kwargs.items():
|
|
debug('changing environment variable "%s" to "%s"', key, value)
|
|
old = os.environ.get(key, None)
|
|
os.environ[key] = fill_in(value)
|
|
backup[key] = old
|
|
yield
|
|
finally:
|
|
for key, value in backup.items():
|
|
debug('restoring old value of environment variable "%s"', key)
|
|
if value is None:
|
|
del os.environ[key]
|
|
else:
|
|
os.environ[key] = value
|
|
|
|
|
|
def recipe(name, nargs=0):
|
|
def real_decorator(fn):
|
|
@fill_in_args
|
|
def wrapper(*args, **kwargs):
|
|
target = [str(arg) for arg in args[:min(nargs, len(args))]]
|
|
if len(target) > 0:
|
|
target = [target[0], fill_in(name)] + target[1:]
|
|
target = '-'.join(target)
|
|
else:
|
|
target = fill_in(name)
|
|
target = target.replace('_', '-')
|
|
target = target.replace('/', '-')
|
|
stamp = path.join('{stamps}', target)
|
|
|
|
if not path.exists('{stamps}'):
|
|
mkdir('{stamps}')
|
|
if not path.exists(stamp):
|
|
fn(*args, **kwargs)
|
|
touch(stamp)
|
|
else:
|
|
info('already done "%s"', target)
|
|
return wrapper
|
|
return real_decorator
|
|
|
|
|
|
@recipe('python-setup', 1)
|
|
def python_setup(name, **kwargs):
|
|
dest_dir = kwargs.get('dest_dir', '{host}')
|
|
with cwd(path.join('{build}', name)):
|
|
execute(sys.executable, 'setup.py', 'build')
|
|
execute(sys.executable, 'setup.py', 'install', '--prefix=' + dest_dir)
|
|
|
|
|
|
@recipe('fetch', 1)
|
|
def fetch(name, url):
|
|
if url.startswith('http') or url.startswith('ftp'):
|
|
if not path.exists(name):
|
|
download(url, name)
|
|
else:
|
|
info('File "%s" already downloaded.', name)
|
|
elif url.startswith('svn'):
|
|
if not path.exists(name):
|
|
execute('svn', 'checkout', url, name)
|
|
else:
|
|
with cwd(name):
|
|
execute('svn', 'update')
|
|
elif url.startswith('git'):
|
|
if not path.exists(name):
|
|
execute('git', 'clone', url, name)
|
|
else:
|
|
with cwd(name):
|
|
execute('git', 'pull')
|
|
elif url.startswith('file'):
|
|
if not path.exists(name):
|
|
_, src = url.split('://')
|
|
copy(src, name)
|
|
else:
|
|
panic('URL "%s" not recognized!', url)
|
|
|
|
|
|
@recipe('unpack', 1)
|
|
def unpack(name, work_dir='{sources}', top_dir=None, dst_dir=None):
|
|
try:
|
|
src = (glob(path.join('{archives}', name) + '*') +
|
|
glob(path.join('{submodules}', name) + '*'))[0]
|
|
except IndexError:
|
|
panic('Missing files for "%s".', name)
|
|
|
|
dst = path.join(work_dir, dst_dir or name)
|
|
|
|
info('preparing files for "%s"', name)
|
|
|
|
if path.isdir(src):
|
|
if top_dir is not None:
|
|
src = path.join(src, top_dir)
|
|
copytree(src, dst, exclude=['.svn', '.git'])
|
|
else:
|
|
tmpdir = mkdtemp(dir='{tmpdir}')
|
|
with cwd(tmpdir):
|
|
unarc(src)
|
|
copytree(path.join(tmpdir, top_dir or name), dst)
|
|
rmtree(tmpdir)
|
|
|
|
def removemodule(name):
|
|
mbuild = path.join('{build}', name)
|
|
info('removing build for module %s : "%s"', name, mbuild)
|
|
|
|
mstamp = path.join('{stamps}', name + '-*')
|
|
for f in glob(mstamp):
|
|
remove(f)
|
|
|
|
rmtree(mbuild)
|
|
|
|
def checkstamps(name):
|
|
target = fill_in(name)
|
|
target = target.replace('_', '-')
|
|
target = target.replace('/', '-')
|
|
if not path.exists('{stamps}'):
|
|
mkdir('{stamps}')
|
|
stamp = path.join('{stamps}', target + '-make')
|
|
info('checking %s with %s', name, stamp)
|
|
mtime = 0
|
|
if path.exists(stamp):
|
|
mtime = os.stat(stamp).st_mtime
|
|
|
|
ins = None
|
|
for n in find('{stamps}', include=[target + '*install*']):
|
|
ins = n
|
|
break
|
|
if ins == None:
|
|
remove(stamp)
|
|
|
|
submodule = path.join('submodules/', name)
|
|
for root, dirs, files in os.walk(submodule):
|
|
for filename in files:
|
|
mf = os.path.join(root, filename)
|
|
mt = os.stat(mf).st_mtime
|
|
if mt > mtime:
|
|
touch(stamp)
|
|
return True
|
|
return False
|
|
|
|
@recipe('patch', 1)
|
|
def patch(name, work_dir='{sources}'):
|
|
with cwd(work_dir):
|
|
for name in find(path.join('{patches}', name),
|
|
only_files=True, exclude=['*~']):
|
|
if fnmatch(name, '*.diff'):
|
|
execute('patch', '-t', '-p0', '-i', name)
|
|
else:
|
|
dst = path.relpath(name, '{patches}')
|
|
mkdir(path.dirname(dst))
|
|
copy(name, dst)
|
|
|
|
|
|
@recipe('configure', 1)
|
|
def configure(name, *confopts, **kwargs):
|
|
info('configuring "%s"', name)
|
|
|
|
if 'from_dir' in kwargs:
|
|
from_dir = kwargs['from_dir']
|
|
else:
|
|
from_dir = path.join('{sources}', name)
|
|
|
|
if kwargs.get('copy_source', False):
|
|
rmtree(path.join('{build}', name))
|
|
copytree(path.join('{sources}', name), path.join('{build}', name))
|
|
from_dir = '.'
|
|
|
|
with cwd(path.join('{build}', name)):
|
|
remove(find('.', include=['config.cache']))
|
|
execute(path.join(from_dir, 'configure'), *confopts)
|
|
|
|
|
|
@recipe('make', 2)
|
|
def make(name, target=None, makefile=None, **makevars):
|
|
info('running make "%s"', target)
|
|
|
|
with cwd(path.join('{build}', name)):
|
|
args = ['%s=%s' % item for item in makevars.items()] + ['-j{numThreads}']
|
|
if target is not None:
|
|
args = [target] + args
|
|
if makefile is not None:
|
|
args = ['-f', makefile] + args
|
|
execute('make', *args)
|
|
|
|
|
|
def require_header(headers, lang='c', errmsg='', symbol=None, value=None):
|
|
debug('require_header "%s"', headers[0])
|
|
|
|
for header in headers:
|
|
cmd = {'c': os.environ['CC'], 'c++': os.environ['CXX']}[lang]
|
|
cmd = fill_in(cmd).split()
|
|
opts = ['-fsyntax-only', '-x', lang, '-']
|
|
proc = subprocess.Popen(cmd + opts,
|
|
stdin=subprocess.PIPE,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
|
|
proc_stdin = ['#include <%s>' % header]
|
|
if symbol:
|
|
if value:
|
|
proc_stdin.append("#if %s != %s" % (symbol, value))
|
|
else:
|
|
proc_stdin.append("#ifndef %s" % symbol)
|
|
proc_stdin.append("#error")
|
|
proc_stdin.append("#endif")
|
|
|
|
proc_stdout, proc_stderr = proc.communicate('\n'.join(proc_stdin))
|
|
proc.wait()
|
|
|
|
if proc.returncode == 0:
|
|
return
|
|
|
|
panic(errmsg)
|
|
|
|
|
|
__all__ = ['setvar', 'fill_in', 'panic', 'cmpver', 'find_executable', 'chmod', 'execute',
|
|
'rmtree', 'mkdir', 'copy', 'copytree', 'unarc', 'fetch', 'cwd',
|
|
'symlink', 'remove', 'move', 'find', 'textfile', 'env', 'path',
|
|
'add_site_dir', 'find_site_dir', 'python_setup', 'recipe',
|
|
'unpack', 'patch', 'configure', 'make', 'require_header', 'touch',
|
|
'checkstamps', 'removemodule']
|