aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorLaurent Bachelier <laurent@bachelier.name>2012-12-27 18:29:11 +0100
committerLaurent Bachelier <laurent@bachelier.name>2012-12-27 18:29:11 +0100
commit1452eaab9d8380f89f23e2404fe4fc79c9a31609 (patch)
tree1b9b970883d8fcf8ac37541240b59c273132e31c
parentIgnore setup.py artifacts (diff)
downloadbrutha-1452eaab9d8380f89f23e2404fe4fc79c9a31609.tar.xz
Chose "best" output and jobs
-rw-r--r--brutha/__main__.py17
-rw-r--r--brutha/output.py25
-rw-r--r--brutha/util.py28
3 files changed, 56 insertions, 14 deletions
diff --git a/brutha/__main__.py b/brutha/__main__.py
index c69e92c..14ea60e 100644
--- a/brutha/__main__.py
+++ b/brutha/__main__.py
@@ -5,11 +5,14 @@ import sys
from StringIO import StringIO
from brutha.tree import Tree
-from brutha.util import uprint
+from brutha.util import uprint, detect_cores, default_output
from brutha.output import PRINTERS, EXECUTORS
def main():
+ cores = detect_cores()
+ output = default_output(cores)
+
parser = argparse.ArgumentParser(
description='Sync FLAC music files to Ogg Vorbis (or keep lossy as-is).')
parser.add_argument('-q', '--quality', default=8, type=int,
@@ -18,8 +21,8 @@ def main():
help='Compute ReplayGain if missing')
parser.add_argument('-d', '--delete', action='store_true',
help='Delete extraneous files in destination')
- parser.add_argument('-o', '--output', default='sh', choices=PRINTERS.keys(),
- help='Command list type, default sh')
+ parser.add_argument('-o', '--output', default=output, choices=PRINTERS.keys(),
+ help='Command list type, default %s' % output)
parser.add_argument('-R', '--maxrate', type=int,
help='Maximum sample rate allowed (e.g. 44100)', metavar='RATE')
parser.add_argument('-B', '--maxbits', type=int,
@@ -28,6 +31,8 @@ def main():
help='Show started commands')
parser.add_argument('-x', '--execute', action='store_true',
help='Execute the script instead of printing it')
+ parser.add_argument('-j', '--jobs', type=int, default=cores,
+ help='Number of concurrent jobs, default %s' % cores)
parser.add_argument('src', help='Source directory', metavar='SOURCE')
parser.add_argument('dest', help='Destination directory', metavar='DESTINATION')
args = parser.parse_args()
@@ -41,11 +46,13 @@ def main():
else:
p = uprint(sys.stdout)
outf = PRINTERS[args.output]
- outf(p, tree.commands(), args.echo)
+ outf(p, tree.commands(), args.echo, args.jobs)
if args.execute:
+ jobs = args.jobs if args.output in ('parallel', 'make') else 1
+ print >>sys.stderr, 'Synchronizing %s to %s, using %s concurrent jobs.' % (args.src, args.dest, jobs)
exef = EXECUTORS[args.output]
- sys.exit(exef(s))
+ sys.exit(exef(s, args.jobs))
if __name__ == '__main__':
diff --git a/brutha/output.py b/brutha/output.py
index 02fe014..9368bba 100644
--- a/brutha/output.py
+++ b/brutha/output.py
@@ -3,7 +3,7 @@ from __future__ import division
import subprocess
-from brutha.util import find_executable
+from brutha.util import require_executable
def pbar(cur, total, color=True):
@@ -26,8 +26,8 @@ def pbar(cur, total, color=True):
# PRINTERS
-def sh(p, commands, echo=False):
- p('#!%s' % find_executable('sh', ['bash', 'zsh', 'dash', 'sh']))
+def sh(p, commands, echo=False, jobs=None):
+ p('#!%s' % require_executable('sh', ['bash', 'zsh', 'dash', 'sh']))
p('set -eu')
if echo:
p('set -x')
@@ -40,13 +40,15 @@ def sh(p, commands, echo=False):
p()
-def parallel(p, commands, echo=False):
- p('#!%s --shebang%s --' % (find_executable('parallel'), (' --verbose' if echo else '')))
+def parallel(p, commands, echo=False, jobs=None):
+ verbose = ' --verbose' if echo else ''
+ jobs = ' --jobs %s' % jobs if jobs else ''
+ p('#!%s --shebang%s%s --' % (require_executable('parallel'), jobs, verbose))
for i, subcommands in enumerate(commands):
p(" && ".join(subcommands + [pbar(i+1, len(commands))]))
-def make(p, commands, echo=False):
+def make(p, commands, echo=False, jobs=None):
prefix = '' if echo else '@'
targets = ' '.join('d%s' % i for i in xrange(0, len(commands)))
p('.PHONY: all %s' % targets)
@@ -65,7 +67,7 @@ PRINTERS = {'sh': sh, 'parallel': parallel, 'make': make}
# EXECUTORS
-def shebang(stream):
+def shebang(stream, jobs=None):
"""
Extracts the shebang (#!) line and runs it, with the script provided as stdin.
This is known to work at least with bash and GNU parallel.
@@ -79,8 +81,13 @@ def shebang(stream):
return p.returncode
-def emake(stream):
- p = subprocess.Popen([find_executable('make', ['gmake', 'make']), '-f', '-'], shell=False, stdin=subprocess.PIPE)
+def emake(stream, jobs=None):
+ if jobs:
+ addon = ['-j', str(jobs)]
+ else:
+ addon = []
+ p = subprocess.Popen([require_executable('make', ['gmake', 'make']), '-f', '-'] + addon,
+ shell=False, stdin=subprocess.PIPE)
p.communicate(stream.getvalue())
return p.returncode
diff --git a/brutha/util.py b/brutha/util.py
index ded3d90..723a374 100644
--- a/brutha/util.py
+++ b/brutha/util.py
@@ -1,6 +1,7 @@
# -*- coding: utf-8 -*-
import os
+import re
def escape(x):
@@ -35,4 +36,31 @@ def find_executable(name, names=None):
fpath = os.path.join(path, name) + ext
if os.path.exists(fpath) and os.access(fpath, os.X_OK):
return fpath
+
+
+def require_executable(name, names=None):
+ e = find_executable(name, names)
+ if e:
+ return e
raise Exception('Could not find executable: %s' % name)
+
+
+def detect_cores():
+ try:
+ import numpy.distutils.cpuinfo
+ return numpy.distutils.cpuinfo.cpuinfo()._getNCPUs()
+ except ImportError:
+ try:
+ with open('/proc/cpuinfo') as f:
+ return len(re.findall('^processor\s+:\s+(\d+)$', f.read(), re.MULTILINE))
+ except IOError:
+ pass
+
+
+def default_output(cores=None):
+ if cores and cores > 1:
+ if find_executable('make', ['gmake', 'make']):
+ return 'make'
+ if find_executable('parallel'):
+ return 'parallel'
+ return 'sh'