#!/usr/bin/python

# Copyright (c) 2010 Peter Palfrader
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.


# Imports all mirrorruns found in a git repository for dumps into the database.
# Just imports metadata, does not get any actualy content (i.e. does not
# populate the farm).

import sys
import errno
import yaml
import optparse
import os
import tempfile
import shutil
import subprocess
import fcntl
import time
thisscriptdir = os.path.abspath(os.path.dirname(sys.argv[0]))
sys.path.append(os.path.join(thisscriptdir, '../../lib'))
from dbhelper import DBHelper

block_size = 4096

def read_fd_to_file(i, path):
    o = open(path, "w")
    while True:
        buf = i.read(block_size)
        if not buf: break
        o.write(buf)
    o.close()
    i.close()

def get_lock(fn, wait=5*60):
    f = open(fn, "w")
    sl = 0.1
    ends = time.time() + wait

    while True:
        success = False
        try:
            fcntl.flock(f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
            return f
        except IOError:
            pass
        if time.time() >= ends:
            return None
        sl = min(sl*2, 10, ends - time.time())
        time.sleep(sl)
    return f


parser = optparse.OptionParser()
parser.set_usage("%prog --config=<conffile>")
parser.add_option("-c", "--config", dest="conffile", metavar="CONFFILE",
  help="Config file location.")
parser.add_option("-v", "--verbose", action="store_true",
  help="Be verbose.")
parser.add_option("-s", "--snapshot", action="store", metavar="PATH",
  help="Path to the snapshot script.")
parser.add_option("-a", "--adder", action="store", metavar="PATH",
  help="Path to the add-dump-to-git script.")
parser.add_option("-e", "--extracter", action="store", metavar="PATH",
  help="Path to the extract-dumps script.")
parser.add_option("-b", "--backing", dest="backing_git", metavar="GITDIR",
  default = 'backing-git',
  help="Location of backing git working copy.")

(options, args) = parser.parse_args()
if options.conffile is None:
    parser.print_help()
    sys.exit(1)

if options.snapshot is None:
    options.snapshot = os.path.join(thisscriptdir, '../../snapshot')
if options.extracter is None:
    options.extracter = os.path.join(thisscriptdir, 'extract-dumps')
if options.adder is None:
    options.adder = os.path.join(thisscriptdir, 'add-dump-to-git')

config = yaml.load(open(options.conffile).read())
db = DBHelper(config['db']['connectstring'])

if not os.path.exists(options.snapshot) or not os.access(options.snapshot, os.X_OK):
    sys.stderr.write("%s does not exist or is not exectuable\n"%(options.snapshot))
    sys.exit(1)
if not os.path.exists(options.extracter) or not os.access(options.extracter, os.X_OK):
    sys.stderr.write("%s does not exist or is not exectuable\n"%(options.extracter))
    sys.exit(1)
if not os.path.exists(options.adder) or not os.access(options.adder, os.X_OK):
    sys.stderr.write("%s does not exist or is not exectuable\n"%(options.adder))
    sys.exit(1)

options.extracter = os.path.abspath(options.extracter)
extractcall = [options.extracter]
extractcall += ['--backing', options.backing_git]

options.adder = os.path.abspath(options.adder)
addcall = [options.adder]
addcall += ['--backing', options.backing_git]
if options.verbose: addcall += ['--verbose']

if options.verbose: optional_quiet = []
else:               optional_quiet = ['--quiet']

tmpdir = None
lock = None
counter = 1
try:
    tmpdir = tempfile.mkdtemp(prefix='snapshot.add-new-dumps-to-git.')
    lockfilename = os.path.join(options.backing_git, '.lock-add-new-dumps-to-git')
    lockfile = get_lock(lockfilename)
    if lockfile is None:
        sys.stderr.write("Could not acquire lock.\n")
        sys.exit(1)

    have_uuids = set()
    p = subprocess.Popen(extractcall+['list'], stdout=subprocess.PIPE)
    for line in p.stdout:
        line = line.rstrip()
        uuid, objectspec = line.split(None, 1)
        have_uuids.add(uuid)

    rows = db.query('SELECT mirrorrun_id, mirrorrun_uuid FROM mirrorrun ORDER BY run')
    for row in rows:
        if row['mirrorrun_uuid'] in have_uuids:
            continue

        if options.verbose: print "Doing %d (%s)."%(row['mirrorrun_id'], row['mirrorrun_uuid'])

        fn = os.path.join(tmpdir, "%d"%(row['mirrorrun_id']))

        # dump mirrorun to fn
        c = [options.snapshot, '-c', options.conffile, '--mirrorrun', '%d'%(row['mirrorrun_id'])]
        if options.verbose: c.append('--verbose')
        c.append('dump')
        if options.verbose: print " [%s]"%(" ".join(c))
        p = subprocess.Popen(c, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
        p.stdin.close()
        read_fd_to_file(p.stdout, fn)
        p.wait()

        # import into git
        args = []
        counter += 1
        if counter % 50 != 0:
            args.append('--no-gc')
        args.append(fn)
        subprocess.check_call(addcall+args)

        os.unlink(fn)

    db.close()

    os.chdir(options.backing_git)
    if options.verbose: print "# git gc --auto"; sys.stdout.flush()
    subprocess.check_call(['git', 'gc', '--auto'] + optional_quiet)
    lockfile.close()
finally:
    if not tmpdir is None:
        shutil.rmtree(tmpdir)

# vim:set et:
# vim:set ts=4:
# vim:set shiftwidth=4: