#!/bin/sh
# -*- perl -*-
# This code allows us to start perl from our path or an environment variable
# rather than hardcoding a path into the #! line.  It works from sh or csh.
(exit $?0) && eval 'exec ${QPERLQ-perl} -x $0 ${1+"$@"}'
if (! $?QPERLQ) setenv QPERLQ perl
exec $QPERLQ -x $0 $argv:q

#!/usr/local/bin/perl -w
#
# $Id: make_sync,v 1.84 1999/02/12 19:56:59 ejb Exp $
# $Source: /home/ejb/source/qsync/util/RCS/make_sync,v $
# $Author: ejb $
#
# Author: E. Jay Berkenbilt
#
# This program is used to make a synchronization package to be
# extracted by the program "apply_sync".  This program generates a
# qsfiles database by running "qsfiles" with a specified filter.  It
# then runs "qsdiff" to generate a list of changes that it can use to
# create a synchronization package suitable for apply_sync to
# extract. 
#
# This script can be used to do unidirectional or bidirectional
# synchronization between non-connected file systems, to do backups,
# to pack files for other sites, and undoubtedly for other uses as
# well.  When make_sync is run, it must be given a synchronization
# directory.  This directory may contain a configuration file called
# qsync.conf.  The configuration file may specify several parameters.
# Each line in the configuration file is a colon-separated key-value
# pair.  The keys along with their meanings and default values are
# described below.  All pathnames are taken to be relative to the
# synchronization directory passed to make_sync unless otherwise
# specified.  If no qsync.conf file exists, defaults will be used for
# all parameters.
# 
#   filter: filter_file
#     Default: filter if it exists, or ../filter if
#	filter does not exist and the synchronziation directory, as
#	specified, has at least two components to its pathname.
#     Meaning: This is the filter file that qsfiles and qsdiff use.
#       It determines what files are to be included and not included
#       in the synchronziation.  Note: all though the path to the
#       filter file itself is given relative to the sync directory,
#       files mentioned within the filter file are relative to the
#       current directory from which make_sync, and therefore qsfiles
#       and qsdiff are run.
#
#   last_sync: last_files_database
#     Default: last_sync
#     Meaning: this qsfiles database contains the last known state of
#       the file system that we need to synchronize with our current
#       state.  Differences between the current state and this file
#       determine what actions should be taken.  The filename
#       specified is taken to be relative to the synchronization if
#       peer is not specified or to the peer directory if it is
#       specified. 
#       
#   new_sync: new_files_database_name
#     Default: new_sync
#     Meaning: this is the name of the qsfiles database that we will
#       create to contain our current state.  It must not exist.  If
#       the rename_to key is specified, the program will offer to
#       remove or rename the file if it exists.  Otherwise, it is an
#       error for this file to exist.
#       
#   peer: peer_directory
#     Default: none
#     Meaning: If specified, the last_sync database is expected to be
#       in this directory and a configuration file with the same name
#       as ours in that directory is implicitly included in the
#       synchronization.  This is typically used for bidirectional
#       synchronization.  In this case, there is generally a directory
#       with two subdirectories of each of which refer to the other as
#       a peer.  It is possible to tell make_sync from the commandline
#       to ignore peer specification.
#       
#   rename_to: filename
#     Default: if neither the last_sync nor the new_sync key was
#       specified, the default value for this parameter is
#       "last_sync".  Otherwise, there is no default value.
#     Meaning: If specified, make_sync will offer to delete new_sync
#       or rename it to this if new_sync exists when the program
#       starts.  Also, apply_sync will rename new_sync to
#       this when it finishes if there are no errors.
#       
#   live: absolute_path
#     Default: none
#     Meaning: If specified, make_sync assumes that has live access to
#	the file system to be updated.	If this option is specified,
#	last_sync, new_sync, peer, rename_to, and umask are all
#	ignored, and the -live option to qsdiff is used to generate
#	file system differences.  No database files are used in this
#	case.  The other file system is expected to have a sync_dir in
#	it with the same name as the one used for this
#	synchronization.
#
#   umask:
#     Default: none
#     Meaning: Ignored if peer is not specified.  If peer is
#       specified, it must also contain a qsync.conf file that
#       specifies a umask.  In this case, the umasks should reflect
#       the umask that is in place in each location, and files whose
#       protection differences are explainable based on that alone are
#       not considered to have different protections.
#       
#   prune:
#     Default: none
#     Meaning: The value of this keyword is a space-separated list of
#       directories to be pruned by qsfiles.  The directories are all
#       given as arguments to the -prune flag to qsfiles.  Directories
#       in this list are not traversed, but they themselves are
#       considered.  See qsfiles --help for more information.
#
#   extra_qsfiles_args:
#     Default: none
#     Meaning: This is a catch-all -- whatever string appears here
#       will simply be passed on to qsfiles (or qsdiff in live mode).
#
#   backup:
#     Default: false
#     Meaning: If true, defaults are changed to be suitable for using
#       make_sync to generate backups.  This causes exclude_cores to
#       be false, junkfiles to be preserve, nonfile_mtimes to be true,
#       ownerships to be true, and one_filesys to be true.  See below
#       for discriptions and normal default values for those fields.
# 
# The remaining items on the configuration file may be overridden from
# the commandline. 
#
#   exclude_cores:
#     Default: true
#     Meaning: must be true or false.  If true, */core is implicitly
#       excluded. 
#
#   junkfiles:
#     Default: cleanup
#     Meaning: Must be remove, ignore, or preserve.  Certain normally
#       junk files are either removed automatically, ignored, or
#       treated normally as they are encountered.  A value of remove
#       here causes -cleanup to be passed to qsfiles during the
#       generation of the new database.  A value of ignore causes the
#       -ignore flag to be passed to qsfiles. 
#       
#   nonfile_mtimes:
#     Default: false
#     Meaning: Must be true or false  Determines whether
#       differences in directory modification times are considered
#       important. 
#
#   ownerships:
#     Default: false
#     Meaning: Must be true or false.  Determines whether differences
#       in ownership are considered important.
#
#   one_filesys:
#     Default: false
#     Meaning: Must be true or false.  Determines whether file system
#       boundaries should be traversed while generating databases.  A
#       value of true here causes -xdev to be passed to qsfiles and
#       therefore means that mountpoints should not be traversed.
#       
#   tmpdir:
#     Default: /tmp
#     Meaning: Path to directory to be used for temporary files.
#       make_sync (as of version 1.3) uses this only when in samba
#       mode in which it needs to create files to copy to the server
#       and to operate on files copied from the server.
#
# This program can be invoked in a mode that generates the list
# of files to go into package and creates the package, or it can
# be run in a mode that causes just the generation or just the
# packaging.  See the usage message (defined in the usage
# subroutine) for more detail on the invocation of the program.
#

use strict;

$main::progname = ($0 =~ m,([^/\\]*)$,) ? $1 : $0;
my $whoami = $main::progname;
my $dirname = ($0 =~ m,(.*)[/\\][^/\\]+$,) ? $1 : ".";
push(@INC, "$dirname/qsutil_modules");

require qsync_utils;
require QsyncConfig;

if ((@ARGV == 1) && ($ARGV[0] eq "--help"))
{
    &help(*STDOUT);
    exit 0;
}

if ((@ARGV == 1) && ($ARGV[0] eq "--version"))
{
    print "$whoami version 1.4 -- requires qsync version 1.6 or later\n";
    exit 0;
}

&usage() unless (@ARGV >= 1);

# Global information
my $sync_dir = shift(@ARGV);
$sync_dir =~ s:/$::;
$sync_dir =~ s:^\./::;
# Other configuration options
my $to_stdout = 0;
my $make_tar = 1;
my $gen_data = 1;
my $quiet = 0;
my $slow = 0;
my $blocksize = 20;

my $tarbase;
{
    my $lastcomp = ($sync_dir =~ m,([^/]*)$,) ? $1 : $sync_dir;
    $tarbase = "sync.$lastcomp.$$.tar.gz";
}
my $tarfile = "/tmp/$tarbase";
my $backupdb = undef;

my $data_dir = undef;
{
    my $i;
    for ($i = 0; $i < scalar(@ARGV) - 1; $i++)
    {
	if ($ARGV[$i] eq "-datadir")
	{
	    $data_dir = $ARGV[$i+1];
	    last;
	}
    }
}

{
    my $check_dir = $data_dir || $sync_dir;
    if (! -d $check_dir)
    {
	&gethelp("$whoami: $check_dir does not exist or is not a directory");
    }
}

my $config;
&parse_argv(@ARGV);

if ($config->samba())
{
    # We need to verify sync dir on the remote server.  If it exists,
    # we need to copy all the files from there into data_dir and then
    # reparse commandline arguments!  This is to prevent problems arising
    # from invalid data_dir contents prior to reconciliation.

    &copy_from_samba($config);

    &parse_argv(@ARGV);

    if ($config->samba() && $config->live())
    {
	die "$whoami: samba mode and live mode cannot be used together.\n";
    }
    
}

$config->validate_config($to_stdout, $gen_data);

my $pending = $config->pending();
my $actions = $config->actions();
my $tarlist = $config->tarlist();
my $olddata = $config->olddata();
my $diff = $config->diff();

$| = 1;

{
    my $exit = new OnExit(\&del_pending, [$pending, $config]);

    open(PENDING, ">$pending") || 
	die "$whoami: can't write $pending: $!\n";
    close(PENDING);
    # Do an explicit modification time update to avoid cases in which
    # opening an empty file for writing and then not writing to it fails
    # to force the modification time to be updated.  This also works around
    # a possible NFS race condition.
    my $now = time;
    utime $now, $now, $actions;

    if ($gen_data)
    {
	&gen_data();
    }
    
    if ($make_tar)
    {
	&make_tar();
    }
    else
    {
	if (! $quiet)
	{
	    if (! -z $actions)
	    {
		print STDERR "$whoami: the following actions are required:\n";
		&qsystem("cat $actions 1>&2");
	    }
	    
	    print STDERR "$whoami: tarfile would contain the following:\n";
	    &qsystem("cat $tarlist 1>&2");
	}
    }
}

exit 0;

sub parse_argv
{
    my @ARGV = @_;
    $config = new QsyncConfig($whoami, $sync_dir, $data_dir);

    while (@ARGV)
    {
	my $arg = shift(@ARGV);
	if ($arg eq "-tarfile")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -tarfile requires an argument");
	    }
	    $tarfile = shift(@ARGV);
	}
	elsif ($arg eq "-tardir")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -tardir requires an argument");
	    }
	    $tarfile = shift(@ARGV) . "/" . $tarbase;
	    $tarfile =~ s,//+,/,g;
	}
	elsif ($arg eq "-backupdb")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -backupdb requires an argument");
	    }
	    $backupdb = shift(@ARGV);
	}
	elsif ($arg eq "-to-stdout")
	{
	    $to_stdout = 1;
	}
	elsif ($arg eq "-no-tar")
	{
	    $make_tar = 0;
	}
	elsif ($arg eq "-use-existing")
	{
	    $gen_data = 0;
	}
	elsif ($arg eq "-quiet")
	{
	    $quiet = 1;
	}
	elsif ($arg eq "-slow")
	{
	    $slow = 1;
	}
	elsif ($arg eq "-samba")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -samba requires an argument");
	    }
	    $config->parse_samba(shift(@ARGV));
	}
	elsif ($arg eq "-blocksize")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -blocksize requires an argument");
	    }
	    $blocksize = shift(@ARGV);
	}
	elsif ($arg eq "-datadir")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -datadir requires an argument");
	    }
	    $data_dir = shift(@ARGV);
	}
	elsif ($arg eq "-include-cores")
	{
	    $config->set_config("exclude_cores", "false");
	}
	elsif ($arg eq "-junkfiles")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -junkfiles requires an argument");
	    }
	    $config->set_config("junkfiles", shift(@ARGV));
	}
	elsif ($arg eq "-nonfile-mtimes")
	{
	    $config->set_config("nonfile_mtimes", "true");
	}
	elsif ($arg eq "-ownerships")
	{
	    $config->set_config("ownerships", "true");
	}
	elsif ($arg eq "-one-filesys")
	{
	    $config->set_config("one_filesys", "true");
	}
	elsif ($arg eq "-no-peer")
	{
	    $config->set_config("peer", "");
	}
	elsif ($arg eq "-tmpdir")
	{
	    if (@ARGV < 1)
	    {
		&gethelp("$whoami: -tmpdir requires an argument");
	    }
	    $config->set_config("tmpdir", shift(@ARGV));
	}
	else
	{
	    &gethelp("$whoami: unrecognized argument $arg");
	}
    }

    if (($gen_data == 0) && ($make_tar == 0))
    {
	print STDERR
	    "$whoami: Nothing to do with both -no-tar and -use-existing.\n";
	exit(1);
    }

    if (($config->samba() != 0) != (defined $data_dir))
    {
	die "$whoami: -datadir and -samba must always occur together.\n";
    }

    if ((defined $data_dir) && ($data_dir ne $config->data_dir()))
    {
	die "$whoami: data_dir has been set inconsistently probably due to " .
	    "invalid arguments.\n";
    }
}

sub gen_data
{
    my $samba = $config->samba();

    # Must open these files before running qsfiles so that they will
    # get picked up in the synchronization.

    # There may be some NFS race condition that prevents just opening
    # the files from working in some cases.  We will do an explicit
    # modification time update.
    my $now = time;
    utime $now, $now, $actions;
    utime $now, $now, $tarlist;
    utime $now, $now, $diff;
    utime $now, $now, $olddata;
    utime $now, $now, $backupdb if defined $backupdb;

    # Open and close these files now so that they will make it into the
    # database.  In win32, open files can't be statted, so we can't leave
    # these open.
    &touch($actions);
    &touch($tarlist);
    &touch($diff);
    &touch($olddata);
    &touch($backupdb) if defined $backupdb;

    if ($config->live())
    {
	my $live_path = $config->get_config("live");
	my $qsfiles_args = &get_qsfiles_args($config);
	my $qsdiff_args = &get_qsdiff_args($config);

	print STDERR "$whoami: generating file system differences\n";
	if (&qsystem("qsdiff $qsdiff_args " .
		     "-live $qsfiles_args $live_path -endlive " .
		     "-live $qsfiles_args . -endlive " .
		     "> $diff") != 0)
	{
            die "$whoami: qsdiff of . and $live_path failed.\n";
	}
    }
    else
    {
	my $new_sync = $config->get_config("new_sync");
	my $last_sync = $config->get_config("last_sync");

	print STDERR "$whoami: generating files database\n";
	
	if ($samba)
	{
	    local(*T);
	    open(T, ">$new_sync") && close(T);

	    &copy_to_samba($config);

	    my ($host, $service, $user, $pass, $dir) =
		($config->samba_config()->host(),
		 $config->samba_config()->service(),
		 $config->samba_config()->user(),
		 $config->samba_config()->pass(),
		 $config->samba_config()->dir()
		 );

	    $user = "-" if $user eq "";
	    $pass = "-" if $pass eq "";
	    $dir =~ s/[^\w\s]/\\$&/g;
	    
	    my $slowflag = ($slow ? "-S" : "");
	    (&qsystem("qsfiles_from_samba $slowflag " .
		      "$host $service $user $pass $dir " .
		      "> $new_sync") == 0) ||
			  die "$whoami: qsfiles_from_samba failed.\n";
	}
	else
	{
	    my $qsfiles_args = &get_qsfiles_args($config);
	    (&qsystem("qsfiles $qsfiles_args -db $new_sync . 1>&2") == 0) ||
		die "$whoami: qsfiles command failed.\n";
	}
	
	if (defined($backupdb))
	{
	    unlink $backupdb;
	    system("cp -p $new_sync $backupdb");
	}

	my $qsdiff_args = &get_qsdiff_args($config);
	print STDERR "$whoami: generating files database differences\n";
	(&qsystem("qsdiff $qsdiff_args $last_sync $new_sync > $diff") == 0) ||
	    die "$whoami: qsdiff $last_sync $new_sync failed.\n";
    }

    my @tarlist = ();
    my @actions = ();
    my @olddata = ();

    print STDERR "$whoami: processing differences\n";
    &process_diff($config, $diff, \@tarlist, \@actions, \@olddata);

    # Gnu tar understands typical \ codes, so we need to quote all \'s in
    # the file. 
    open(TARLIST, ">$tarlist") ||
	die "$whoami: can't write $tarlist: $!\n";
    for (@tarlist)
    {
	if ($samba)
	{
	    # Change path separator to backslash and eliminate leading .
	    s,^\./+,,;
	    s,/,\\,g;
	}
	else
	{
	    # Handle \\ character in filenames
	    s,\\,\\\\,g;
	}
	print TARLIST "$_\n";
    }
    close(TARLIST);

    open(ACTIONS, ">$actions") ||
	die "$whoami: can't write $actions: $!\n";
    for (@actions)
    {
	print ACTIONS "$_\n";
    }
    close(ACTIONS);

    open(OLDDATA, ">$olddata") ||
	die "$whoami: can't write $olddata: $!\n";
    for (@olddata)
    {
	print OLDDATA "$_\n";
    }
    close(OLDDATA);

    if ($samba)
    {
	&copy_to_samba($config);
    }
}

sub touch
{
    my $file = shift;
    open(F, ">$file") or die "$whoami: can't write $file: $!\n";
    close(F);
}

sub make_tar
{
    if (! $quiet)
    {
	if (! -z $actions)
	{
	    print STDERR "$whoami: the following actions are required:\n";
	    &qsystem("cat $actions 1>&2");
	}
    }

    my $cmd;
    if ($config->samba())
    {
	my ($host, $service, $user, $pass, $dir) =
	    ($config->samba_config()->host(),
	     $config->samba_config()->service(),
	     $config->samba_config()->user(),
	     $config->samba_config()->pass(),
	     $config->samba_config()->dir()
	     );

	my $user_args = ($user eq "" ? "" : "-U $user");
	my $pass_args = ($pass eq "" ? "-N" : $pass);
	$dir =~ s/[^\w\s]/\\$&/g;
	$cmd = "smbclient '\\\\$host\\$service' $pass_args $user_args " .
	    "-E -D $dir -d 0 -TqcbF $blocksize - $tarlist";
    }
    else
    {
	my $tar_command = "";
	$tar_command = (&qsystem("gtar --version >/dev/null 2>&1") == 0)
	    ? "gtar" : "tar";
	
	if (&qsystem("$tar_command --version >/dev/null 2>&1") != 0)
	{
	    print STDERR "$whoami: gnu tar is required to make tarfile " .
		"automatically.\n";
	    print STDERR "$whoami: please make tarfile by hand from " .
		"$tarlist.\n";
	    exit(1);
	}
	my $verbose = ($quiet ? "" : "-v");
	$cmd = "$tar_command -c $verbose --sparse --atime-preserve -f - " .
	    "-b $blocksize --files-from $tarlist";
    }
    my $output = ($to_stdout ? "" : " | gzip > $tarfile");
    $cmd .= " $output";

    if ($output ne "")
    {
	print STDERR "$whoami: creating tarfile ($tarfile)\n";
    }
    else
    {
	print STDERR "$whoami: writing to stdout\n";
    }
    if (&qsystem($cmd) != 0)
    {
	print STDERR "$whoami: $cmd failed\n";
	exit(1);
    }
    if ($output ne "")
    {
	my @stat = stat($tarfile);
	my $size = (scalar(@stat) ? $stat[7] : '?');
	print STDERR
	    "$whoami: synchronization file in $tarfile ($size bytes)\n";
    }
    else
    {
	print STDERR "$whoami: synchronization file written to stdout\n";
    }
}

sub del_pending
{
    my ($pending, $config) = @_;
    if (-f $pending)
    {
	unlink $pending;
	if ($config->samba())
	{
	    my ($smb, $spec) = &get_smb($config);
	    $smb->delete("pending");
	    undef $smb;
	}
    }
}

sub help
{
    my $conf_name = &QsyncConfig::conf_name();

    local(*F) = shift;
    print F <<EOF
Usage: $whoami sync_dir [ options ]
  sync_dir must contain a file called $conf_name.  See comments in
  $0 for details about the format of this file.

  Options can be from among the following:

    -tarfile file    overrides default output filename of the synchronization
	             package.  Refers to a gzipped tarfile
    -tardir fil e    overrides default output directory of the synchronization
	             package.  Refers to a gzipped tarfile
    -to-stdout	     do not compress output; write output to stdout.
	             This option also suppress all user interaction.
    -backupdb file   make a backup copy of new_sync to file before
		     packaging. 
    -no-tar	     do not create a tarfile; just generate tarlist
		     and actions files.
    -use-existing    create tarfile from existing tarlist and actions
		     files. 
    -quiet	     print diagnostic and status information but no
		     lists of files.
    -slow	     causes make_sync to slow down the generation of the
	             files database in samba mode.  This flag should
	             be specified when backing up Windows 95 shares to
	             work around a Windows 95 bug.
    -samba [user[/pass]@]//host/service[:dir]
	             This flag indicates that the file system to be
	             synchronized is located in dir, relative to the
	             top of SMB share \\host\service (or relative to the
	             top of dir is not specified).  In this mode,
	             make_sync may create temporary files.  The
	             location of the temporary files may be specified
	             with the -tmpdir flag or the tempdir key in the
	             configuration file.  In this mode, $whoami
	             replaces qsfiles with qsfiles_from_samba and uses
	             smbclient to create the tarfile and to copy the
	             data files back and forth between the share and
	             the temporary directory.  There are some
	             limitations with this: for example, no prune
	             directories may currently be specified.  As of
	             version 1.0.2, apply_sync does not support
	             synchronization packages created this way.

  Options from here down override values in the configuration file.
  The defaults depend upon whether $whoami is run in backup mode
  or not.

    -include-cores   prevents implicit exclusion of core files
    -junkfiles cleanup | ignore | preserve
		     determines what should happen to files that are
		     normally junk files such as "*.bak", "*~", "#*",
		     and ".#*".  The default is ordinarily cleanup in
		     which case the files are deleted.
    -nonfile-mtimes  causes modification time differences on nonfiles
		     (such as devices or directories) to be
		     reconciled.
    -ownerships	     causes user and group ownership differences
		     to be reconciled.
    -one-filesys     causes mountpoints to not be traversed
    -no-peer	     causes a peer as specified in the configuration
		     file to be ignored.
    -tmpdir	     directory to be used for temporary files.
		     Currently, $whoami creates temporary files only
		     when run in samba mode.

      OR

  $whoami --help | --version
EOF
    ;
}

sub usage
{
    &help(*STDERR);
    exit(2);
}

sub gethelp
{
    my $msg = shift;
    warn $msg, "\n";
    die "Run $whoami --help for more information.\n";
}