. 2vars.sh

#needed for ${file%-+([0-9.])}
shopt -s extglob

# 1529836137.199681:  + cabal list
# 1529836138.574177:  <stdout>: commitBuffer: invalid argument (invalid character)
export LANG=C.UTF-8
export LC_ALL=C.UTF-8
locale

cabalhome=$workarea/home
buildlogs=$workarea/buildlogs
extrabin=$target/hackage-bin/versioned
path_pprotoc=$workarea/proto-lens-protoc-path
hooglepatched=$target/hoogle-patched

gpkg=$workarea/initial-ghc-pkg
#useful for manual investigation
morepack=$workarea/more-packages

# change this to /tmp/patched-builds for interactive.  we can't leave it $workarea because running 2 or more at once results in conflict due to chmod $workarea that happens at the end.
patchedbuilds=$workarea/patched-builds
completedbuilds=$workarea/sources

function rn () {
    local file
    file=$1
    shift
    rm -fr $patchedbuilds/$file
    mkdir -p /tmp/rn
    single_install $file $@ 2>&1 | ts '%.s: ' > /tmp/rn/$file.log
}

function gt () {
    local file
    file=$1
    shift
    HOME=$cabalhome cabal get $file
    pushd $file
    git config --global user.name "Ken Takusagawa"
    git config --global user.email "kenta@mit.edu"
    git init
    git add .
    git commit -m init
    if [ -e $HOME/i/do-with-stack/package-patches/packages/$file ]
    then for patch in $HOME/i/do-with-stack/package-patches/packages/$file/*.patch
         do cat $patch
            patch -p1 < $patch
            git add .
            git commit -m "old patch $(basename $patch)"
         done
         find . -name '*.rej'
    fi
}

allow_newer=--allow-newer

#my_gcc_dir=/afs/sipb.mit.edu/project/gcc-6plus

#my_gcc_extra_lib=--extra-lib-dir=$my_gcc_dir/lib64
my_gcc_extra_lib=

# we add the link paths globally because packages have dependencies which also need them.
function gen_link_flags() {
    local lib
    lib=$1
    echo "--extra-lib-dirs=$lib --ghc-option=-optl-Wl,-rpath,$lib"
}

# 2 = simple -v switch
# https://github.com/haskell/cabal/issues/5240ell/cabal/blob/master/Cabal/Distribution/Verbosity.hs
cabal_verbosity=deafening+nowrap
# 1 is too little for understanding *.h errors.

offline=--offline

function single_install() {
    local file checksum profiling usesse2 savepath minifile extra_flags oldstuff g x didfail extra_link
    didfail=0
    file=$1
    shift
    #remove version suffix
    minifile=${file%-+([0-9.])}

    # see perldoc -f unpack
    checksum=$(perl -we 'printf("%01x",unpack("%8C*",$ARGV[0])%16)' "$file")

    extra_flags=$(bash extraflags.sh "$file")
    extra_link=""

    # for proto-lens-protobuf-types, which uses proto-lens-protoc which has gotten compiled earlier and so has appeared in the PATH .
    hash -r

    #remaining positional parameters assumed to be flags
    # ghc-vis-0.8
    if [ \( $minifile = ghc-vis \) -o \( $minifile = ghc-heap-view \) -o \( $file = tds-0.1.0.0 -a $ghc_version = 8.6.4 \) -o \( $file = hmatrix-tests-0.19.0.0 -a $ghc_version = 8.6.4 \) -o \( $file = hmatrix-tests-0.19.0.0 -a $ghc_version = 8.6.5 \) -o \( $file = lame-0.1.1 -a $ghc_version = 8.6.5 \) \
            -o \( $file = lame-0.2.0 -a $ghc_version = 8.6.5 \) \
            -o \( $file = tds-0.1.0.0 -a $ghc_version = 8.6.5 \) \
            -o \( $ghc_version = 8.6.5 -a $file = hasktorch-signatures-partial-0.0.1.0 \) \
            -o \( $ghc_version = 8.6.5 -a $file = hasktorch-signatures-support-0.0.1.0 \) \
       ]
       # tds, 24 hours and still trying to compile Proto.  2.4 days still.
       # not sure how to deal with later ghc versions also failing
       # hmatrix-tests "out of memory condition"
       # lame: tried (under profiling) --ghc-option=-fsimpl-tick-factor=10000, which ate 32 GB ram + 35 gb swap (but other processes running), and ghc died.
       # hasktorch-signatures-(partial,support) is doing something complicated with backpack: Failed to load interface for 'Torch.Sig.State' Perhaps you haven't installed the profiling libraries for package 'hasktorch-signatures-types-0.0.1.0'?
    then profiling=
    else # OPTIMIZATION for first pass, omit -p
        profiling=$default_profiling
    fi

    #if [ $minifile = nettle ]
    #then extra_flags=--extra-include-dirs=$nettle_root/usr/include
    #fi

    # hruby it seems to have an autodetect feature nowadays, at least as exercised by language-puppet
    #if [ $minifile = hruby ]
    #then #extra_flags=--configure-option=--rubyversion=20\ --rubyinc=/usr/include/ruby-2.0.0\ --rubyinc=/usr/include/x86_64-linux-gnu/ruby-2.0.0\ --rubylib=ruby-2.0
        # could not figure out how to quote the above.  note, it has to become nothing, not an empty string, when empty
        # maybe we want rubyversion=21 despite everything else being 2.5.0
    #    extra_flags=--configure-option=--rubyversion=25\ --configure-option=--rubyinc=/usr/include/ruby-2.5.0\ --configure-option=--rubyinc=/usr/include/x86_64-linux-gnu/ruby-2.5.0\ --configure-option=--rubylib=ruby-2.5
    #fi
    if [ \( $minifile = jni \) -o  \( $minifile = HJVM \)  -o \( $minifile = android-activity \) -o \( $minifile = jvm \) ]
    then extra_flags="--extra-include-dirs=/usr/lib/jvm/default-java/include --extra-include-dirs=/usr/lib/jvm/default-java/include/linux"
         #having to include both is probably a bug in packaging or source
         extra_link=$(gen_link_flags /usr/lib/jvm/default-java/lib/server)
    fi
    if [ $minifile = android-activity ]
    then extra_flags="$extra_flags --extra-include-dirs=/usr/include/android"
         # android/log.h is also available in /usr/include/android/log
         # and other packages
    fi
    if [ $minifile = PostgreSQL ]
    then extra_flags="--extra-include-dirs=/usr/include/postgresql"
    fi
    if [ $minifile = haskell-mpi ]
    then extra_flags="--extra-include-dirs=/usr/include/mpi"
    fi
    if [ $minifile = FreeTypeGL ]
    then extra_flags="--extra-include-dirs=/usr/include/freetype2"
    fi
    if [ \( $minifile = al \) -o \( $minifile = sfml-audio \) ]
    then extra_flags="--extra-include-dirs=/usr/include/AL"
    fi
    if [ $minifile = libxml ]
    then extra_flags="--extra-include-dirs=/usr/include/libxml2"
        #might still fail because it is a direct gcc call
    fi
    if [ $minifile = HsSVN ]
    then #extra_flags="--extra-include-dirs=/usr/include/subversion-1"
        extra_flags="--configure-option=--with-subversion-prefix=/usr"
    fi
    if [ $minifile = hslibsvm ]
    then extra_flags="--extra-include-dirs=/usr/include/libsvm"
    fi
    if [ $minifile = SDL-mpeg ]
    then extra_flags="--extra-include-dirs=/usr/include/smpeg --extra-include-dirs=/usr/include/SDL2"
    fi
    if [ $minifile = hCsound ]
    then extra_flags="--extra-include-dirs=/usr/include/csound"
    fi
    if [ $minifile = kqueue ]
    then extra_flags="--extra-include-dirs=/usr/include/kqueue"
    fi
    if [ $minifile = HsPerl5 ]
    then extra_flags="--extra-include-dirs=$(find /usr/lib/x86_64-linux-gnu/perl -name CORE|tail -1)"
    fi
    if [ $minifile = HABQT ]
       # it has a foreign library which is installed in a weird place.
    then extra_flags="--libdir=$target/lib-separated/$checksum/x86_64-linux-ghc-$ghc_version"
    fi
    if [ $minifile = llvm-ht ]
    then extra_flags="--extra-lib-dirs=/usr/lib/llvm-7/lib --extra-include-dirs=/usr/include/llvm-c-7"
         # no .so libraries available, so no need hardcode a LD_RUN_PATH
         # llvm-ht is mostly a lost cause.  it also needs a path with llvm-config in it.
         # even with this flag, and the path manually hacked, it's configure script fails.
         # checking for LLVMModuleCreateWithName in -lLLVMCore... no
         # configure: error: could not find LLVM C bindings
    fi
    if [ \( $minifile = bindings-hdf5 \) -o \( $minifile = hdf5-lite \) ]
       # this is where libhdf5-dev puts it
    then extra_flags="--extra-include-dirs=/usr/include/hdf5/serial"
         extra_link=$(gen_link_flags /usr/lib/x86_64-linux-gnu/hdf5/serial )
    fi
    if [ $minifile = ipopt-hs ]
    then extra_link=$(gen_link_flags /usr/lib/gcc/x86_64-linux-gnu/8 )
    fi
    if [ $minifile = nvvm ]
    then extra_link="--ghc-option=-optl-Wl,-rpath,/usr/local/cuda-10.0/nvvm/lib64/"
    fi
    if [ $minifile = Salsa ]
    then extra_flags="--extra-include-dirs=/usr/include/mono-2.0"
    fi
    if [ \( $minifile = ClustalParser \)  ]
       # it says default is 100.
    then extra_flags="--ghc-option=-fsimpl-tick-factor=1000"
    fi
    if [ \( $minifile = AutoForms \) -o \( $minifile = binding-wx \) ]
    then if [ ${LD_LIBRARY_PATH-} ]
         then true ERROR LD_LIBRARY_PATH has a value $LD_LIBRARY_PATH
              oldstuff=$LD_LIBRARY_PATH
         else unset oldstuff
              # this does not fail even if oldstuff is already unset
         fi
         set +e
         # we need libwxc.so
         # setting LD_LIBRARY_PATH is always a hack, but I can't think of a way to avoid it.
         # It is very convenient that wxcore provides the path to the libarary which is actually part of wxc.  wxcore obtains the path through sketchy means in its Setup.hs
         export LD_LIBRARY_PATH=$(ghc-pkg field wxcore ld-options | perl -nlwe 'die "ERROR no wxcore" unless m!^ld-options: "-Wl,-rpath,(.*)"$!;print$1')
         set -e
    fi
    if [ \( $minifile = c0parser \) -o \( $minifile = lame \) ]
       # c0parser: 1000 does not work but 2000 does.  requires a huge amount of ram, maybe 20 gb.
       # lame needs this only for profiling build, 1000 not enough.  Actually even 10000 is not enough (so see note above) but we'll try again with this limit for new GHC versions.
    then extra_flags="--ghc-option=-fsimpl-tick-factor=2000"
    fi
    if [ $minifile = hadoop-rpc ]
    then extra_flags="--ghc-option=-freduction-depth=0"
         # "Reduction stack overflow; size = 38"
         # 0 disables it, let us hope for the best.
    fi
    if [ $minifile = gf ]
    then oldstuff=$TERM
        export TERM=dumb
        # "unknown" ended poorly
    fi
    if [ $minifile = oden-go-packages ]
    then oldstuff=$PATH
        export PATH=$PATH:/usr/lib/go-1.10/bin
    fi

    # strace -f -o $stracedir/$file.strace         # OPTIMIZATION for first pass
    do_strace=
    #do_strace="strace -f -o $stracedir/$file.strace"

    #extra-lib-dir=$my_gcc_dir/lib is for clang-pure, this is the only way to get the link flag to hsc2hs which it uses
    local main_cabal_flags="--verbose=$cabal_verbosity --global --prefix=$target --bindir=$extrabin/$checksum/$file --libdir=$target/lib-separated/$checksum --ghc-option=-dppr-cols=999999999 --gcc-option=-fmessage-length=0 --enable-documentation --haddock-hoogle"
    if [ -e package-patches/packages/$file ]
    then mkdir -p $patchedbuilds
         if [ -e $patchedbuilds/$file ]
         then true ERROR $patchedbuilds/$file already exists
              didfail=1
         else pushd $patchedbuilds
              HOME=$cabalhome cabal get $file
              popd
              for patch in package-patches/packages/$file/*.patch
              do cat $patch
                 patch -d $patchedbuilds/$file -p1 < $patch || true ERROR patch failed
              done
              pushd $patchedbuilds/$file
              if ! HOME=$cabalhome nice time $do_strace cabal v1-install $allow_newer $offline $profiling $main_cabal_flags $my_gcc_extra_lib $extra_flags $extra_link "$@"
              then didfail=1
              fi
              popd
         fi
    else if ! HOME=$cabalhome nice time $do_strace cabal v1-install $allow_newer $offline $profiling $main_cabal_flags $my_gcc_extra_lib $extra_flags $extra_link "$@" $file
         then
             didfail=1
         fi
    fi
    if [ $didfail -eq 0 ]
    then true INFO ffgg good
         if [ "$( ghc-pkg field $file exposed )" = "exposed: True" ]
         then if ghc-pkg field $file exposed-modules | perl ghc-pkg-parse-exposed-modules.pl | perl ghc-pkg-should-be-hidden.pl $file $gpkg
              then ghc-pkg hide $file
              fi
         fi

         # make available an unpacked copy of sources for every successfully built package; useful for grepping for the proper way to do something.
         mkdir -p $completedbuilds
         pushd $completedbuilds
         HOME=$cabalhome cabal get $file
         popd
         # TODO: do git here
         if [ -e package-patches/packages/$file ]
         then for patch in package-patches/packages/$file/*.patch
              do patch -d $completedbuilds/$file -p1 < $patch || true ERROR patch failed $patch
              done
         fi
    else true INFO ffgg bad
         if [ ${impatient:-0} -eq 0 ]
         then HOME=$cabalhome cabal info $file
         fi
         # see what we are missing out on
         true ROOT FAILURE $file
         touch $buildlogs/$file.failed
    fi
    # we ought to just use a subshell
    if [ $minifile = gf ]
    then export TERM=$oldstuff
    fi
    if [ $minifile = oden-go-packages ]
    then export PATH=$oldstuff
    fi
    if [ \( $minifile = AutoForms \) -o \( $minifile = binding-wx \) ]
    then unset LD_LIBRARY_PATH
         if [ ${oldstuff-} ]
         then true ERROR restoring the old value of LD_LIBRARY_PATH
              export LD_LIBRARY_PATH="$oldstuff"
         fi
    fi
    # does this do signficantly better than compress=lzo btrfs mount option?
    #nice -19 pigz --fast -f $stracedir/$file
    # new plan: selectively manually compress some while the build happens
    # big ones: hakaru 73 GiB, 1h45m to xz compress
    # gf 2.5 GB, serialize-instances 21 GB
    # control-invariants 41 GB
    # axiomatic-classes 22 GB
    # existential 54 GB

    # lock down permissions.
    # However, chmod is ultimately only advisory as a sufficiently malicious script can change permissions to writeable prior to writing.  Future: chown to a second user with a sudo script.
    if [ ${impatient:-0} -eq 0 ]
       #maybe replace with a find.
    then
    pushd $target
    chmod -R a-w .
    chmod -R u+w workarea
    g=ghc-$ghc_version
    x=x86_64-linux-$g
    # fltkhs with bundled needs lib-separated/c/fltk-bundled
    # todo: for hackage-bin and lib-separated, unlock and relock only the $checksum dir
    chmod -R u+w hackage-bin/versioned || true "dir does not exist yet"
    for file in libexec/$x lib/$g/package.conf.d lib/$g/package.conf.d/package.cache lib-separated lib-separated/* lib-separated/*/$x share/doc/$x share/man share/man/* share/$x
    do if [ -e "$file" ]
        then chmod u+w "$file"
        fi
    done
    popd
    fi
}

function merge_extrabin () {
    pushd $extrabin
    chmod u+w ..
    mkdir ../unversioned
    true begin loop
    for file in */*
    do bname=$(basename "$file")
       ln -s ../versioned/"$file" ../unversioned/"${bname%-+([0-9.])}"
    done
    true finished loop
    popd
    mkdir $extrabin/../merged

    true begin loop
    find $extrabin -type f | perl extrabin-count.pl $extrabin | bash -x -e
    true finished loop
}

function ifupeth0 () {
    sudo ip link set eth0 up
    sleep 5
}

export PATH=$afspath/cabal2/current/bin:$afspath/buildtools-ghc/current/bin:$afspath/buildtools-more/current/bin:$afspath/arch/$ATHENA_SYS/install/$fullsuffix/bin:$PATH

#avoid initial spew when interactive
set -x