#!/bin/bash

export LANG=C

if [ "$TRUSER" == "" ] ; then
   TRUSER=test
fi

if [ "$TRSERVER" == "" ] ; then
   TRSERVER=lab.roegiest.com:33333
fi

rm -f $TRUSER

wget -O $TRUSER $TRSERVER/start/$TRUSER/$TRRUN >> $CMD.out 2>&1 || exit


TOPIC=`~/js .topic $TRUSER`
CORP=`~/js .corpus $TRUSER`
LOGIN=`~/js .runid $TRUSER`
OLDCORP=nothing

echo Running ID=$LOGIN TRRUN=$TRRUN TRTEST=$TRTEST TRUSER=$TRUSER TRSERVER=$TRSERVER  >> $CMD.out
echo ID=$LOGIN TRRUN=$TRRUN TRTEST=$TRTEST TRUSER=$TRUSER TRSERVER=$TRSERVER `date`  >> $CMD.allruns
echo   ... progress log: cmd.log/log.$LOGIN on shared drive >> $CMD.out
echo   ... interim results:  $TRSERVER/results/$LOGIN >> $CMD.out
echo   ... more details: login to VM as guest and see `pwd`/nohup.out >> $CMD.out
echo user $TRUSER `date` >> $LOG.$LOGIN
echo runid $LOGIN `date` >> $LOG.$LOGIN
echo runname $TRRUN `date` >> $LOG.$LOGIN
echo topic $TOPIC `date` >> $LOG.$LOGIN
echo corpus $CORP `date` >> $LOG.$LOGIN


while [ "$CORP" != "-1" ] ; do

echo corps $CORP $OLDCORP
if [ "$CORP" != "$OLDCORP" ] ; then

   echo newcorpus $CORP `date` >> $LOG.$LOGIN

   rm -f $CORP

   wget -O $CORP $TRSERVER/crawl/corpus/$CORP/$LOGIN
   #wget $TRSERVER/crawl/corpus/$CORP/$LOGIN
   
   #ZIPFILE=`cut -d'"' -f8 $CORP`
   ZIPFILE=`~/js .url $CORP`
   ZIPF=`echo $ZIPFILE | sed -e 's/.*\///'`
   rm -f $ZIPF
   wget $ZIPFILE

   rm -rf unzipdir
   mkdir unzipdir

   rm -rf data
   mkdir data

   if [ -e ~/featurekit ] ; then
       ~/featurekit/dofast $ZIPF
       cut -d' ' -f1 svm.fil | sed -e 's/.*/& &/' > docfil
       cut -d' ' -f1 docfil | cat -n > data/docfils
       mv svm.fil data/svm.fil
       export LAMBDA=0.0001
   else
      (cd unzipdir ; tar xf ../$ZIPF)
      find `pwd`/unzipdir -type f | sort > manifest
      sed -e 's/.*/& &/' -e 's/[^ ]*\///' manifest | sort > docfil
      sed -e 's/.*/& &/' manifest | kit/svmify > data/svm.fil
      cat -n manifest > data/docfils
      export LAMBDA=0.001
   fi

   (cd data; sed -e 's/[^ ]*/0/' svm.fil | ../kit/dosplit)
fi



rm -f $TOPIC
wget $TRSERVER/topic/need/$LOGIN/$TOPIC

~/js .need $TOPIC > $TOPIC.seed.doc
if [ -e ~/featurekit ] ; then
   ~/featurekit/dofeaturesseed $TOPIC.seed.doc unzipdir
   sed -e 's/[^ ]*/1/' svm.$TOPIC.seed.doc.fil > data/$TOPIC.seed
else
   echo 1 $TOPIC.seed.doc | kit/svmify | sed -e 's/[^ ]*/1/' > data/$TOPIC.seed
fi
touch data/rel.$TOPIC.fil
touch data/prel.$TOPIC

pushd data

L=1
R=100

rm -f new[0-9][0-9].$TOPIC tail[0-9][0-9].$TOPIC self*.$TOPIC gold*.$TOPIC
touch new00.$TOPIC

NDOCS=`cat docfils | wc -l`
NDUN=0

for x in 0 1 2 3 4 5 6 7 8 9 ; do
   for y in 0 1 2 3 4 5 6 7 8 9 ; do
   if [ $NDUN -lt $NDOCS ] ; then
      export N=$x$y
      cp $TOPIC.seed trainset
      cut -f2 docfils | sort -R | head -$R | sort | join - svm.fil | sed -e's/[^ ]*/-1/' >> trainset
      cat new[0-9][0-9].$TOPIC > seed
      cat seed | sort | join - rel.$TOPIC.fil | sed -e 's/^/1 /' > x
      cat seed | sort | join -v1 - rel.$TOPIC.fil | sort -R | head -50000 | sed -e 's/^/-1 /' >> x
      sort -k2 x | join -12 - svm.fil | cut -d' ' -f2- | sort -n >> trainset
      /home/user/sof*/src/sof*ml --learner_type logreg-pegasos --loop_type roc --lambda $LAMBDA --iterations 200000 --training_file trainset --dimensionality 1100000 --model_out svm_model
      #/home/user/svmlight/svm_learn trainset

      RES=$?
      if [ "$RES" -eq "0" ] ; then
         for z in svm.test.* ; do
            /home/user/sof*/src/sof*ml --test_file $z --dimensionality 1100000 --model_in svm_model --results_file pout.$z
            #/home/user/svmlight/svm_classify $z svm_model pout.$z
         done
      else
         rm -f pout.svm.test.*
         cut -f2 docfils | sort -R | cat -n | sort -k2 | sed -e 's/ */-/' > pout.svm.test.1
      fi
      cut -f1 pout.svm.test.* | ../kit/fixnum | cat -n | join -o2.2,1.2 -t'	' - docfils > inlr.out
      sort seed | join -v2 - inlr.out | sort -rn -k2 | cut -d' ' -f1 > new$N.$TOPIC
      cat new[0-9][0-9].$TOPIC > x
      if [ "$N" != "99" ] ; then
         head -$L new$N.$TOPIC > y ; mv y new$N.$TOPIC
      fi

#      for X in `cat new$N.$TOPIC` ; do
#         d=`echo $X | sed -e 's/.*\///'`
#         rm -f $d
#         wget $TRSERVER/judge/$LOGIN/$TOPIC/$d
#         cat $d
#         grep -q '"judgement":1' $d && echo $X >> rel.$TOPIC.fil
#      done

      sed -e 's/.*\///' -e 's/.*/"&"/' new$N.$TOPIC | tr '\n' ',' | sed -e 's/^/[/' -e 's/,$/]/' | curl -XPOST -H 'Content-Type:application/json' "$TRSERVER/judge/$LOGIN/$TOPIC" -d @- | tr '}' '\n' | grep 'judgement.:1' | cut -d'"' -f4 | sort | join -o2.2 - ../docfil >> rel.$TOPIC.fil
      sort rel.$TOPIC.fil | sed -e 's/$/ 1/' > prel.$TOPIC
      cut -d' ' -f1 prel.$TOPIC > rel.$TOPIC.fil

      NDUN=$((NDUN+L))
      L=$((L+(L+9)/10))
      if [ "$TOPIC" != "$REASONABLE" -a "`sort new*.$TOPIC | join -v1 - prel.$TOPIC | wc -l`" -ge $((1000+`cat prel.$TOPIC | wc -l`/2)) ] ; then
            curl -X POST "$TRSERVER/judge/shot/$LOGIN/$TOPIC/reasonable"
            echo "Called shot REASONABLE for topic $TOPIC at $NDUN" >> $LOG.$LOGIN
            REASONABLE="$TOPIC"
      fi
   fi
   done
done

popd

OLDCORP="$CORP"
rm -f $TOPIC
wget $TRSERVER/topic/$LOGIN/$TOPIC
NUTOPIC=`~/js .topic $TOPIC`
CORP=`~/js .corpus $TOPIC`
TOPIC=$NUTOPIC
echo newtopic $TOPIC `date` >> $LOG.$LOGIN
done

wget $TRSERVER/finalize/$LOGIN

echo done `date` >> $LOG.$LOGIN

exit 0