//Source file: D:\\Projects\\src\\yi\\iu\\imgdb\\learn\\StatModelFoldExperiment.java

package yi.iu.imgdb.learn;

import java.io.*;
import java.lang.reflect.*;
import java.util.*;

import org.jmat.gui.*;
import yi.ds.fold.*;
import yi.ds.hash.*;
import yi.util.config.*;
import yi.util.data.*;
import yi.util.io.*;
import yi.util.io.output.*;

/**
 * A statistical model tester tests the models by folds.  The instances will be
 * divided to n folds and the experiments (learn/test) will be performanced n
 * times.  Each time, one fold will be picked up for testing and other folds will
 * be merged as learning instances.
 */
public class StatModelFoldExperiment implements Serializable
{

  /**
   * n as in n-fold
   */
  int n;

  /**
   * n models for n experiments.
   */
  StatModelAdapter m_models[];

  /**
   * The training images and testing images for each fold
   */
  ArrayList m_folds;

  /**
   * The test result
   */
  StatTestRst m_statTestRst;

  ImgDataCenter m_imgDataCenter;

  LinkedList m_featureList;

  LinkedList m_imageList;

  LinkedList m_objectList;

  String m_statModelAdapterName;

  boolean once;

  /**
   * preprocess the data to add noise to avoid covariance matrix singular.
   * The noise will follow normal distribution.  NOISE_LEVEL * deviation as
   * unit one.
   */
  double NOISE_LEVEL = 1e-4;

  static final long serialVersionUID = -3294827109357082156L;

  /**
   * Performance n-fold experiments on the specified model, specified images, on
   * specified features and specified objects.
   *
   * @param model_name
   * @param data_center
   * @param img_list pass in null if want all the images in image data center
   * @param ft_list pass in null if want all the features in image data center
   * @param ft_attr_sel select specified attributes for features
   * @param obj_list pass in null if want all the objects in image data center
   * @param n
   */
  public StatModelFoldExperiment(String model_name, ImgDataCenter data_center, LinkedList img_list,
                                 LinkedList ft_list, HashMap ft_attr_sel, LinkedList obj_list, int n)
  {
    m_statModelAdapterName = model_name;
    data_center.selectAttributes(ft_attr_sel);
//    data_center.convertCIELabtoHSI();
    m_imgDataCenter = data_center;
    m_imageList = (img_list!=null)? img_list: data_center.getImageList();
    m_featureList = (ft_list!=null)? ft_list: data_center.getFeatureList();
    m_objectList = (obj_list!=null)? obj_list: data_center.getObjectList();
    this.n = Math.abs(n);
    once = (n<0);
    m_models = new StatModelAdapter[this.n];
  }

  public StatModelAdapter[] getModels() {
    return m_models;
  }

  public ArrayList getFolds() {
    return m_folds;
  }

  /**
   * Normally, training a classifier doesn't require the testing set.  But to
   * analysis the classifier's performance, testing set may be needed.  Here
   * is the temporary solution for the classifier to get the testing set
   */
  public static LinkedList testing = null;
  /**
   * Perform n-fold experiments.  Summary the test resutls from different
   * experiments and return them.
   * @return StatTestRst
   */
  public StatTestRst perform(String rst_output_file) throws Exception
  {
    // folding...
    m_folds = (new DataFolder()).nFold(m_imageList, n);

    // arguments to initialize the StatModelAdapter
    Class[] argumentTypes = {
      ImgDataCenter.class,
      LinkedList.class,
      LinkedList.class,
      LinkedList.class
    };
    // the StatModelAdapter class
    Class model_adapter_class = Class.forName(m_statModelAdapterName);
    // the constructor
    Constructor model_adapter_constructor =
             model_adapter_class.getConstructor( argumentTypes );

    // the result summary
    m_statTestRst = null;
    // the verification error summary (sum)
    DoubleScoreHashMap ver_err_sum = new DoubleScoreHashMap();

    // add some noise to the data to keep the covariance matrices from becoming singular
    System.out.println("add noise ...");
    m_imgDataCenter.addNoise(NOISE_LEVEL);

    // for each fold
    for (int i=0; i<n; i++) {

      System.out.println("fold #" + i + "\n");

      // get the training set and testing set
      TwoLinkedListData two_list = (TwoLinkedListData) m_folds.get(i);
      LinkedList learing = two_list.first;
      LinkedList testing = two_list.second;
//
//      System.out.println("use 1-4500 for training and 4501-5000 for testing");
//      LinkedList learing = new LinkedList();
//      for (int j=1; j<=4500; j++) learing.add(new String(Integer.toString(j)));
//      LinkedList testing = new LinkedList();
//      for (int j=4501; j<=5000; j++) testing.add(new String(Integer.toString(j)));


      this.testing = testing;

      // create the model adapter
      Object[] arguments = {
          m_imgDataCenter,
          learing,             // different learning set for different experiment
          m_featureList,
          m_objectList
      };


      StatModelAdapter model_adapter =
          (StatModelAdapter)model_adapter_constructor.newInstance(arguments);
      m_models[i] = model_adapter;

      // test the model
      StatTestRst rst = model_adapter.test(testing);

      // if the model provide verification error
      if (model_adapter instanceof VerificationErrorProvider) {
        DoubleScoreHashMap ver_errors = ((VerificationErrorProvider)model_adapter).getVerificationErrors();
        ver_err_sum.addScores(ver_errors);
        if (i== 0) {
          m_statTestRst = new StatTestRstWithVerErr();
          ((StatTestRstWithVerErr)m_statTestRst).setVerificationErrors(ver_err_sum);
        }
      }
      if (rst instanceof StatTestRstWithVerErr) {
        ((StatTestRstWithVerErr)m_statTestRst).setThresholds(((StatTestRstWithVerErr)rst).getThresholds());
      }

      HashMap rpcs = rst.rpcAnalysis(m_imgDataCenter); // right positive curves
      DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
      System.out.println("roc scores:");
      System.out.println(roc_scores.toStringBykeyOrder());

      if (m_statTestRst != null) {
        m_statTestRst.putAllInDeep(rst);
      } else {
        m_statTestRst = rst;
      }

      System.out.println("total testing number: " + testing.size());

//      // save the model to disk
//      if (rst_output_file != null) {
//        model_adapter.writeToFile(rst_output_file + ".m" + i + ".mdl");
//        OutputResult.printStringCollection(OutputResult.getPrintStream(
//            rst_output_file + ".l" + i + ".list"), learing, '\n');
//        OutputResult.printStringCollection(OutputResult.getPrintStream(
//            rst_output_file + ".t" + i + ".list"), testing, '\n');
//      }

      if (once) {
        break;
      }
    }

    if (rst_output_file != null) {
      writeToFile(rst_output_file + ".exp");
    }

    // if verification errors are provided
    if (ver_err_sum.size() != 0) {
      int fold_num  = once? 1: n;
      for (Iterator i=ver_err_sum.keySet().iterator(); i.hasNext();) {
        Object key = i.next();
        ver_err_sum.putScore(key, ver_err_sum.getScore(key) / fold_num);
      }
    }

    return m_statTestRst;
  }

  /**
   * Write the experiments and rsults as a java object to file
   * @param file
   */
  public void writeToFile(String file) throws java.io.IOException {
    DOU_Ser.output(file, this);
  }

  public static StatModelFoldExperiment createFromFile(String file) throws Exception {
    return (StatModelFoldExperiment) yi.util.io.input.DIU_Ser.input(file);
  }

  public final static String USAGE = "Usage: -d img_data_center_file [-a stat_model_adapter_name] [-i img_list_file] [-f feature_class_name[:attr_index[:attr_index]*]][,feature_class_name[:attr_index[:attr_index]*]]*] [-o object_name[,object_name]*] [-r rst_file_name] [-e output_experiments] [-n num_fold]";


  public static void main(String[] args) {

    try {
      // init configuration
      Configuration.init(args);

      // get the datacenter
      String data_center_name = ReadArgu.getOption('d', args);
      if (data_center_name == null) {
        System.err.println(USAGE);
        System.exit(1);
      }

      String rep_path = Configuration.getParameter("REPRESENTATION_PATH", "");
      String learn_path = Configuration.getParameter("LEARNING_PATH", "");

      ImgDataCenter data_center = ImgDataCenter.createFromFile(
         rep_path + data_center_name);

      // get the image list file name
      String img_list_file = ReadArgu.getOption('i', args);
      // read image list from a file if it is specified
      LinkedList img_list = null;
      if (img_list_file != null) {
        String[] images = yi.util.io.ReadArgu.readStrings(img_list_file);
        img_list = yi.util.data.Conversion.asLinkedList(images);
      }

      // get the feature data class names
      String ft_class_names = ReadArgu.getOption(
         'f', args);
     if (ft_class_names == null) {
       StringBuffer str = new StringBuffer();
       Object[] features = data_center.getFeatureList().toArray();
       for (int i=0; i<features.length; i++) {
         if (i!=0) str.append(",");
         str.append(features[i]);
       }
       ft_class_names = str.toString();
       System.out.println("use feature: " + ft_class_names);
     }

      // feature list
      String[] features = ft_class_names.split(",");
      LinkedList ft_list = new LinkedList();
      // the selected attributes for each feature
      HashMap ft_attr_sel = new HashMap();
      for (int i=0; i<features.length; i++) {
        StringTokenizer tokenizer = new StringTokenizer(features[i], ":");
        String ft_name = tokenizer.nextToken();
        int[] attr_sel = new int[tokenizer.countTokens()];
        int t_index = 0;
        while (tokenizer.hasMoreTokens()) {
          attr_sel[t_index]= Integer.parseInt(tokenizer.nextToken());
          t_index++;
        }
        ft_list.add(ft_name);
        ft_attr_sel.put(ft_name, attr_sel);
      }

      // get the object names
      String obj_names = ReadArgu.getOption('o', args);
      LinkedList obj_list = null;
      if (obj_names != null) {
        String obj_names_wspace = obj_names.replace('_', ' ');
        String[] objects = obj_names_wspace.split(",");
        obj_list = yi.util.data.Conversion.asLinkedList(objects);
      }

      // get the statistical model adapter name
      String stat_model_adapter_name = ReadArgu.getOption(
          'a', args
          );
      if (stat_model_adapter_name == null) {
        System.err.println(USAGE);
        System.exit(1);
      }
      String num_fold_str = ReadArgu.getOption(
         'n', args,
         "-5"
      );

      int num_fold = Integer.parseInt(num_fold_str);


      StatModelFoldExperiment experiment = new StatModelFoldExperiment(
          stat_model_adapter_name,
          data_center,
          img_list,
          ft_list,
          ft_attr_sel,
          obj_list, num_fold);

      String rst_output_file = ReadArgu.getOption('r', args);
      if (rst_output_file == null){
//        rst_output_file = yi.util.data.StringUtil.getFileSuffixNameRemoved(data_center_name) + "." +
//            yi.util.data.StringUtil.getClassShortName(stat_model_adapter_name) +
//            ((obj_names == null)? "" : ("." +obj_names));
        rst_output_file = StatTestRst.getStatTestRstOutputName(data_center_name, stat_model_adapter_name, obj_names, false, false);
      }
      boolean output_experiment = (Boolean.valueOf(ReadArgu.getOption('e', args, "false"))).booleanValue();

      if (output_experiment) {
        yi.util.io.File.makeDirectory(learn_path + rst_output_file, true);
      }
      StatTestRst rst = experiment.perform(output_experiment? learn_path + rst_output_file: null);
//      System.out.println(rst);

      HashMap rpcs = rst.rpcAnalysis(data_center); // right positive curves

      DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
      System.out.println("roc scores:");
      System.out.println(roc_scores.toStringBykeyOrder());

      // verification error print out
      if (rst instanceof StatTestRstWithVerErr) {
        DoubleScoreHashMap ver_errors = ((StatTestRstWithVerErr)rst).getVerficationErrors();
        System.out.println("verification errors:");
        System.out.println(ver_errors.toStringBykeyOrder());
      }


      Plot2DPanel pp = rst.drawROCs(rpcs);

      if (rst_output_file != null) {
        yi.util.io.File.makeDirectory(learn_path + rst_output_file, true);
        rst.writeToFile(learn_path + rst_output_file + ".rst");
        pp.setFixedBounds(0,0,1);
        pp.setFixedBounds(1,0,1);
        pp.invalidate();

        pp.toGraphicFile(new java.io.File(learn_path + rst_output_file + ".png"));
        System.exit(0);
      }


    } catch (Exception e) {
      e.printStackTrace();
    }
  }

  /**
   * call StatModelAdapter to locate the recognized object
   */
  public static class LocateObject {
    public final static String USAGE = "Usage: -e experiment_file [-c use_cross_valid_fold] [-o object]  [-y(only use positive images)] [-l(use learing images)] [-t(use testing image)] [-i image_list]";

    public static void main(String[] args) {

      // init configuration
      Configuration.init(args);

      String represent_path = Configuration.getParameter("REPRESENTATION_PATH", "");
      String learn_path = Configuration.getParameter("LEARNING_PATH", "");

      try {

        String exp_file = ReadArgu.getOption('e', args, null);
        if (exp_file == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        String cross_valid_fold_str = ReadArgu.getOption('c', args, "0");
        int cross_valid_fold = Integer.parseInt(cross_valid_fold_str);

        StatModelFoldExperiment exp = StatModelFoldExperiment.createFromFile(
            learn_path + exp_file);
        StatModelAdapter model = (StatModelAdapter)exp.m_models[cross_valid_fold];

        // object
        String obj_str = ReadArgu.getOption('o', args);
        String[] obj_list = ObjectDescriptor.parse(obj_str, exp.m_objectList);

        // feature
        FeatureDescriptor[] ft_descs = FeatureDescriptor.parse(
            ReadArgu.getOption('f', args), exp.m_imgDataCenter.getFeatureList());
        String feature = yi.util.data.StringUtil.getClassShortName(ft_descs[0].generator);

        // represent
        RepresentationDescriptor[] rep_descs = RepresentationDescriptor.parse(
            ReadArgu.getOption('r', args), exp.m_imgDataCenter.getFeatureList());

        // image list
        LinkedList img_list = null;

        boolean use_learning_images = Boolean.valueOf(ReadArgu.getOption('l', args)).booleanValue();
        if (use_learning_images) {
          img_list = ((TwoLinkedListData)exp.m_folds.get(cross_valid_fold)).first;
        }
        else {
          boolean use_testing_images = Boolean.valueOf(ReadArgu.getOption('t', args)).booleanValue();
          if (use_testing_images) {
            img_list = ((TwoLinkedListData)exp.m_folds.get(cross_valid_fold)).second;
          } else {
            // get the image list file name
            String img_list_file = ReadArgu.getOption('i', args);

            if (img_list_file != null) {
              String[] images = yi.util.io.ReadArgu.readStrings(img_list_file);
              img_list = yi.util.data.Conversion.asLinkedList(images);
            }
            else {
              img_list = exp.m_imgDataCenter.getImageList();
            }
          }
        }
        model.m_imgDataCenter = exp.m_imgDataCenter;
        boolean use_positive_images = Boolean.valueOf(ReadArgu.getOption('y', args)).booleanValue();
        if (use_positive_images) {
          String obj = obj_list[0];
          for (Iterator i=img_list.iterator(); i.hasNext(); ) {
            String img = (String)i.next();
            LinkedList objs = model.m_imgDataCenter.getImgDescriptions(img);
            if (!objs.contains(obj)) {
              i.remove();
            }
          }
        }


//        model.locateObject(img_list, yi.util.data.Conversion.toLinkedList(obj_list), ft_descs[0], rep_descs[0],
//                           StringUtil.getFileSuffixNameRemoved(exp_file) + ".c." + cross_valid_fold);

        model.locateObject(img_list, yi.util.data.Conversion.toLinkedList(obj_list), ft_descs, rep_descs,
                           StringUtil.getFileSuffixNameRemoved(exp_file) + ".c." + cross_valid_fold, 128);
      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

  public static class Test {

    public final static String USAGE = "Usage: -e experiment_file [-c use_cross_valid_fold] [-l(use learing images)] [-t(use testing image)] [-i img_list_file] [-r rst_file_name]";

    public static void main(String[] args) {

      // init configuration
      Configuration.init(args);

      String represent_path = Configuration.getParameter("REPRESENTATION_PATH", "");
      String learn_path = Configuration.getParameter("LEARNING_PATH", "");

      try {
        String exp_file = ReadArgu.getOption('e', args, null);
        if (exp_file == null) {
          System.err.println(USAGE);
          System.exit(1);
        }

        String cross_valid_fold_str = ReadArgu.getOption('c', args, "0");
        int cross_valid_fold = Integer.parseInt(cross_valid_fold_str);

        StatModelFoldExperiment exp = StatModelFoldExperiment.createFromFile(
            learn_path + exp_file);
        StatModelAdapter model = (StatModelAdapter)exp.m_models[cross_valid_fold];
//        // experiment for CVPR 2005
//        GMMNNAdapter model1 = (GMMNNAdapter)exp.m_models[cross_valid_fold];
//        GMMNNAdapterNoCompMean model = new GMMNNAdapterNoCompMean(model1.m_imgDataCenter, model1.m_imageList
//            , model1.m_featureList, model1.m_objectList);
//        model.m_gmmModels = model1.m_gmmModels;

        // read in the datacenter
        ImgDataCenter data_center = exp.m_imgDataCenter;

        // get the image list file name
        // image list
        LinkedList img_list = null;

        boolean use_learning_images = Boolean.valueOf(ReadArgu.getOption('l', args)).booleanValue();
        if (use_learning_images) {
          img_list = ((TwoLinkedListData)exp.m_folds.get(cross_valid_fold)).first;
        }
        else {
          boolean use_testing_images = Boolean.valueOf(ReadArgu.getOption('t', args)).booleanValue();
          if (use_testing_images) {
            img_list = ((TwoLinkedListData)exp.m_folds.get(cross_valid_fold)).second;
          } else {
            // get the image list file name
            String img_list_file = ReadArgu.getOption('i', args);

            if (img_list_file != null) {
              String[] images = yi.util.io.ReadArgu.readStrings(img_list_file);
              img_list = yi.util.data.Conversion.asLinkedList(images);
            }
            else {
              img_list = model.m_imageList;
            }
          }
        }

        model.VERBOSE_IN_TEST = true;
        StatTestRst rst = model.test(img_list);

        HashMap rpcs = rst.rpcAnalysis(data_center); // right positive curves


        DoubleScoreHashMap roc_scores = rst.getROCScores(rpcs);
        System.out.println("roc scores:");
        System.out.println(roc_scores.toStringBykeyOrder());

        Plot2DPanel pp = rst.drawROCs(rpcs);

        String rst_output_file = ReadArgu.getOption('r', args);
        if (rst_output_file != null) {
          pp.setFixedBounds(0, 0, 1);
          pp.setFixedBounds(1, 0, 1);
          pp.invalidate();
          pp.toGraphicFile(new java.io.File(learn_path + rst_output_file + ".png"));
          rst.writeToFile(learn_path + rst_output_file);
        }

      } catch (Exception e) {
        e.printStackTrace();
      }
    }
  }

}
